From 74329d2e4227baf79c93fd7ee777a443bb232f48 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Mon, 26 Apr 2021 20:14:08 -0400 Subject: [PATCH 1/3] Clarion clio 4b4f5418d20b06e739ccd9a191fc2ce0a5399b89 --- libraries/clio/CMakeLists.txt | 54 ++ libraries/clio/include/clio/bytes.hpp | 13 + .../clio/include/clio/bytes/from_json.hpp | 12 + libraries/clio/include/clio/bytes/to_bin.hpp | 6 + libraries/clio/include/clio/bytes/to_json.hpp | 12 + libraries/clio/include/clio/compress.hpp | 157 ++++ libraries/clio/include/clio/error.hpp | 17 + libraries/clio/include/clio/flatbuf.hpp | 748 +++++++++++++++++ libraries/clio/include/clio/fpconv.h | 336 ++++++++ libraries/clio/include/clio/from_bin.hpp | 288 +++++++ .../clio/include/clio/from_bin/varint.hpp | 18 + libraries/clio/include/clio/from_json.hpp | 762 ++++++++++++++++++ .../clio/include/clio/from_json/varint.hpp | 18 + libraries/clio/include/clio/from_protobuf.hpp | 187 +++++ libraries/clio/include/clio/get_type_name.hpp | 172 ++++ libraries/clio/include/clio/gql.hpp | 96 +++ libraries/clio/include/clio/json/any.hpp | 188 +++++ libraries/clio/include/clio/member_proxy.hpp | 86 ++ libraries/clio/include/clio/murmur.hpp | 55 ++ libraries/clio/include/clio/name.hpp | 198 +++++ libraries/clio/include/clio/powers.h | 77 ++ libraries/clio/include/clio/protobuf/any.hpp | 143 ++++ libraries/clio/include/clio/protobuf/json.hpp | 172 ++++ .../clio/include/clio/protobuf/query.hpp | 201 +++++ .../clio/include/clio/protobuf/schema.hpp | 187 +++++ libraries/clio/include/clio/reflect.hpp | 388 +++++++++ libraries/clio/include/clio/schema.hpp | 288 +++++++ libraries/clio/include/clio/stream.hpp | 323 ++++++++ libraries/clio/include/clio/to_bin.hpp | 172 ++++ libraries/clio/include/clio/to_bin/deque.hpp | 12 + libraries/clio/include/clio/to_bin/list.hpp | 12 + libraries/clio/include/clio/to_bin/map.hpp | 12 + libraries/clio/include/clio/to_bin/set.hpp | 12 + libraries/clio/include/clio/to_bin/varint.hpp | 17 + libraries/clio/include/clio/to_json.hpp | 283 +++++++ libraries/clio/include/clio/to_json/map.hpp | 28 + .../clio/include/clio/to_json/varint.hpp | 17 + libraries/clio/include/clio/to_key.hpp | 303 +++++++ libraries/clio/include/clio/to_protobuf.hpp | 152 ++++ libraries/clio/include/clio/translator.hpp | 402 +++++++++ libraries/clio/include/clio/tuple.hpp | 85 ++ libraries/clio/include/clio/varint.hpp | 432 ++++++++++ libraries/clio/tests/CMakeLists.txt | 23 + libraries/clio/tests/benchmark.cpp | 404 ++++++++++ libraries/clio/tests/clio_tests.cpp | 2 + libraries/clio/tests/crypto.cpp | 66 ++ libraries/clio/tests/database.cpp | 727 +++++++++++++++++ libraries/clio/tests/db.cpp | 222 +++++ libraries/clio/tests/flat_views.cpp | 232 ++++++ libraries/clio/tests/gql.cpp | 66 ++ libraries/clio/tests/protobuf_query.cpp | 77 ++ 51 files changed, 8960 insertions(+) create mode 100644 libraries/clio/CMakeLists.txt create mode 100644 libraries/clio/include/clio/bytes.hpp create mode 100644 libraries/clio/include/clio/bytes/from_json.hpp create mode 100644 libraries/clio/include/clio/bytes/to_bin.hpp create mode 100644 libraries/clio/include/clio/bytes/to_json.hpp create mode 100644 libraries/clio/include/clio/compress.hpp create mode 100644 libraries/clio/include/clio/error.hpp create mode 100644 libraries/clio/include/clio/flatbuf.hpp create mode 100644 libraries/clio/include/clio/fpconv.h create mode 100644 libraries/clio/include/clio/from_bin.hpp create mode 100644 libraries/clio/include/clio/from_bin/varint.hpp create mode 100644 libraries/clio/include/clio/from_json.hpp create mode 100644 libraries/clio/include/clio/from_json/varint.hpp create mode 100644 libraries/clio/include/clio/from_protobuf.hpp create mode 100644 libraries/clio/include/clio/get_type_name.hpp create mode 100644 libraries/clio/include/clio/gql.hpp create mode 100644 libraries/clio/include/clio/json/any.hpp create mode 100644 libraries/clio/include/clio/member_proxy.hpp create mode 100644 libraries/clio/include/clio/murmur.hpp create mode 100644 libraries/clio/include/clio/name.hpp create mode 100644 libraries/clio/include/clio/powers.h create mode 100644 libraries/clio/include/clio/protobuf/any.hpp create mode 100644 libraries/clio/include/clio/protobuf/json.hpp create mode 100644 libraries/clio/include/clio/protobuf/query.hpp create mode 100644 libraries/clio/include/clio/protobuf/schema.hpp create mode 100644 libraries/clio/include/clio/reflect.hpp create mode 100644 libraries/clio/include/clio/schema.hpp create mode 100644 libraries/clio/include/clio/stream.hpp create mode 100644 libraries/clio/include/clio/to_bin.hpp create mode 100644 libraries/clio/include/clio/to_bin/deque.hpp create mode 100644 libraries/clio/include/clio/to_bin/list.hpp create mode 100644 libraries/clio/include/clio/to_bin/map.hpp create mode 100644 libraries/clio/include/clio/to_bin/set.hpp create mode 100644 libraries/clio/include/clio/to_bin/varint.hpp create mode 100644 libraries/clio/include/clio/to_json.hpp create mode 100644 libraries/clio/include/clio/to_json/map.hpp create mode 100644 libraries/clio/include/clio/to_json/varint.hpp create mode 100644 libraries/clio/include/clio/to_key.hpp create mode 100644 libraries/clio/include/clio/to_protobuf.hpp create mode 100644 libraries/clio/include/clio/translator.hpp create mode 100644 libraries/clio/include/clio/tuple.hpp create mode 100644 libraries/clio/include/clio/varint.hpp create mode 100644 libraries/clio/tests/CMakeLists.txt create mode 100644 libraries/clio/tests/benchmark.cpp create mode 100644 libraries/clio/tests/clio_tests.cpp create mode 100644 libraries/clio/tests/crypto.cpp create mode 100644 libraries/clio/tests/database.cpp create mode 100644 libraries/clio/tests/db.cpp create mode 100644 libraries/clio/tests/flat_views.cpp create mode 100644 libraries/clio/tests/gql.cpp create mode 100644 libraries/clio/tests/protobuf_query.cpp diff --git a/libraries/clio/CMakeLists.txt b/libraries/clio/CMakeLists.txt new file mode 100644 index 000000000..45e45a802 --- /dev/null +++ b/libraries/clio/CMakeLists.txt @@ -0,0 +1,54 @@ +add_library( clio INTERFACE ) +target_link_libraries( clio INTERFACE rapidjson ) +target_include_directories( clio INTERFACE include ${Boost_INCLUDE_DIRS} ) + +target_sources( clio INTERFACE + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/fpconv.h + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/powers.h + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/compress.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/error.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/reflect.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/schema.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/translator.hpp + + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/stream.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/to_key.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/tuple.hpp + + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/flatbuf.hpp + + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/to_json.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/to_json/map.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/to_json/varint.hpp + + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/json/any.hpp + + + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/to_bin.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/to_bin/map.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/to_bin/set.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/to_bin/list.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/to_bin/deque.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/to_bin/varint.hpp + + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/from_bin.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/from_json.hpp + + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/varint.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/from_bin/varint.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/from_json/varint.hpp + + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/bytes.hpp + + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/bytes/from_json.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/bytes/to_json.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/bytes/to_bin.hpp + + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/from_protobuf.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/to_protobuf.hpp + + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/protobuf/any.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/protobuf/json.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/protobuf/query.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/protobuf/schema.hpp +) diff --git a/libraries/clio/include/clio/bytes.hpp b/libraries/clio/include/clio/bytes.hpp new file mode 100644 index 000000000..e02cf8020 --- /dev/null +++ b/libraries/clio/include/clio/bytes.hpp @@ -0,0 +1,13 @@ +#pragma once +#include +#include + +namespace clio { + +struct bytes { + std::vector data; +}; + +CLIO_REFLECT(bytes, data); + +} // namespace clio diff --git a/libraries/clio/include/clio/bytes/from_json.hpp b/libraries/clio/include/clio/bytes/from_json.hpp new file mode 100644 index 000000000..216722542 --- /dev/null +++ b/libraries/clio/include/clio/bytes/from_json.hpp @@ -0,0 +1,12 @@ +#pragma once +#include +#include + +namespace clio { + +template +void from_json(bytes& obj, S& stream) { + clio::from_json_hex(obj.data, stream); +} + +} // namespace clio diff --git a/libraries/clio/include/clio/bytes/to_bin.hpp b/libraries/clio/include/clio/bytes/to_bin.hpp new file mode 100644 index 000000000..16e44ea6e --- /dev/null +++ b/libraries/clio/include/clio/bytes/to_bin.hpp @@ -0,0 +1,6 @@ +#pragma once +#include + +namespace clio { + +} diff --git a/libraries/clio/include/clio/bytes/to_json.hpp b/libraries/clio/include/clio/bytes/to_json.hpp new file mode 100644 index 000000000..6a3cd258e --- /dev/null +++ b/libraries/clio/include/clio/bytes/to_json.hpp @@ -0,0 +1,12 @@ +#pragma once +#include +#include + +namespace clio { + +template +void to_json(const bytes& obj, S& stream) { + clio::to_json_hex(obj.data.data(), obj.data.size(), stream); +} + +} // namespace clio diff --git a/libraries/clio/include/clio/compress.hpp b/libraries/clio/include/clio/compress.hpp new file mode 100644 index 000000000..1bac24fe3 --- /dev/null +++ b/libraries/clio/include/clio/compress.hpp @@ -0,0 +1,157 @@ +#pragma once +#include + +namespace clio { + +template +void capp_unpack( InStream& in, OutStream& out ) { + while( in.remaining() ) { + uint8_t word_bits; + in.read(&word_bits,1); + + const uint64_t zero_word = 0; + + if( word_bits == 0x00 ) { + out.write(&zero_word,8); + + uint8_t zero_words; + in.read( &zero_words, 1 ); + + while( zero_words ) { + out.write(&zero_word,8); + --zero_words; + } + if( in.pos >= in.end ) { + throw_error( clio::stream_error::overrun ); + } + } else if( word_bits == 0xff ) { + if( in.remaining() < 8 ) { + out.write( in.pos, in.remaining() ); + in.skip( in.remaining() ); + return; + } + + uint64_t word; + in.read( &word, sizeof(word) ); + out.write( &word, 8 ); + + uint8_t raw_words; + in.read( &raw_words,1); + if( raw_words ) { + if( in.remaining() < uint32_t(raw_words * 8) ) + throw_error( clio::stream_error::overrun ); + out.write( in.pos, uint32_t(raw_words) * 8 ); + in.pos += uint32_t(raw_words) * 8; + } + } else { + uint64_t out_word = 0; + uint8_t* out_p = (uint8_t*)&out_word; + for( auto i = 7; i >= 0; --i ) { + if( word_bits & (1< +void capp_pack( InStream& in, OutStream& out ) { + while( in.remaining() >= 8 ) { + uint64_t next_word; + in.read_raw(next_word); + + if( not next_word ) { + out.write(char(0)); + + uint32_t num_zero = 0; + in.read_raw(next_word); + + while( not next_word && num_zero < 254) { + in.read_raw(next_word); + ++num_zero; + } + out.write( char(num_zero) ); + } + + + uint8_t* inbyte = (uint8_t*)&next_word; + uint8_t* endin = inbyte + 8; + + uint8_t temp[9]; + uint8_t& zeros = temp[0]; + uint8_t* outbyte = &temp[1]; + zeros = 0; + + do { + zeros <<= 1; + if( bool(*inbyte) ) { + zeros += 1; + *outbyte = *inbyte; + ++outbyte; + } + ++inbyte; + } while( inbyte != endin ); + out.write( temp, outbyte-temp ); + + if( zeros == 0xff ) { + uint8_t num_raw = 0; + auto* p = in.pos; + auto* e = in.end; + while( *p and p != e ) ++p; + + num_raw = (p - in.pos)/8; + out.write( num_raw ); + out.write( in.pos, num_raw * 8 ); + in.skip( num_raw * 8 ); + + // in.read_raw(next_word); + /// TODO: once we enter "raw" mode we shouldn't exit until + /// we have seen a certain percentage of zero bytes... for example, + /// one zero per word isn't worth encoding... + } + + + } + if( in.remaining() ) { + out.write( char(0xff) ); /// indicate all bytes are present, receiver will truncate + out.write( in.pos, in.remaining() ); + in.skip( in.remaining() ); + } + return; +}; + +inline std::vector capn_compress( const std::vector& c ) { + clio::input_stream in( c.data(), c.size() ); + /* + clio::size_stream sizess; + capp_unpack( in, sizess); + in.pos = c.data(); + */ + + std::vector out; + out.reserve( c.size() ); + clio::vector_stream vout(out); + capp_pack( in, vout ); + return out; +} +inline std::vector capn_uncompress( const std::vector& c ) { + clio::input_stream in( c.data(), c.size() ); + /* + clio::size_stream sizess; + capp_unpack( in, sizess); + in.pos = c.data(); + */ + + std::vector out; + out.reserve( c.size()*1.5 ); + clio::vector_stream vout(out); + capp_unpack( in, vout ); + return out; +} + +} /// clio diff --git a/libraries/clio/include/clio/error.hpp b/libraries/clio/include/clio/error.hpp new file mode 100644 index 000000000..010f1a489 --- /dev/null +++ b/libraries/clio/include/clio/error.hpp @@ -0,0 +1,17 @@ +#pragma once + +namespace clio { + /** + * On some platforms we need to disable + * exceptions and do something else, this + * wraps that. + */ + template + [[noreturn]] void throw_error( T&& e ){ + #ifdef __cpp_exceptions + throw e; + #else + abort(); + #endif + } +}; diff --git a/libraries/clio/include/clio/flatbuf.hpp b/libraries/clio/include/clio/flatbuf.hpp new file mode 100644 index 000000000..077c9f237 --- /dev/null +++ b/libraries/clio/include/clio/flatbuf.hpp @@ -0,0 +1,748 @@ +#pragma once +#include +#include +#include + +namespace clio { + + struct flat_view_proxy_impl; + + template + class flat_ptr; + + template + struct is_flat_ptr : std::false_type {}; + + template + struct is_flat_ptr> : std::true_type { using value_type = T; }; + + template + class flat { + using T::flat_type_not_defined; + }; + + template<> + class flat; + + template + class flat >; + + + template + class flat>; + + template + auto get_view_type() { +// if constexpr( is_flat_ptr::value ) { +// return get_view_type(); + if constexpr( reflect::is_struct ) { + using view_type = typename reflect::template proxy; + return (view_type*)(nullptr); + } + else if constexpr( std::is_trivially_copyable::value ) + return (T*)(nullptr); + else + return (flat*)nullptr; + } + + template + using flat_view = std::remove_pointer_t())>; + + struct offset_ptr { + uint32_t offset; + + template + auto get()const; + }; + + template + constexpr bool contains_offset_ptr(); + + template + struct tuple_contains_offset; + + + template + constexpr uint32_t get_contains_offset_ptr() { + if constexpr( sizeof...(Ts) == 0 ) + return contains_offset_ptr(); + else { + return contains_offset_ptr() | + get_contains_offset_ptr(); + } + } + + template + struct tuple_contains_offset< std::tuple > { + static constexpr const auto value = get_contains_offset_ptr(); + }; + + + /** + * Recursively checks the types for any field which requires dynamic allocation, + */ + template + constexpr bool contains_offset_ptr() { + if constexpr ( is_flat_ptr::value ) { + return true; + } else if constexpr( is_std_tuple::value ) { + return tuple_contains_offset::value; + } else if constexpr( is_std_variant::value ) { + return contains_offset_ptr::alts_as_tuple>(); + } else if constexpr( std::is_same_v ) { + return true; + } else if constexpr( is_std_vector::value ) { + return true; + } else if constexpr( clio::reflect::is_struct ) { + bool is_flat = true; + clio::reflect::for_each( [&]( const clio::meta& ref, auto mptr ){ + using member_type = std::decay_t; + is_flat &= not contains_offset_ptr(); + }); + return not is_flat; + } else if constexpr( std::is_arithmetic_v ) { + return false; + } else if constexpr( std::is_trivially_copyable::value ) { + return false; + } else { + T::contains_offset_ptr_not_defined; + } + } + + + template + constexpr uint32_t flatpack_size() { + if constexpr( is_flat_ptr::value ) { + return sizeof(offset_ptr); + } else if constexpr( is_std_variant::value ) { + return 16; + } else if constexpr( reflect::is_struct ) { + uint32_t size = 0; + reflect::for_each( [&]( const meta& ref, const auto& mptr ){ + using member_type = decltype(result_of_member(mptr)); + if constexpr ( contains_offset_ptr() ) { + size += sizeof(offset_ptr); + } else { + size += flatpack_size(); + } + }); + return size; + } else if constexpr( std::is_same_v || is_std_vector::value) { + return sizeof(offset_ptr); + } else if constexpr( std::is_arithmetic_v ) { + return sizeof(T); + } else if constexpr( std::is_trivially_copyable::value ) { + return sizeof(T); + } else { + T::flatpack_size_not_defined; + } + } + + template + struct get_tuple_offset; + + template + constexpr uint32_t get_offset() { + static_assert( Idx < sizeof...(Ts) + 1, "index out of range" ); + if constexpr ( Idx == 0 ) return 0; + else if constexpr( sizeof...(Ts) == 0 ) + if constexpr( contains_offset_ptr() ) return 4; + /// if contains ptr then 8 else flatpack_size... + else return flatpack_size(); + // return get_flat_size( ((const First*)(nullptr)) ); + else { + if constexpr( contains_offset_ptr() ) + return get_offset() + 4; //flatpack_size(); + else return get_offset() + flatpack_size(); + // return get_offset< Idx-1, Ts...>() + get_flat_size( ((const First*)(nullptr)) ); + } + } + + template + struct get_tuple_offset< I, std::tuple > { + static constexpr const uint32_t value = get_offset(); + }; + + + struct flat_view_proxy_impl { + + /** This method is called by the reflection library to get the field */ + template + constexpr auto get() { + using class_type = decltype( clio::class_of_member(MemberPtr) ); + using tuple_type = typename clio::reflect::struct_tuple_type; + using member_type = decltype( clio::result_of_member(MemberPtr) ); + + constexpr uint32_t offset = clio::get_tuple_offset::value; + + char* out_ptr = reinterpret_cast(this)+offset; + + if constexpr ( contains_offset_ptr() ) { + clio::offset_ptr* ptr = reinterpret_cast(out_ptr); + return ptr->get(); + } else { + return reinterpret_cast(out_ptr); + } + } + + template + constexpr const auto get()const { + using class_type = decltype( clio::class_of_member(MemberPtr) ); + using tuple_type = typename clio::reflect::struct_tuple_type; + using member_type = decltype( clio::result_of_member(MemberPtr) ); + + constexpr uint32_t offset = clio::get_tuple_offset::value; + + auto out_ptr = reinterpret_cast(this)+offset; + + if constexpr ( contains_offset_ptr() ) { + const clio::offset_ptr* ptr = reinterpret_cast(out_ptr); + return ptr->get(); + } else { + return reinterpret_cast(out_ptr); + } + } + + }; + + /** + * A flat view of a flat pointer. + */ + template + class flat > { + public: + auto get() { + return reinterpret_cast< decltype( get_view_type() ) >(_data); + } + private: + uint32_t _size = 0; + char _data[]; + }; + + + template<> + class flat { + public: + uint32_t size()const { return _size; } + const char* c_str()const { return _size ? _data : (const char*)this; } + const char* data()const { return _size ? _data : nullptr; } + char* data() { return _size ? _data : nullptr; } + + operator std::string_view()const { return std::string_view( _data, _size ); } + + template + friend S& operator << ( S& stream, const flat& str ) { + return stream << str.c_str(); + } + private: + uint32_t _size = 0; + char _data[]; + }; + + template + class flat> { + public: + uint64_t type = 0; + uint32_t flat_data = 0; + offset_ptr offset_data; + + flat() { + static_assert( sizeof(flat) == 16 ); + static_assert( std::is_trivially_copyable< flat >::value ); + } + + int64_t index_from_type()const { + return get_index_from_type(); + } + + void init_variant( std::variant& v ) { + _init_variant( v ); + } + + template + void visit( Visitor&& v ) { + _visit_variant( std::forward(v) ); + } + private: + template + int64_t get_index_from_type()const { + if constexpr( sizeof...(Rest) == 0 ) { + return get_type_hashname() != type; + } else { + if( get_type_hashname() == type ) + return 0; + else return 1 + get_index_from_type(); + } + } + + template + void _init_variant( std::variant& v )const { + if( get_type_hashname() == type ) + v = First(); + else if constexpr( sizeof...(Rest) > 0 ) { + _init_variant( v ); + } + } + template + void _visit_variant( Visitor&& v )const { + if( get_type_hashname() == type ) { + if constexpr ( not get_contains_offset_ptr() and flatpack_size() <= 8 ) { + v( *((const flat_view*)&flat_data) ); + } else { + v( *((const flat_view*) ( ((char*)(&offset_data)) + offset_data.offset ) ) ); + } + } + else if constexpr( sizeof...(Rest) > 0 ) { + _visit_variant( std::forward(v) ); + } + } + }; + + + /// T == value of the array elements + template + class flat> { + public: + auto& operator[]( uint32_t index ) { + if( index >= _size ) + throw_error( stream_error::overrun ); + /** in this case the data is a series of offset_ptr<> */ + if constexpr( std::is_same::value ) { + auto ptr_array = reinterpret_cast(_data); + return *reinterpret_cast*>( ptr_array[index].get< std::vector >() ); + } else if constexpr( contains_offset_ptr() ) { + auto ptr_array = reinterpret_cast(_data); + return *reinterpret_cast*>( ptr_array[index].get< std::vector >() ); + } else if constexpr ( reflect::is_struct ) { /// the data is a series of packed T + const auto offset = index * flatpack_size(); + return *reinterpret_cast*>( &_data[offset] ); + } else if constexpr ( std::is_trivially_copyable::value ) { + auto T_array = reinterpret_cast(_data); + return T_array[index]; + } else { + T::is_not_a_known_flat_type; + } + } + const auto& operator[]( uint32_t index )const { + if( index >= _size ) + throw_error( stream_error::overrun ); + + /** in this case the data is a series of offset_ptr<> */ + if constexpr( std::is_same::value ) { + auto ptr_array = reinterpret_cast(_data); + return *reinterpret_cast*>( ptr_array[index].get< std::vector >() ); + }else if constexpr( contains_offset_ptr() ) { + auto ptr_array = reinterpret_cast(_data+sizeof(offset_ptr)*index); + const auto& r = *reinterpret_cast*>( ptr_array->get< std::vector >() ); + return r; + } else if constexpr ( reflect::is_struct ) { /// the data is a series of packed T + const auto offset = index * flatpack_size(); + return *reinterpret_cast*>( &_data[offset] ); + } else if constexpr ( std::is_trivially_copyable::value ) { + auto T_array = reinterpret_cast(_data); + return T_array[index]; + } else { + T::is_not_a_known_flat_type; + } + } + + + uint32_t size()const { return _size; } + + private: + uint32_t _size = 0; + char _data[]; + }; + + template + auto offset_ptr::get()const { + const auto ptr = ((char*)this)+offset; + if constexpr( is_flat_ptr::value ) { + return reinterpret_cast< decltype(get_view_type()) >(ptr+4); + } else if constexpr( reflect::is_struct ) { + return reinterpret_cast*>(ptr); + } else if constexpr( std::is_same_v ) { + return reinterpret_cast< flat* >(ptr); + } else if constexpr( is_std_vector::value ) { + return reinterpret_cast< flat* >(ptr); + } else { + T::is_not_reflected_for_offset_ptr; + } + } + + + /** + * Verifies that the type pointed at could be unpacked without a buffer overflow, + * which means that all offset pointers are in bounds. It does this by unpacking + * the stream without allocating any memory. + */ + struct validate_input_stream : public input_stream { + using input_stream::input_stream; + }; + + template + void flatcheck( InputStream& stream ) { + if constexpr ( is_flat_ptr::value ) { + uint32_t size; + stream.read( &size, sizeof(size) ); + stream.skip( size ); + } else if constexpr ( is_std_variant::value ) { + flat fv; + stream.read( &fv, sizeof(fv) ); + + T temp; /// this could do memory allocation for certain types... but hopefully won't, + /// this is used to do std::visit in the next line... in theory we could do + /// a dispatcher that only deals in types and not values. + fv.init_variant( temp ); + std::visit( [&]( auto& iv ){ + using item_type = std::decay_t< decltype(iv) >; + if constexpr ( get_contains_offset_ptr() ) { + /// the stream.pos is at the END of reading the variant, which + /// should be the same as the end of flat::offset_data + InputStream in( stream.pos + fv.offset_data.offset-sizeof(offset_ptr), stream.end ); + flatunpack( iv, in ); + } else if constexpr ( flatpack_size() <= 8 ){ + } else { + InputStream in( stream.pos + fv.offset_data.offset-sizeof(offset_ptr), stream.end ); + flatunpack( iv, in ); + } + }, temp); + /// maybe deref pointer.. + } else if constexpr ( std::is_same_v ) { + uint32_t size; + stream.read( &size, sizeof(size) ); + stream.skip( size ); + } else if constexpr ( is_std_vector::value ) { + uint32_t size; + stream.read( &size, sizeof(size) ); + if constexpr( contains_offset_ptr< typename is_std_vector::value_type >() ) { + auto start = stream.pos; + stream.skip( size * sizeof( offset_ptr ) ); + + offset_ptr* ptr = (offset_ptr*)(start); + for( uint32_t i = 0; i < size; ++i ) { + InputStream in( ((char*)ptr) + ptr->offset, stream.end ); + flatcheck(in); + } + } else { + stream.skip( size * flatpack_size() ); + } + } else if constexpr ( reflect::is_struct ) { + if constexpr( contains_offset_ptr< T >() ) { + reflect::for_each( [&]( const meta& ref, const auto& mptr ){ + using member_type = decltype(result_of_member(mptr)); + + if constexpr ( contains_offset_ptr() ) { + offset_ptr ptr; + stream.read(&ptr,sizeof(ptr)); + + InputStream substream( stream.pos+ptr.offset-sizeof(ptr), stream.end ); + flatcheck( substream ); + } else { + stream.skip( flatpack_size() ); + } + }); + } else { + stream.skip( flatpack_size() ); + } + } else if constexpr( std::is_trivially_copyable::value ) { + stream.skip( sizeof(T) ); + } else { + T::is_not_defined; + } + } + + template + void flatunpack( T& v, S& stream ) { + if constexpr ( is_flat_ptr::value ) { + uint32_t size; + stream.read( &size, sizeof(size) ); + v.reset(size); + stream.read( v.data(), size ) ; + } else if constexpr ( is_std_variant::value ) { + flat fv; + stream.read( &fv, sizeof(fv) ); + fv.init_variant( v ); + std::visit( [&]( auto& iv ){ + using item_type = std::decay_t< decltype(iv) >; + if constexpr ( get_contains_offset_ptr() ) { + /// the stream.pos is at the END of reading the variant, which + /// should be the same as the end of flat::offset_data + input_stream in( stream.pos + fv.offset_data.offset-sizeof(offset_ptr), stream.end ); + flatunpack( iv, in ); + } else if constexpr ( flatpack_size() <= 8 ){ + input_stream st( (const char*)&fv.flat_data, 8 ); + flatunpack( iv, st ); + } else { + input_stream in( stream.pos + fv.offset_data.offset-sizeof(offset_ptr), stream.end ); + flatunpack( iv, in ); + } + }, v); + } else if constexpr ( std::is_same_v ) { + uint32_t size; + stream.read( &size, sizeof(size) ); + v.resize(size); + stream.read( v.data(), size ) ; + stream.skip(1); // null + } else if constexpr ( is_std_vector::value ) { + uint32_t size; + stream.read( &size, sizeof(size) ); + v.resize( size ); + + if constexpr( contains_offset_ptr< typename is_std_vector::value_type >() ) { + for( auto& item : v ) { + offset_ptr ptr; + stream.read(&ptr,sizeof(ptr)); + + /// TODO: we don't know the size of the buffer here...is it safe? + input_stream in( stream.pos + ptr.offset-sizeof(ptr), stream.end ); + flatunpack( item, in ); + } + } else { + for( auto& item : v ) { + flatunpack( item, stream ); + } + } + } else if constexpr ( reflect::is_struct ) { + reflect::for_each( [&]( const meta& ref, const auto& mptr ){ + auto& member = v.*mptr; + using member_type = decltype(result_of_member(mptr)); + + if constexpr ( contains_offset_ptr() ) { + offset_ptr ptr; + stream.read(&ptr,sizeof(ptr)); + + input_stream substream( stream.pos+ptr.offset-sizeof(ptr), stream.end ); + flatunpack( member, substream ); + } else { + flatunpack( member, stream ); + } + }); + } else if constexpr( std::is_trivially_copyable::value ) { + stream.read( (char*)&v, sizeof(v) ); + } else { + T::is_not_defined; + } + } + + template + uint32_t flatpack( const T& v, S& stream ) { + uint32_t alloc_pos = 0; + uint32_t cur_pos = 0; + + if constexpr ( is_flat_ptr::value ) { + uint32_t size = v.size(); + stream.write( &size, sizeof(size) ); + cur_pos += sizeof(size); + + if( size ) { + stream.write( v.data(), v.size() ); + } + } else if constexpr ( is_std_variant::value ) { + alloc_pos = flatpack_size(); + std::visit( [&]( const auto& iv ){ + using item_type = std::decay_t< decltype(iv) >; + flat fv; + fv.type = get_type_hashname(); + if constexpr ( not get_contains_offset_ptr() and flatpack_size() <= 8 ) { + fixed_buf_stream st( (char*)&fv.flat_data, 8 ); + flatpack( iv, st ); + + static_assert( sizeof(fv) == 16 ); + stream.write( &fv, sizeof(fv) ); + } else { + fv.offset_data.offset = (alloc_pos - (cur_pos+12)); //+8 because we haven't written the type yet / pad + size_stream size_str; + flatpack( iv, size_str ); + alloc_pos += size_str.size; + + static_assert( sizeof(fv) == 16 ); + stream.write( &fv, sizeof(fv) ); + + if constexpr( std::is_same_v ) { + stream.skip( size_str.size ); /// we already calculated this above + } + else { + /// now pack the member into the allocated spot + fixed_buf_stream substream( stream.pos+fv.offset_data.offset-sizeof(fv.offset_data), + size_str.size ); + if( substream.end > stream.end ) + throw_error( stream_error::overrun ); + flatpack( iv, substream ); + } + } + }, v ); + } else if constexpr ( std::is_same_v ) { + uint32_t size = v.size(); + stream.write( &size, sizeof(size) ); + cur_pos += sizeof(size); + + if( size ) { + stream.write( v.data(), v.size() ); + stream.write( "\0", 1 ); /// null term + cur_pos += v.size() + 1; + } + } else if constexpr ( is_std_vector::value ) { + + if constexpr( contains_offset_ptr< typename T::value_type >() ) { + + uint32_t size = v.size(); + stream.write( &size, sizeof(size) ); + cur_pos += sizeof(size); + alloc_pos += sizeof(size) + size * sizeof(offset_ptr); + + + for( const auto& member: v ) { + + if constexpr ( std::is_same_v || is_std_vector::value ) { + if( member.size() == 0 ) { + offset_ptr ptr = { .offset = 0 }; + stream.write( &ptr, sizeof(ptr) ); + cur_pos += sizeof(ptr); + continue; + } + } + + size_stream size_str; + flatpack( member, size_str ); + + offset_ptr ptr = { .offset = (alloc_pos - cur_pos) }; + + alloc_pos += size_str.size; + + stream.write( &ptr, sizeof(ptr) ); + + cur_pos += sizeof(ptr); + + //? cur_pos += size_str.size; + if constexpr( std::is_same_v ) { + stream.skip( size_str.size ); /// we already calculated this above + } + else { + /// now pack the member into the allocated spot + fixed_buf_stream substream( stream.pos+ptr.offset-sizeof(ptr), + size_str.size ); //ptr.size ); + if( substream.end > stream.end ) + throw_error( stream_error::overrun ); + flatpack( member, substream ); + } + } + } + else { + // std::cout << "vector type, T, is flat types: " << boost::core::demangle(typeid(typename T::value_type).name()) <<"\n"; + uint32_t size = v.size(); + stream.write( &size, sizeof(size) ); + cur_pos += sizeof(size); + for( const auto& item : v ) { + cur_pos += flatpack( item, stream ); + } + } + } else if constexpr ( reflect::is_struct ) { + alloc_pos = flatpack_size();//::struct_tuple_type>::value; + reflect::for_each( [&]( const meta& ref, const auto& mptr ){ + + const auto& member = v.*mptr; + + using member_type = decltype(result_of_member(mptr)); + if constexpr ( contains_offset_ptr() ) { + + if constexpr ( std::is_same_v || is_std_vector::value ) { + if( member.size() == 0 ) { + offset_ptr ptr = { .offset = 0 }; + stream.write( &ptr, sizeof(ptr) ); + cur_pos += sizeof(ptr); + + return; + } + } + + size_stream size_str; + flatpack( member, size_str ); + + offset_ptr ptr = { .offset = alloc_pos - cur_pos }; + + if( ptr.offset == 0 ) exit(-1); + + + alloc_pos += size_str.size; + stream.write( &ptr, sizeof(ptr) ); + cur_pos += sizeof(ptr); + + + if constexpr( std::is_same_v ) { + stream.skip( size_str.size ); /// we already calculated this above + } + else { + /// now pack the member into the allocated spot + fixed_buf_stream substream( stream.pos+ptr.offset-sizeof(ptr), size_str.size ); + + if( substream.end > stream.end ) { + throw_error( stream_error::overrun ); + } + flatpack( member, substream ); + } + } else { + cur_pos += flatpack( member, stream ); + } + }); + } else if constexpr( std::is_trivially_copyable::value ) { + stream.write( &v, sizeof(v) ); + cur_pos += sizeof(v); + } else { + T::flatpack_is_not_defined; + } + return cur_pos; + } + + + /** + * Behaves like a shared_ptr, copies will all point to the same flat data array. + */ + template + class flat_ptr { + public: + typedef T value_type; + + flat_ptr( const T& from ) { + clio::size_stream ss; + clio::flatpack( from, ss ); + _size = ss.size; + if( _size ) { + _data = std::shared_ptr( new char[_size], [](char* c){ delete[] c; } ); + fixed_buf_stream out( _data.get(), _size ); + clio::flatpack( from, out ); + } + } + flat_ptr() {}; + operator bool()const { return _size; }; + + auto operator->() { + return reinterpret_cast*>(_data.get()); + } + auto operator->()const { + return reinterpret_cast*>(_data.get()); + } + const char* data()const { return _data.get(); } + char* data() { return _data.get(); } + size_t size()const { return _size; } + + void reset( size_t s = 0 ) { + _size = s; + if( _size ) + _data = std::shared_ptr( new char[_size], [](char* c){ delete[] c; } ); + else _data.reset(); + } + + operator T()const { + T tmp; + input_stream in(_data.get(), _size ); + clio::flatunpack( tmp, in ); + return tmp; + } + + private: + std::shared_ptr _data; + size_t _size = 0; + }; + +} /// namespace clio diff --git a/libraries/clio/include/clio/fpconv.h b/libraries/clio/include/clio/fpconv.h new file mode 100644 index 000000000..4c9962823 --- /dev/null +++ b/libraries/clio/include/clio/fpconv.h @@ -0,0 +1,336 @@ + +/// From https://github.com/night-shift/fpconv +/// Boost Software License 1.0 +/// See accompanying license file + +#include +#include + +#include + +#define fracmask 0x000FFFFFFFFFFFFFU +#define expmask 0x7FF0000000000000U +#define hiddenbit 0x0010000000000000U +#define signmask 0x8000000000000000U +#define expbias (1023 + 52) + +#define absv(n) ((n) < 0 ? -(n) : (n)) +#define minv(a, b) ((a) < (b) ? (a) : (b)) + +static constexpr uint64_t tens[] = { + 10000000000000000000U, 1000000000000000000U, 100000000000000000U, + 10000000000000000U, 1000000000000000U, 100000000000000U, + 10000000000000U, 1000000000000U, 100000000000U, + 10000000000U, 1000000000U, 100000000U, + 10000000U, 1000000U, 100000U, + 10000U, 1000U, 100U, + 10U, 1U +}; + +static inline uint64_t get_dbits(double d) +{ + union { + double dbl; + uint64_t i; + } dbl_bits = { d }; + + return dbl_bits.i; +} + +static inline Fp build_fp(double d) +{ + uint64_t bits = get_dbits(d); + + Fp fp; + fp.frac = bits & fracmask; + fp.exp = (bits & expmask) >> 52; + + if(fp.exp) { + fp.frac += hiddenbit; + fp.exp -= expbias; + + } else { + fp.exp = -expbias + 1; + } + + return fp; +} + +static inline void normalize(Fp* fp) +{ + while ((fp->frac & hiddenbit) == 0) { + fp->frac <<= 1; + fp->exp--; + } + + int shift = 64 - 52 - 1; + fp->frac <<= shift; + fp->exp -= shift; +} + +static inline void get_normalized_boundaries(Fp* fp, Fp* lower, Fp* upper) +{ + upper->frac = (fp->frac << 1) + 1; + upper->exp = fp->exp - 1; + + while ((upper->frac & (hiddenbit << 1)) == 0) { + upper->frac <<= 1; + upper->exp--; + } + + int u_shift = 64 - 52 - 2; + + upper->frac <<= u_shift; + upper->exp = upper->exp - u_shift; + + + int l_shift = fp->frac == hiddenbit ? 2 : 1; + + lower->frac = (fp->frac << l_shift) - 1; + lower->exp = fp->exp - l_shift; + + + lower->frac <<= lower->exp - upper->exp; + lower->exp = upper->exp; +} + +static inline Fp multiply(Fp* a, Fp* b) +{ + const uint64_t lomask = 0x00000000FFFFFFFF; + + uint64_t ah_bl = (a->frac >> 32) * (b->frac & lomask); + uint64_t al_bh = (a->frac & lomask) * (b->frac >> 32); + uint64_t al_bl = (a->frac & lomask) * (b->frac & lomask); + uint64_t ah_bh = (a->frac >> 32) * (b->frac >> 32); + + uint64_t tmp = (ah_bl & lomask) + (al_bh & lomask) + (al_bl >> 32); + /* round up */ + tmp += 1U << 31; + + Fp fp = { + ah_bh + (ah_bl >> 32) + (al_bh >> 32) + (tmp >> 32), + a->exp + b->exp + 64 + }; + + return fp; +} + +static inline void round_digit(char* digits, int ndigits, uint64_t delta, uint64_t rem, uint64_t kappa, uint64_t frac) +{ + while (rem < frac && delta - rem >= kappa && + (rem + kappa < frac || frac - rem > rem + kappa - frac)) { + + digits[ndigits - 1]--; + rem += kappa; + } +} + +static inline int generate_digits(Fp* fp, Fp* upper, Fp* lower, char* digits, int* K) +{ + uint64_t wfrac = upper->frac - fp->frac; + uint64_t delta = upper->frac - lower->frac; + + Fp one; + one.frac = 1ULL << -upper->exp; + one.exp = upper->exp; + + uint64_t part1 = upper->frac >> -one.exp; + uint64_t part2 = upper->frac & (one.frac - 1); + + int idx = 0, kappa = 10; + const uint64_t* divp; + /* 1000000000 */ + for(divp = tens + 10; kappa > 0; divp++) { + + uint64_t div = *divp; + unsigned digit = part1 / div; + + if (digit || idx) { + digits[idx++] = digit + '0'; + } + + part1 -= digit * div; + kappa--; + + uint64_t tmp = (part1 <<-one.exp) + part2; + if (tmp <= delta) { + *K += kappa; + round_digit(digits, idx, delta, tmp, div << -one.exp, wfrac); + + return idx; + } + } + + /* 10 */ + const uint64_t* unit = tens + 18; + + while(true) { + part2 *= 10; + delta *= 10; + kappa--; + + unsigned digit = part2 >> -one.exp; + if (digit || idx) { + digits[idx++] = digit + '0'; + } + + part2 &= one.frac - 1; + if (part2 < delta) { + *K += kappa; + round_digit(digits, idx, delta, part2, one.frac, wfrac * *unit); + + return idx; + } + + unit--; + } +} + +static inline int grisu2(double d, char* digits, int* K) +{ + Fp w = build_fp(d); + + Fp lower, upper; + get_normalized_boundaries(&w, &lower, &upper); + + normalize(&w); + + int k; + Fp cp = find_cachedpow10(upper.exp, &k); + + w = multiply(&w, &cp); + upper = multiply(&upper, &cp); + lower = multiply(&lower, &cp); + + lower.frac++; + upper.frac--; + + *K = -k; + + return generate_digits(&w, &upper, &lower, digits, K); +} + +static inline int emit_digits(char* digits, int ndigits, char* dest, int K, bool neg) +{ + int exp = absv(K + ndigits - 1); + + /* write plain integer */ + if(K >= 0 && (exp < (ndigits + 7))) { + memcpy(dest, digits, ndigits); + memset(dest + ndigits, '0', K); + + return ndigits + K; + } + + /* write decimal w/o scientific notation */ + if(K < 0 && (K > -7 || exp < 4)) { + int offset = ndigits - absv(K); + /* fp < 1.0 -> write leading zero */ + if(offset <= 0) { + offset = -offset; + dest[0] = '0'; + dest[1] = '.'; + memset(dest + 2, '0', offset); + memcpy(dest + offset + 2, digits, ndigits); + + return ndigits + 2 + offset; + + /* fp > 1.0 */ + } else { + memcpy(dest, digits, offset); + dest[offset] = '.'; + memcpy(dest + offset + 1, digits + offset, ndigits - offset); + + return ndigits + 1; + } + } + + /* write decimal w/ scientific notation */ + ndigits = minv(ndigits, 18 - neg); + + int idx = 0; + dest[idx++] = digits[0]; + + if(ndigits > 1) { + dest[idx++] = '.'; + memcpy(dest + idx, digits + 1, ndigits - 1); + idx += ndigits - 1; + } + + dest[idx++] = 'e'; + + char sign = K + ndigits - 1 < 0 ? '-' : '+'; + dest[idx++] = sign; + + int cent = 0; + + if(exp > 99) { + cent = exp / 100; + dest[idx++] = cent + '0'; + exp -= cent * 100; + } + if(exp > 9) { + int dec = exp / 10; + dest[idx++] = dec + '0'; + exp -= dec * 10; + + } else if(cent) { + dest[idx++] = '0'; + } + + dest[idx++] = exp % 10 + '0'; + + return idx; +} + +static inline int filter_special(double fp, char* dest) +{ + if(fp == 0.0) { + dest[0] = '0'; + return 1; + } + + uint64_t bits = get_dbits(fp); + + bool nan = (bits & expmask) == expmask; + + if(!nan) { + return 0; + } + + if(bits & fracmask) { + dest[0] = 'n'; dest[1] = 'a'; dest[2] = 'n'; + + } else { + dest[0] = 'i'; dest[1] = 'n'; dest[2] = 'f'; + } + + return 3; +} + +static inline int fpconv_dtoa(double d, char dest[24]) +{ + char digits[18]; + + int str_len = 0; + bool neg = false; + + if(get_dbits(d) & signmask) { + dest[0] = '-'; + str_len++; + neg = true; + } + + int spec = filter_special(d, dest + str_len); + + if(spec) { + return str_len + spec; + } + + int K = 0; + int ndigits = grisu2(d, digits, &K); + + str_len += emit_digits(digits, ndigits, dest + str_len, K, neg); + + return str_len; +} diff --git a/libraries/clio/include/clio/from_bin.hpp b/libraries/clio/include/clio/from_bin.hpp new file mode 100644 index 000000000..011bb1e9e --- /dev/null +++ b/libraries/clio/include/clio/from_bin.hpp @@ -0,0 +1,288 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace clio { + +template +void from_bin(T& obj, S& stream); + +template +void varuint32_from_bin(uint32_t& dest, S& stream) { + dest = 0; + int shift = 0; + uint8_t b = 0; + do { + if (shift >= 35) + throw_error(stream_error::invalid_varuint_encoding); + from_bin(b, stream); + dest |= uint32_t(b & 0x7f) << shift; + shift += 7; + } while (b & 0x80); +} + +template +void varuint64_from_bin(uint64_t& dest, S& stream) { + dest = 0; + int shift = 0; + uint8_t b = 0; + do { + if (shift >= 70) + throw_error(stream_error::invalid_varuint_encoding); + from_bin(b, stream); + dest |= uint64_t(b & 0x7f) << shift; + shift += 7; + } while (b & 0x80); +} + +template +void varint32_from_bin(int32_t& result, S& stream) { + uint32_t v; + varuint32_from_bin(v, stream); + if (v & 1) + result = ((~v) >> 1) | 0x8000'0000; + else + result = v >> 1; +} + +template +void from_bin_assoc(T& v, S& stream) { + uint32_t size; + varuint32_from_bin(size, stream); + for (size_t i = 0; i < size; ++i) { + typename T::value_type elem; + from_bin(elem, stream); + v.emplace(elem); + } +} + +template +void from_bin_sequence(T& v, S& stream) { + uint32_t size; + varuint32_from_bin(size, stream); + for (size_t i = 0; i < size; ++i) { + v.emplace_back(); + from_bin(v.back(), stream); + } +} + +template +void from_bin(T (&v)[N], S& stream) { + uint32_t size; + varuint32_from_bin(size, stream); + if (size != N) + throw_error(stream_error::array_size_mismatch); + if constexpr (has_bitwise_serialization()) { + stream.read(reinterpret_cast(v), size * sizeof(T)); + } else { + for (size_t i = 0; i < size; ++i) { OUTCOME_TRY(from_bin(v[i], stream)); } + } +} + +template +void from_bin(std::vector& v, S& stream) { + if constexpr (has_bitwise_serialization()) { + if constexpr (sizeof(size_t) >= 8) { + uint64_t size; + varuint64_from_bin(size, stream); + stream.check_available(size * sizeof(T)); + v.resize(size); + stream.read(reinterpret_cast(v.data()), size * sizeof(T)); + } else { + uint32_t size; + varuint32_from_bin(size, stream); + stream.check_available(size * sizeof(T)); + v.resize(size); + stream.read(reinterpret_cast(v.data()), size * sizeof(T)); + } + } else { + uint32_t size; + varuint32_from_bin(size, stream); + v.resize(size); + for (size_t i = 0; i < size; ++i) { + from_bin(v[i], stream); + } + } +} + +template +void from_bin(std::set& v, S& stream) { + from_bin_assoc(v, stream); +} + +template +void from_bin(std::map& v, S& stream) { + uint32_t size; + varuint32_from_bin(size, stream); + for (size_t i = 0; i < size; ++i) { + std::pair elem; + from_bin(elem, stream); + v.emplace(elem); + } +} + +template +void from_bin(std::deque& v, S& stream) { + from_bin_sequence(v, stream); +} + +template +void from_bin(std::list& v, S& stream) { + from_bin_sequence(v, stream); +} + +template +void from_bin(input_stream& obj, S& stream) { + if constexpr (sizeof(size_t) >= 8) { + uint64_t size; + varuint64_from_bin(size, stream); + stream.check_available(size); + stream.read_reuse_storage(obj.pos, size); + obj.end = obj.pos + size; + } else { + uint32_t size; + varuint32_from_bin(size, stream); + stream.check_available(size); + stream.read_reuse_storage(obj.pos, size); + obj.end = obj.pos + size; + } +} + +template +void from_bin(std::pair& obj, S& stream) { + from_bin(obj.first, stream); + from_bin(obj.second, stream); +} + +template +inline void from_bin(std::string& obj, S& stream) { + uint32_t size; + varuint32_from_bin(size, stream); + obj.resize(size); + stream.read(obj.data(), obj.size()); +} + +template +inline void from_bin(std::string_view& obj, S& stream) { + uint32_t size; + varuint32_from_bin(size, stream); + obj = std::string_view(stream.get_pos(),size); + stream.skip(size); +} + +template +void from_bin(std::optional& obj, S& stream) { + bool present; + from_bin(present, stream); + if (!present) { + obj.reset(); + return; + } + obj.emplace(); + from_bin(*obj, stream); +} + +template +void variant_from_bin(std::variant& v, uint32_t i, S& stream) { + if constexpr (I < std::variant_size_v>) { + if (i == I) { + auto& x = v.template emplace(); + from_bin(x, stream); + } else { + variant_from_bin(v, i, stream); + } + } else { + throw_error(stream_error::bad_variant_index); + } +} + +template +void from_bin(std::variant& obj, S& stream) { + uint32_t u; + varuint32_from_bin(u, stream); + variant_from_bin<0>(obj, u, stream); +} + +template +void from_bin(std::array& obj, S& stream) { + for (T& elem : obj) { from_bin(elem, stream); } +} + +template +void from_bin_tuple(T& obj, S& stream) { + if constexpr (N < std::tuple_size_v) { + from_bin(std::get(obj), stream); + from_bin_tuple(obj, stream); + }} + +template +void from_bin(std::tuple& obj, S& stream) { + from_bin_tuple<0>(obj, stream); +} + +template +void from_bin(T& obj, S& stream) { + if constexpr (has_bitwise_serialization()) { + stream.read(reinterpret_cast(&obj), sizeof(T)); + } else { //if constexpr (std::is_same_v, void>) { + reflect::for_each( [&]( const clio::meta&, auto m ) { + if constexpr( not std::is_member_function_pointer_v ) { + from_bin( obj.*m, stream ); + } + }); + } /*else { + // TODO: This can operate in place for standard serializers + decltype(serialize_as(obj)) temp; + OUTCOME_TRY(from_bin(temp, stream)); + convert(temp, obj, choose_first); + return outcome::success(); + } */ +} + +template +void from_bin_vec(T& obj, const std::vector& bin) { + input_stream stream{ bin }; + from_bin(obj, stream); +} +template +void from_bin_vec(T& obj, std::vector& bin) { + input_stream stream{ bin }; + from_bin(obj, stream); +} + +template +void from_bin_vec(T& obj, const bytes& bin) { + from_bin_vec(obj, bin.data); +} +template +void from_bin_vec(T& obj, bytes& bin) { + from_bin_vec(obj, bin.data); +} + +template +T from_bin(const std::vector& bin) { + T obj; + from_bin_vec(obj, bin); + return obj; +} + +template +T from_bin(S& stream) { + T obj; + from_bin(obj, stream); + return obj; +} + + + + +} // namespace clio diff --git a/libraries/clio/include/clio/from_bin/varint.hpp b/libraries/clio/include/clio/from_bin/varint.hpp new file mode 100644 index 000000000..08e0b3314 --- /dev/null +++ b/libraries/clio/include/clio/from_bin/varint.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +namespace clio { + +template +void from_bin(varuint32& obj, S& stream) { + varuint32_from_bin(obj.value, stream); +} + +template +void from_bin(varint32& obj, S& stream) { + varint32_from_bin(obj.value, stream); +} + +} /// namespace clio diff --git a/libraries/clio/include/clio/from_json.hpp b/libraries/clio/include/clio/from_json.hpp new file mode 100644 index 000000000..25ff07ad8 --- /dev/null +++ b/libraries/clio/include/clio/from_json.hpp @@ -0,0 +1,762 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace clio { +enum class from_json_error { + no_error, + + expected_end, + expected_null, + expected_bool, + expected_string, + expected_hex_string, + hex_string_incorrect_length, + invalid_signature, + invalid_name, + expected_start_object, + expected_key, + expected_end_object, + expected_start_array, + expected_end_array, + expected_positive_uint, + expected_field, + expected_variant, + expected_public_key, + expected_private_key, + expected_signature, + expected_number, + expected_int, + expected_time_point, + expected_symbol_code, + expected_symbol, + expected_asset, + invalid_type_for_variant, + unexpected_field, + number_out_of_range, + from_json_no_pair, + + // These are from rapidjson: + document_empty, + document_root_not_singular, + value_invalid, + object_miss_name, + object_miss_colon, + object_miss_comma_or_curly_bracket, + array_miss_comma_or_square_bracket, + string_unicode_escape_invalid_hex, + string_unicode_surrogate_invalid, + string_escape_invalid, + string_miss_quotation_mark, + string_invalid_encoding, + number_too_big, + number_miss_fraction, + number_miss_exponent, + terminated, + unspecific_syntax_error, +}; // from_json_error +} // namespace clio + +namespace std { +template <> +struct is_error_code_enum : true_type {}; +} // namespace std + +namespace clio { + +class from_json_error_category_type : public std::error_category { + public: + const char* name() const noexcept override final { return "ConversionError"; } + + std::string message(int c) const override final { + switch (static_cast(c)) { + // clang-format off + case from_json_error::no_error: return "No error"; + + case from_json_error::expected_end: return "Expected end of json"; + case from_json_error::expected_null: return "Expected null"; + case from_json_error::expected_bool: return "Expected true or false"; + case from_json_error::expected_string: return "Expected string"; + case from_json_error::expected_hex_string: return "Expected string containing hex"; + case from_json_error::hex_string_incorrect_length: return "Hex string has incorrect length"; + case from_json_error::invalid_signature: return "Invalid signature format"; + case from_json_error::invalid_name: return "Invalid name"; + case from_json_error::expected_start_object: return "Expected {"; + case from_json_error::expected_key: return "Expected key"; + case from_json_error::expected_end_object: return "Expected }"; + case from_json_error::expected_start_array: return "Expected ["; + case from_json_error::expected_end_array: return "Expected ]"; + case from_json_error::expected_positive_uint: return "Expected positive integer"; + case from_json_error::expected_field: return "Expected field"; + case from_json_error::expected_variant: return R"(Expected variant: ["type", value])"; + case from_json_error::expected_public_key: return "Expected public key"; + case from_json_error::expected_private_key: return "Expected private key"; + case from_json_error::expected_signature: return "Expected signature"; + case from_json_error::expected_number: return "Expected number or boolean"; + case from_json_error::expected_int: return "Expected integer"; + case from_json_error::expected_time_point: return "Expected time point"; + case from_json_error::expected_symbol_code: return "Expected symbol code"; + case from_json_error::expected_symbol: return "Expected symbol"; + case from_json_error::expected_asset: return "Expected asset"; + case from_json_error::invalid_type_for_variant: return "Invalid type for variant"; + case from_json_error::unexpected_field: return "Unexpected field"; + case from_json_error::number_out_of_range: return "number is out of range"; + case from_json_error::from_json_no_pair: return "from_json does not support std::pair"; + + case from_json_error::document_empty: return "The document is empty"; + case from_json_error::document_root_not_singular: return "The document root must not follow by other values"; + case from_json_error::value_invalid: return "Invalid value"; + case from_json_error::object_miss_name: return "Missing a name for object member"; + case from_json_error::object_miss_colon: return "Missing a colon after a name of object member"; + case from_json_error::object_miss_comma_or_curly_bracket: return "Missing a comma or '}' after an object member"; + case from_json_error::array_miss_comma_or_square_bracket: return "Missing a comma or ']' after an array element"; + case from_json_error::string_unicode_escape_invalid_hex: return "Incorrect hex digit after \\u escape in string"; + case from_json_error::string_unicode_surrogate_invalid: return "The surrogate pair in string is invalid"; + case from_json_error::string_escape_invalid: return "Invalid escape character in string"; + case from_json_error::string_miss_quotation_mark: return "Missing a closing quotation mark in string"; + case from_json_error::string_invalid_encoding: return "Invalid encoding in string"; + case from_json_error::number_too_big: return "Number too big to be stored in double"; + case from_json_error::number_miss_fraction: return "Miss fraction part in number"; + case from_json_error::number_miss_exponent: return "Miss exponent in number"; + case from_json_error::terminated: return "Parsing was terminated"; + case from_json_error::unspecific_syntax_error: return "Unspecific syntax error"; + // clang-format on + + default: return "unknown"; + } + } +}; // from_json_error_category_type + +inline const from_json_error_category_type& from_json_error_category() { + static from_json_error_category_type c; + return c; +} + +inline std::error_code make_error_code(from_json_error e) { + return { static_cast(e), from_json_error_category() }; +} + +inline from_json_error convert_error(rapidjson::ParseErrorCode err) { + switch (err) { + // clang-format off + case rapidjson::kParseErrorNone: return from_json_error::no_error; + case rapidjson::kParseErrorDocumentEmpty: return from_json_error::document_empty; + case rapidjson::kParseErrorDocumentRootNotSingular: return from_json_error::document_root_not_singular; + case rapidjson::kParseErrorValueInvalid: return from_json_error::value_invalid; + case rapidjson::kParseErrorObjectMissName: return from_json_error::object_miss_name; + case rapidjson::kParseErrorObjectMissColon: return from_json_error::object_miss_colon; + case rapidjson::kParseErrorObjectMissCommaOrCurlyBracket: return from_json_error::object_miss_comma_or_curly_bracket; + case rapidjson::kParseErrorArrayMissCommaOrSquareBracket: return from_json_error::array_miss_comma_or_square_bracket; + case rapidjson::kParseErrorStringUnicodeEscapeInvalidHex: return from_json_error::string_unicode_escape_invalid_hex; + case rapidjson::kParseErrorStringUnicodeSurrogateInvalid: return from_json_error::string_unicode_surrogate_invalid; + case rapidjson::kParseErrorStringEscapeInvalid: return from_json_error::string_escape_invalid; + case rapidjson::kParseErrorStringMissQuotationMark: return from_json_error::string_miss_quotation_mark; + case rapidjson::kParseErrorStringInvalidEncoding: return from_json_error::string_invalid_encoding; + case rapidjson::kParseErrorNumberTooBig: return from_json_error::number_too_big; + case rapidjson::kParseErrorNumberMissFraction: return from_json_error::number_miss_fraction; + case rapidjson::kParseErrorNumberMissExponent: return from_json_error::number_miss_exponent; + case rapidjson::kParseErrorTermination: return from_json_error::terminated; + case rapidjson::kParseErrorUnspecificSyntaxError: return from_json_error::unspecific_syntax_error; + // clang-format on + + default: return from_json_error::unspecific_syntax_error; + } +} + +enum class json_token_type { + type_unread, + type_null, + type_bool, + type_string, + type_start_object, + type_key, + type_end_object, + type_start_array, + type_end_array, +}; + +struct json_token { + json_token_type type = {}; + std::string_view key = {}; + bool value_bool = {}; + std::string_view value_string = {}; +}; + +class json_token_stream : public rapidjson::BaseReaderHandler, json_token_stream> { + private: + rapidjson::Reader reader; + rapidjson::InsituStringStream ss; + + public: + json_token current_token; + + // This modifies json (ok... why?) + json_token_stream(char* json) : ss{ json } { reader.IterativeParseInit(); } + + bool complete() { return reader.IterativeParseComplete(); } + + std::reference_wrapper peek_token() { + if (current_token.type != json_token_type::type_unread) + return current_token; + else if (reader.IterativeParseNext(ss, + *this)) + return current_token; + else + throw_error(convert_error(reader.GetParseErrorCode())); + } + + void eat_token() { current_token.type = json_token_type::type_unread; } + + void get_end() { + if (current_token.type != json_token_type::type_unread || !complete()) + throw_error(from_json_error::expected_end); + } + + void get_null() { + auto t = peek_token(); + if (t.get().type != json_token_type::type_null) + throw_error( from_json_error::expected_null ); + eat_token(); + } + + bool get_bool() { + auto t = peek_token(); + if (t.get().type != json_token_type::type_bool) + throw_error( from_json_error::expected_bool); + eat_token(); + return t.get().value_bool; + } + + std::string_view get_string() { + auto t = peek_token(); + if (t.get().type != json_token_type::type_string) + throw_error(from_json_error::expected_string); + eat_token(); + return t.get().value_string; + } + + void get_start_object() { + auto t = peek_token(); + if (t.get().type != json_token_type::type_start_object) + throw_error(from_json_error::expected_start_object); + eat_token(); + } + + std::string_view get_key() { + auto t = peek_token(); + if (t.get().type != json_token_type::type_key) + throw_error(from_json_error::expected_key); + eat_token(); + return t.get().key; + } + + void get_end_object() { + auto t = peek_token(); + if (t.get().type != json_token_type::type_end_object) + throw_error(from_json_error::expected_end_object); + eat_token(); + } + + void get_start_array() { + auto t = peek_token(); + if (t.get().type != json_token_type::type_start_array) + throw_error(from_json_error::expected_start_array); + eat_token(); + } + + void get_end_array() { + auto t = peek_token(); + if (t.get().type != json_token_type::type_end_array) + throw_error(from_json_error::expected_end_array); + eat_token(); + } + + // BaseReaderHandler methods + bool Null() { + current_token.type = json_token_type::type_null; + return true; + } + bool Bool(bool v) { + current_token.type = json_token_type::type_bool; + current_token.value_bool = v; + return true; + } + bool RawNumber(const char* v, rapidjson::SizeType length, bool copy) { return String(v, length, copy); } + bool Int(int v) { return false; } + bool Uint(unsigned v) { return false; } + bool Int64(int64_t v) { return false; } + bool Uint64(uint64_t v) { return false; } + bool Double(double v) { return false; } + bool String(const char* v, rapidjson::SizeType length, bool) { + current_token.type = json_token_type::type_string; + current_token.value_string = { v, length }; + return true; + } + bool StartObject() { + current_token.type = json_token_type::type_start_object; + return true; + } + bool Key(const char* v, rapidjson::SizeType length, bool) { + current_token.key = { v, length }; + current_token.type = json_token_type::type_key; + return true; + } + bool EndObject(rapidjson::SizeType) { + current_token.type = json_token_type::type_end_object; + return true; + } + bool StartArray() { + current_token.type = json_token_type::type_start_array; + return true; + } + bool EndArray(rapidjson::SizeType) { + current_token.type = json_token_type::type_end_array; + return true; + } +}; // json_token_stream + +template +[[nodiscard]] bool unhex(DestIt dest, SrcIt begin, SrcIt end) { + auto get_digit = [&](uint8_t& nibble) { + if (*begin >= '0' && *begin <= '9') + nibble = *begin++ - '0'; + else if (*begin >= 'a' && *begin <= 'f') + nibble = *begin++ - 'a' + 10; + else if (*begin >= 'A' && *begin <= 'F') + nibble = *begin++ - 'A' + 10; + else + return false; + return true; + }; + while (begin != end) { + uint8_t h, l; + if (!get_digit(h) || !get_digit(l)) + return false; + *dest++ = (h << 4) | l; + } + return true; +} + +/// \exclude +template +void from_json(T& result, S& stream); + +/// \group from_json_explicit Parse JSON (Explicit Types) +/// Parse JSON and convert to `result`. These overloads handle specified types. +template +void from_json(std::string_view& result, S& stream) { + result = stream.get_string(); +} + +/// \group from_json_explicit Parse JSON (Explicit Types) +/// Parse JSON and convert to `result`. These overloads handle specified types. +template +void from_json(std::string& result, S& stream) { + result = stream.get_string(); +} + +/// \exclude +template +void from_json_int(T& result, S& stream) { + auto r = stream.get_string(); + auto pos = r.data(); + auto end = pos + r.size(); + bool found = false; + result = 0; + T limit; + T sign; + if (std::is_signed_v && pos != end && *pos == '-') { + ++pos; + sign = -1; + limit = std::numeric_limits::min(); + } else { + sign = 1; + limit = std::numeric_limits::max(); + } + while (pos != end && *pos >= '0' && *pos <= '9') { + T digit = (*pos++ - '0'); + // abs(result) can overflow. Use -abs(result) instead. + if (std::is_signed_v && (-sign * limit + digit) / 10 > -sign * result) + throw_error(from_json_error::number_out_of_range); + if (!std::is_signed_v && (limit - digit) / 10 < result) + throw_error(from_json_error::number_out_of_range); + result = result * 10 + sign * digit; + found = true; + } + if (pos != end || !found) + throw_error(from_json_error::expected_int); +} + +/// \group from_json_explicit +template +void from_json(uint8_t& result, S& stream) { + return from_json_int(result, stream); +} + +/// \group from_json_explicit +template +void from_json(uint16_t& result, S& stream) { + return from_json_int(result, stream); +} + +/// \group from_json_explicit +template +void from_json(uint32_t& result, S& stream) { + return from_json_int(result, stream); +} + +/// \group from_json_explicit +template +void from_json(uint64_t& result, S& stream) { + return from_json_int(result, stream); +} + +/// \group from_json_explicit +template +void from_json(unsigned __int128& result, S& stream) { + return from_json_int(result, stream); +} + +/// \group from_json_explicit +template +void from_json(int8_t& result, S& stream) { + return from_json_int(result, stream); +} + +/// \group from_json_explicit +template +void from_json(int16_t& result, S& stream) { + return from_json_int(result, stream); +} + +/// \group from_json_explicit +template +void from_json(int32_t& result, S& stream) { + return from_json_int(result, stream); +} + +/// \group from_json_explicit +template +void from_json(int64_t& result, S& stream) { + return from_json_int(result, stream); +} + +/// \group from_json_explicit +template +void from_json(__int128& result, S& stream) { + return from_json_int(result, stream); +} + +template +void from_json(float& result, S& stream) { + auto sv = stream.get_string(); + if (sv.empty()) + throw_error(from_json_error::expected_number); + std::string s(sv); // strtof expects a null-terminated string + errno = 0; + char* end; + result = std::strtof(s.c_str(), &end); + if (errno || end != s.c_str() + s.size()) + throw_error(from_json_error::expected_number); +} + +template +void from_json(double& result, S& stream) { + auto sv = stream.get_string(); + if (sv.empty()) + throw_error( from_json_error::expected_number ); + std::string s(sv); + errno = 0; + char* end; + result = std::strtod(s.c_str(), &end); + if (errno || end != s.c_str() + s.size()) + throw_error(from_json_error::expected_number); +} + +/* +/// \group from_json_explicit +template +void from_json(int32_t& result, S& stream) { + bool in_str = false; + if (pos != end && *pos == '"') { + in_str = true; + from_json_skip_space(pos, end); + } + bool neg = false; + if (pos != end && *pos == '-') { + neg = true; + ++pos; + } + bool found = false; + result = 0; + while (pos != end && *pos >= '0' && *pos <= '9') { + result = result * 10 + *pos++ - '0'; + found = true; + } + check(found, "expected integer"); + from_json_skip_space(pos, end); + if (in_str) { + from_json_expect(pos, end, '"', "expected integer"); + from_json_skip_space(pos, end); + } + if (neg) + result = -result; +} +*/ +/// \group from_json_explicit +template +void from_json(bool& result, S& stream) { + result = stream.get_bool(); +} + +/// \group from_json_explicit +template +void from_json(std::vector& result, S& stream) { + stream.get_start_array(); + result.clear(); + while (true) { + auto t = stream.peek_token(); + if (t.get().type == json_token_type::type_end_array) + break; + result.emplace_back(); + from_json(result.back(), stream); + } + stream.get_end_array(); +} + +/// \group from_json_explicit +template +void from_json(std::optional& result, S& stream) { + result = std::optional(); + if (stream.get_null()) { + result = std::nullopt; + } else { + result.emplace(); + from_json(*result, stream); + } +} + +template +bool set_variant_impl(std::variant& result, uint32_t type) { + if (type == N) { + result.template emplace(); + return true; + } else if constexpr (N + 1 < sizeof...(T)) { + return set_variant_impl(result, type); + } + return false; +} + +/// TODO: this is attempting to parse variant as [ type, obj ], +/// but to_json is encoding it as { type: "", value: } +/// \group from_json_explicit +template +void from_json(std::variant& result, S& stream) { + stream.get_start_array(); + std::string_view type; + from_json(type, stream); + const char* const type_names[] = { get_type_name((T*)nullptr)... }; + uint32_t type_idx = std::find(type_names, type_names + sizeof...(T), type) - type_names; + if (type_idx >= sizeof...(T)) + throw_error(from_json_error::invalid_type_for_variant); + if( set_variant_impl(result, type_idx) ) { + std::visit([&](auto& x) { return from_json(x, stream); }, result); + } + stream.get_end_array(); +} + +/** + * Accepts null as default value for all tuple elements + * Accepts [] with 0 to N as the first n elements and errors if there are too many elements + * accepts anything else as the presumed first element + * + * TODO: make robust against adding elements to the tuple by droping all remaining values + */ +template +void from_json(std::tuple& result, S& stream) { + result = std::tuple(); + auto t = stream.peek_token(); + if( t.get().type == json_token_type::type_null ) { + return; + } + + if constexpr( sizeof...(T) > 0 ) { + if( t.get().type != json_token_type::type_start_array ) { + from_json( std::get<0>(result), stream ); + } + } + + stream.get_start_array(); + tuple_for_each( result, [&]( int idx, auto& item ) { + auto t = stream.peek_token(); + if( t.get().type == json_token_type::type_end_array ) + return; + from_json( item, stream ); + }); + stream.get_end_array(); +} + +/// \group from_json_explicit +template +void from_json_hex(std::vector& result, S& stream) { + auto s = stream.get_string(); + if (s.size() & 1) + throw_error(from_json_error::expected_hex_string); + result.clear(); + result.reserve(s.size() / 2); + if (!unhex(std::back_inserter(result), s.begin(), s.end())) + throw_error(from_json_error::expected_hex_string); +} + +/// \exclude +template +inline void from_json_object(S& stream, F f) { + stream.get_start_object(); + while (true) { + auto t = stream.peek_token(); + if (t.get().type == json_token_type::type_end_object) + { + break; + } + auto k = stream.get_key(); + f(k); + } + stream.get_end_object(); +} + +template +void from_json_skip_value(S& stream) { + uint64_t depth = 0; + do { + auto t = stream.peek_token(); + auto type = t.get().type; + if (type == json_token_type::type_start_object || type == json_token_type::type_start_array) + ++depth; + else if (type == json_token_type::type_end_object || type == json_token_type::type_end_array) + --depth; + stream.eat_token(); + } while (depth); +} + +/// \output_section Parse JSON (Reflected Objects) +/// Parse JSON and convert to `obj`. This overload works with +/// [reflected objects](standardese://reflection/). +template +void from_json(T& obj, S& stream) { + if constexpr( reflect::is_struct ) { + from_json_object(stream, [&](std::string_view key) -> void { + bool found = false; + reflect::get( key, [&]( auto member ){ + if constexpr( not std::is_member_function_pointer_v< decltype(member) > ) + { + from_json( obj.*member, stream ); + found = true; + } else { + } + }); + if( not found ) { + from_json_skip_value(stream); + } + }); + } else { + T::reflection_not_defined; + } +} + +/// not implimented, so don't link +template +void from_json(std::pair& obj, S& stream); + +/* +/// \output_section Convenience Wrappers +/// Parse JSON and return result. This overload wraps the other `to_json` overloads. +template +T from_json(const std::vector& v) { + const char* pos = v.data(); + const char* end = pos + v.size(); + from_json_skip_space(pos, end); + T result; + from_json(result, pos, end); + from_json_expect_end(pos, end); + return result; +} + +/// Parse JSON and return result. This overload wraps the other `to_json` overloads. +template +T from_json(std::string_view s) { + const char* pos = s.data(); + const char* end = pos + s.size(); + from_json_skip_space(pos, end); + T result; + from_json(result, pos, end); + from_json_expect_end(pos, end); + return result; +} +*/ + +/// Parse JSON and return result. This overload wraps the other `to_json` overloads. +template +T from_json(S& stream) { + T x; + from_json(x, stream); + return x; +} + +template +T from_json( std::string json) { + json_token_stream stream( json.data() ); + return from_json(stream); +} + + +/* +/// \exclude +template +__attribute__((noinline)) void parse_named_variant_impl(tagged_variant& v, size_t i, + const char*& pos, const char* end) { + if constexpr (I < sizeof...(NamedTypes)) { + if (i == I) { + auto& q = v.value; + auto& x = q.template emplace(); + if constexpr (!is_named_empty_type_v>) { + from_json_expect(pos, end, ',', "expected ,"); + from_json(x, pos, end); + } + } else { + return parse_named_variant_impl(v, i, pos, end); + } + } else { + check(false, "invalid variant index"); + } +} + +/// \group from_json_explicit +template +__attribute__((noinline)) void from_json(tagged_variant& result, const char*& pos, + const char* end) { + from_json_skip_space(pos, end); + from_json_expect(pos, end, '[', "expected array"); + + clio::name name; + from_json(name, pos, end); + + for (size_t i = 0; i < sizeof...(NamedTypes); ++i) { + if (name == tagged_variant::keys[i]) { + parse_named_variant_impl<0>(result, i, pos, end); + from_json_expect(pos, end, ']', "expected ]"); + return; + } + } + check(false, "invalid variant index name"); +} +*/ + +} // namespace clio diff --git a/libraries/clio/include/clio/from_json/varint.hpp b/libraries/clio/include/clio/from_json/varint.hpp new file mode 100644 index 000000000..fa4961695 --- /dev/null +++ b/libraries/clio/include/clio/from_json/varint.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +namespace clio { + +template +void from_json(varuint32& obj, S& stream) { + from_json(obj.value, stream); +} + +template +void from_json(varint32& obj, S& stream) { + from_json(obj.value, stream); +} + +} /// namespace clio diff --git a/libraries/clio/include/clio/from_protobuf.hpp b/libraries/clio/include/clio/from_protobuf.hpp new file mode 100644 index 000000000..e05bd9fdf --- /dev/null +++ b/libraries/clio/include/clio/from_protobuf.hpp @@ -0,0 +1,187 @@ +#pragma once +#include +#include + +namespace clio { + +template +void from_protobuf_object( T& obj, S& stream ); + +template +bool set_variant(std::variant& result, uint32_t type) { + if (type == N) { + result.template emplace(); + return true; + } else if constexpr (N + 1 < sizeof...(T)) { + return set_variant(result, type); + } + return false; +} + +template +void skip_member( uint16_t wire_type, S& stream ) { + uint32_t temp = 0; + switch( wire_type ) { + case 0: + varuint32_from_bin( temp, stream ); + break; + case 1: + stream.skip(8); + break; + case 2: + varuint32_from_bin( temp, stream ); + stream.skip(temp); + break; + case 5: + stream.skip(4); + break; + } +} + + +template +void from_protobuf_object(std::variant& obj, S& stream) +{ + uint32_t key = 0; + varuint32_from_bin( key, stream ); + uint32_t field = key >> 3; + + if( set_variant( obj, field-1 ) ) { + std::visit([&](auto& x) { + return from_protobuf_member(x, stream); + }, obj); + } else { + skip_member( uint8_t(key) & 0x07 , stream ); + obj = std::variant(); /// reset to default + } +} +template +void from_protobuf_object(std::tuple& obj, S& stream) +{ + while( stream.remaining() ) { + uint32_t key = 0; + varuint32_from_bin( key, stream ); + uint16_t wire_type = uint8_t(key) & 0x07; + uint32_t number = key >> 3; + + bool skip_it = true; + tuple_get( obj, number-1, [&]( auto& member ) { + from_protobuf_member( member, stream ); + skip_it = false; + }); + if( skip_it ) skip_member( wire_type, stream ); + } +} + + +template +void from_protobuf_object( std::vector& obj, S& stream ) { + obj.clear(); + if constexpr( std::is_arithmetic_v ) { + uint32_t key = 0; + varuint32_from_bin( key, stream ); + uint16_t wire_type = uint8_t(key) & 0x07; + uint32_t number = key >> 3; + if( number != 1 ) + skip_member( wire_type, stream ); + else { + uint32_t size = 0; + varuint32_from_bin( size, stream ); + if( size > stream.remaining() ) + throw_error(stream_error::overrun); + obj.resize( size / sizeof( T ) ); + stream.read( (char*)obj.data(), size ); + } + } else { + while( stream.remaining() ) { + uint32_t key = 0; + varuint32_from_bin( key, stream ); + uint16_t wire_type = uint8_t(key) & 0x07; + uint32_t number = key >> 3; + + bool skip_it = true; + if( number == 1 ) { + skip_it = false; + obj.resize(obj.size()+1); + from_protobuf_member( obj.back(), stream ); + } else { + skip_member( wire_type, stream ); + } + } + } +} + +template +void from_protobuf_member( T& obj, S& stream ) { + if constexpr( std::is_same_v< T, std::string> ) { + uint32_t size = 0; + varuint32_from_bin( size, stream ); + obj.resize(size); + stream.read( obj.data(), size ); + } else if constexpr( std::is_same_v< T, bytes> ) { + uint32_t size = 0; + varuint32_from_bin( size, stream ); + obj.data.resize(size); + stream.read( obj.data.data(), size ); + } + else if constexpr( reflect::is_struct || is_std_variant::value || is_std_vector::value || is_std_tuple::value ) { + uint32_t size = 0; + varuint32_from_bin( size, stream ); + if( size > stream.remaining() ) + throw_error(stream_error::overrun); + input_stream objstream( stream.pos, size ); + stream.skip(size); + from_protobuf_object( obj, objstream ); + } else if constexpr ( 5 == wire_type::value or 1 == wire_type::value ) { + from_bin( obj, stream ); + } else if constexpr ( 0 == wire_type::value ) { + uint32_t v; + varuint32_from_bin( v, stream ); + obj = v; + } else { + T::from_protobuf_is_not_defined; /// used to generate useful compile error + } +} + +template +void from_protobuf_object( T& obj, S& stream ) { + while( stream.remaining() ) { + uint32_t key = 0; + varuint32_from_bin( key, stream ); + uint16_t wire_type = uint8_t(key) & 0x07; + uint32_t number = key >> 3; + + //bool skip_it = true; + if( not reflect::get( number, [&]( auto m ) { + if constexpr ( std::is_member_function_pointer_v ) { + skip_member( wire_type, stream ); /// we cannot store return value of functions in T + } else { + //using member_type = std::decay_t; + //auto& member = obj.*m; + from_protobuf_member( obj.*m, stream ); + } + }) ) { // if not reflect::get(number...) + skip_member( wire_type, stream ); + } + } +} + +template +T from_protobuf( S& stream ) { + T tmp; + from_protobuf_object( tmp, stream ); + return tmp; +} + +template +T from_protobuf( const std::vector& in ) { + clio::input_stream stream(in.data(), in.size() ); + return clio::from_protobuf(stream); +} +template +T from_protobuf( std::vector& in ) { + clio::input_stream stream(in.data(), in.size() ); + return clio::from_protobuf(stream); +} + +} // namespace clio diff --git a/libraries/clio/include/clio/get_type_name.hpp b/libraries/clio/include/clio/get_type_name.hpp new file mode 100644 index 000000000..55c82d144 --- /dev/null +++ b/libraries/clio/include/clio/get_type_name.hpp @@ -0,0 +1,172 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace clio { + +#define CLIO_REFLECT_TYPENAME( T ) \ + constexpr const char* get_type_name( const T* ) { return BOOST_PP_STRINGIZE(T); } + +#define CLIO_REFLECT_TYPENAME_CUSTOM( T, CUSTOM ) \ + constexpr const char* get_type_name( const T* ) { return BOOST_PP_STRINGIZE(CUSTOM); } + +constexpr const char* get_type_name(const bool*) { return "bool"; } +constexpr const char* get_type_name(const int8_t*) { return "int8"; } +constexpr const char* get_type_name(const uint8_t*) { return "uint8"; } +constexpr const char* get_type_name(const int16_t*) { return "int16"; } +constexpr const char* get_type_name(const uint16_t*) { return "uint16"; } +constexpr const char* get_type_name(const int32_t*) { return "int32"; } +constexpr const char* get_type_name(const uint32_t*) { return "uint32"; } +constexpr const char* get_type_name(const int64_t*) { return "int64"; } +constexpr const char* get_type_name(const uint64_t*) { return "uint64"; } +constexpr const char* get_type_name(const float*) { return "float32"; } +constexpr const char* get_type_name(const double*) { return "double"; } +constexpr const char* get_type_name(const char*) { return "char"; } +constexpr const char* get_type_name(const std::string*) { return "string"; } +constexpr const char* get_type_name(const __int128*) { return "int128"; } +constexpr const char* get_type_name(const unsigned __int128*) { return "uint128"; } + +#ifdef __eosio_cdt__ +constexpr const char* get_type_name(const long double*) { return "float128"; } +#endif + +template +constexpr std::array array_cat(std::array lhs, std::array rhs) { + std::array result{}; + for (int i = 0; i < N; ++i) { result[i] = lhs[i]; } + for (int i = 0; i < M; ++i) { result[i + N] = rhs[i]; } + return result; +} + +template +constexpr std::array to_array(std::string_view s) { + std::array result{}; + for (int i = 0; i < N; ++i) { result[i] = s[i]; } + return result; +} + +template +constexpr auto append_type_name(const char (&suffix)[N]) { + constexpr std::string_view name = get_type_name((T*)nullptr); + return array_cat(to_array(name), to_array({ suffix, N })); +} + +template +constexpr auto vector_type_name = append_type_name("[]"); + +template +constexpr auto optional_type_name = append_type_name("?"); + +template +constexpr const char* get_type_name(const std::vector*) { + return vector_type_name.data(); +} + +template +constexpr const char* get_type_name(const std::optional*) { + return optional_type_name.data(); +} + +struct variant_type_appender { + char* buf; + constexpr variant_type_appender operator+(std::string_view s) { + *buf++ = '_'; + for (auto ch : s) *buf++ = ch; + return *this; + } +}; + +template +constexpr auto get_variant_type_name() { + constexpr std::size_t size = sizeof("variant") + ((std::string_view(get_type_name((T*)nullptr)).size() + 1) + ...); + std::array buffer{ 'v', 'a', 'r', 'i', 'a', 'n', 't' }; + (variant_type_appender{ buffer.data() + 7 } + ... + std::string_view(get_type_name((T*)nullptr))); + buffer[buffer.size() - 1] = '\0'; + return buffer; +} + +template +constexpr auto get_tuple_type_name() { + constexpr std::size_t size = sizeof("tuple") + ((std::string_view(get_type_name((T*)nullptr)).size() + 1) + ...); + std::array buffer{ 't', 'u', 'p', 'l', 'e' }; + (variant_type_appender{ buffer.data() + 5 } + ... + std::string_view(get_type_name((T*)nullptr))); + buffer[buffer.size() - 1] = '\0'; + return buffer; +} + +template +constexpr auto variant_type_name = get_variant_type_name(); + +template +constexpr const char* get_type_name(const std::variant*) { + return variant_type_name.data(); +} + +template +constexpr const char* get_type_name() { + return get_type_name( (const T*)nullptr ); +} + +[[nodiscard]] inline constexpr bool char_to_name_digit_strict(char c, uint64_t& result) { + if (c >= 'a' && c <= 'z') { + result = (c - 'a') + 6; + return true; + } + if (c >= '1' && c <= '5') { + result = (c - '1') + 1; + return true; + } + if (c == '.') { + result = 0; + return true; + } + return false; +} + +[[nodiscard]] inline constexpr bool string_to_name_strict(uint64_t& name, std::string_view str) { + name = 0; + unsigned i = 0; + for (; i < str.size() && i < 12; ++i) { + uint64_t x = 0; + // - this is not safe in const expression OUTCOME_TRY(char_to_name_digit_strict(str[i], x)); + auto r = char_to_name_digit_strict(str[i], x); + if( !r ) return false; + name |= (x & 0x1f) << (64 - 5 * (i + 1)); + } + if (i < str.size() && i == 12) { + uint64_t x = 0; + // - this is not safe in const expression OUTCOME_TRY(char_to_name_digit_strict(str[i], x)); + auto r = char_to_name_digit_strict(str[i], x); + if( !r ) return false; + + if (x != (x & 0xf)) + return false; + name |= x; + ++i; + } + if (i < str.size()) + return false; + return true; +} + +inline constexpr uint64_t hash_name( std::string_view str ) { + uint64_t r = 0; + if( not string_to_name_strict(r, str) ) + return murmur64( str.data(), str.size() ); + return r; +} + +template +constexpr uint64_t get_type_hashname() { + return hash_name( get_type_name() ); +} + +} // namespace clio diff --git a/libraries/clio/include/clio/gql.hpp b/libraries/clio/include/clio/gql.hpp new file mode 100644 index 000000000..5211db444 --- /dev/null +++ b/libraries/clio/include/clio/gql.hpp @@ -0,0 +1,96 @@ +#pragma once +#include +#include + +#include +#include + +namespace clio { + + /** + * The concept of our graph-ql like interface is that a graph-ql query is + * parsed and converted into a gql query object and then flattened into something + * that requires no unpacking inorder to dispatch. This means converting gql strings + * into base32 integer representations and/or hashes if he name does not fit in + * base32. + * + * We use base32 (eg eosio::name) based 64bit integers as the key values because they + * map to swtich(key) statements for high performance dispatch by our resolvers. + */ + namespace gql { + + struct null_t{}; + struct entry; + struct scalar; + using object = std::vector; + using array = std::vector; + + + /** + * The scalar is the base value type, it is effectively a JSON object that uses + * base32 names for object keys and supports int64_t and uint64_t types in addition + * to double, string, object, array, and null. + * + * In principle you can convert from JSON to Scalar without any schema, and if you know + * the original keys for your objects, you can query the scalar for the value. In rare + * circumstances two keys may result in a hash collision, this should be detected at compile + * time and can be resolved by changing a field name. + * + * The flat view of a scalar is the same as a variant: + * [char_type][uint64_value] inline. If the type is a string,object,or array then the + * value field is a offset_ptr<> + * + */ + class scalar { + public: + scalar(){} + + template + scalar( const T& v ):value(v){} + + using value_type = std::variant; + + value_type value; + }; + using scalar_value = scalar::value_type; + CLIO_REFLECT_TYPENAME_CUSTOM( null_t, null ) + CLIO_REFLECT_TYPENAME_CUSTOM( scalar_value, scalar ) + + /** + * The key/value pair of an object + */ + struct entry { + uint64_t key; + scalar value; + }; + + + struct query; + struct query_filter; + + /** + * Defines what fields to select from the result of a query, + * if type is non-zero then the filter only applies if the type + * of the return value matches the variant type. + */ + struct query_filter { + std::vector filter; + /// the null type matches everything + uint64_t type = 0; + }; + + struct query { + uint64_t key; + uint64_t alias; + object args; + /// for each type... define the filter... + std::vector filter; + }; + CLIO_REFLECT( scalar, value ) + CLIO_REFLECT( entry, key, value ) + CLIO_REFLECT( query_filter, filter, type ) + CLIO_REFLECT( query, key, alias, args, filter ) + + } // namespace graphql + +} // namespace clio diff --git a/libraries/clio/include/clio/json/any.hpp b/libraries/clio/include/clio/json/any.hpp new file mode 100644 index 000000000..e2e8bf84e --- /dev/null +++ b/libraries/clio/include/clio/json/any.hpp @@ -0,0 +1,188 @@ +#pragma once +#include +#include +#include + +namespace clio { + + namespace json { + struct any; + struct entry; + struct null_t{};// uint8_t none = 0; }; + struct error_t{ std::string what; }; + + using any_object = std::vector; + using any_array = std::vector; + + struct any { + public: + any(){}; + template + any( const T& v ):_value(v){} + + template + const T* get_if()const { return std::get_if(&_value); } + + template + const T& as()const { + if( auto p = std::get_if(&_value) ) + return *p; + + assert( !"invalid type cast from any" ); + // static T dummy; /// used to prevent warnings about no return + // return dummy; + } + template + T& as(){ + if( auto p = std::get_if(&_value) ) + return *p; + + assert( !"invalid type cast from any" ); + __builtin_unreachable(); + } + + const auto& value()const { return _value; } + auto& value(){ return _value; } + + inline any operator[]( const std::string_view& key )const; + any operator[]( uint32_t index )const { + if( auto p = get_if() ) { + if( index >= p->size() ) return error_t{ "index out of bounds" }; + return p->at(index); + } + return error_t{ "not an array" }; + } + + template + void visit( Lambda&& l )const { + std::visit( std::forward(l), _value ); + } + private: + std::variant _value; + + template + friend void to_bin(const any& obj, S& stream) { + to_bin( obj._value, stream ); + } + template + friend void from_bin( any& obj, S& stream) { + from_bin( obj._value, stream ); + } + }; + + struct entry { + std::string key; + any value; + }; + CLIO_REFLECT( entry, key, value ) + CLIO_REFLECT_TYPENAME( null_t ) + CLIO_REFLECT( error_t, what ) + + + inline any any::operator[]( const std::string_view& key )const { + if( auto p = get_if() ) { + for( const auto& i : *p ) { + if( i.key == key ) return i.value; + } + } + return error_t{"not an object"}; + } + + template + void to_json(const any& obj, S& stream) { + std::visit( [&]( const auto& val ) { + using T = std::decay_t; + if constexpr ( std::is_same_v ) { + stream.write( "null", 4 ); + } + else if constexpr ( std::is_same_v ) { + stream.write( '{' ); + int size = val.size(); + for( const auto& e : val ) { + if( size == val.size() ) { + increase_indent(stream); + } + write_newline(stream); + to_json(e.key, stream); + write_colon(stream); + to_json(e.value, stream); + if( --size ) { + stream.write( ',' ); + } + } + if( !val.empty() ) { + decrease_indent(stream); + write_newline(stream); + } + stream.write( '}' ); + } else { + to_json( val, stream ); + } + }, obj.value() ); + } + + template + void from_json(any& obj, S& stream); + + template + void from_json(any_object& obj, S& stream) { + stream.get_start_object(); + while( true ) { + auto t = stream.peek_token(); + if (t.get().type == json_token_type::type_end_object) + break; + auto k = stream.get_key(); + obj.push_back( { .key = std::string(k) } ); + from_json( obj.back().value, stream ); + } + return stream.get_end_object(); + } + + template + void from_json(any& obj, S& stream) { + while( true ) { + auto t = stream.peek_token(); + switch( t.get().type ) { + case json_token_type::type_null: + stream.eat_token(); + obj = null_t{}; + return; + case json_token_type::type_bool: + obj = t.get().value_bool; + stream.eat_token(); + return; + case json_token_type::type_string: + obj = std::string(t.get().value_string); + stream.eat_token(); + return; + case json_token_type::type_start_object: { + obj = any_object(); + from_json( obj.as(), stream ); + return; + } + case json_token_type::type_start_array: { + obj = any_array(); + from_json( obj.as(), stream ); + return; + } + case json_token_type::type_end_array: + case json_token_type::type_end_object: + case json_token_type::type_key: + default: + stream.eat_token(); + throw_error(clio::from_json_error::value_invalid); + }; + } + } + + } /// namespace json + +} /// clio diff --git a/libraries/clio/include/clio/member_proxy.hpp b/libraries/clio/include/clio/member_proxy.hpp new file mode 100644 index 000000000..b1a2a8deb --- /dev/null +++ b/libraries/clio/include/clio/member_proxy.hpp @@ -0,0 +1,86 @@ +#pragma once + +namespace clio { + + template + struct member_proxy { + /** + * This object is created on a type created by the macro, I represents the member number in the + * containing type. The containing type's first member is a ProxyObject. This function does the + * pointer math necessary to find the ProxyObject. + * + * Alternatively the macro code would have to initialize every member_proxy with this, which would + * bloat the size of the member_proxy object + * + * flat_view => reflect::member_proxy + * + * struct member_proxy { + * proxy_impl proxy___; + * member_proxy<0, ptr, proxy_impl> member_zero + * member_proxy<1, ptr, proxy_impl> member_one + * member_proxy<2, ptr, proxy_impl> member_two + * } + * a packed flat buffer. + * + * Let char* buf = point to a flat buffer; + * Let reinterpet buf as memper_proxy*, this makes the address of proxy__ equal to + * the address of member_proxy because it is the first element. + * + * because member_proxy has no values it takes 1 byte in member_proxy and the value of that byte + * is never read by member_proxy... member_proxy always gets the address of proxy___ and then + * does offset math. + * + */ + constexpr auto proxyptr()const { + return (reinterpret_cast( reinterpret_cast(this)-sizeof(*this)*(I+1)) ); + } + constexpr auto proxyptr(){ + return (reinterpret_cast( reinterpret_cast(this)-sizeof(*this)*(I+1)) ); + } + constexpr const auto& get()const { return *(proxyptr()->template get()); } + constexpr auto& get() { return *(proxyptr()->template get()); } + + template + constexpr auto operator()(Ts&&... args) { + return proxyptr()->template call( std::forward(args)...); + } + template + constexpr auto operator()(Ts&&... args)const { + return proxyptr()->template call( std::forward(args)...); + } + + constexpr auto operator->(){ return (proxyptr()->template get()); } + constexpr const auto operator->()const { return (proxyptr()->template get()); } + + constexpr auto& operator*(){ return get(); } + constexpr const auto& operator*()const { return get(); } + + template + constexpr auto& operator[]( T&& k ) { return get()[ std::forward(k)]; } + + template + constexpr const auto& operator[]( T&& k )const { return get()[ std::forward(k)]; } + + template + friend S& operator << ( S& stream, const member_proxy& member) { + return stream << member.get(); + } + + template + auto operator=( R&& r ) { + get() = std::forward(r); + return *this; + } + + template + operator R ()const { return get(); } + + /* + operator decltype( ((ProxyObject*)nullptr)->template get())() + { return get(); } + operator decltype( ((const ProxyObject*)nullptr)->template get()) ()const + { return get(); } + */ + }; + +} /// clio diff --git a/libraries/clio/include/clio/murmur.hpp b/libraries/clio/include/clio/murmur.hpp new file mode 100644 index 000000000..9a231afa7 --- /dev/null +++ b/libraries/clio/include/clio/murmur.hpp @@ -0,0 +1,55 @@ +#pragma once +namespace clio { +namespace { + inline constexpr uint64_t unaligned_load(const char* p) + { + uint64_t r = 0; + for( uint32_t i = 0; i < 8; ++i ) { + r |= p[i]; + r <<= 8; + } + return r; + } + + // Loads n bytes, where 1 <= n < 8. + inline constexpr uint64_t load_bytes(const char* p, int n) + { + uint64_t result = 0; + --n; + do + result = (result << 8) + (unsigned char)(p[n]); + while (--n >= 0); + return result; + } + + inline constexpr uint64_t shift_mix(std::uint64_t v) + { return v ^ (v >> 47);} +} // namespace + +// Implementation of Murmur hash for 64-bit size_t. + inline constexpr uint64_t murmur64(const char* ptr, uint64_t len, uint64_t seed = 0xbadd00d00) + { + const uint64_t mul = (((uint64_t) 0xc6a4a793UL) << 32UL) + + (uint64_t) 0x5bd1e995UL; + const char* const buf = ptr; + + // Remove the bytes not divisible by the sizeof(uint64_t). This + // allows the main loop to process the data as 64-bit integers. + const uint64_t len_aligned = len & ~(uint64_t)0x7; + const char* const end = buf + len_aligned; + uint64_t hash = seed ^ (len * mul); + for (const char* p = buf; p != end; p += 8) { + const uint64_t data = shift_mix(unaligned_load(p) * mul) * mul; + hash ^= data; + hash *= mul; + } + if ((len & 0x7) != 0) { + const uint64_t data = load_bytes(end, len & 0x7); + hash ^= data; + hash *= mul; + } + hash = shift_mix(hash) * mul; + hash = shift_mix(hash); + return hash; + } +} // namespace eosio diff --git a/libraries/clio/include/clio/name.hpp b/libraries/clio/include/clio/name.hpp new file mode 100644 index 000000000..b52050973 --- /dev/null +++ b/libraries/clio/include/clio/name.hpp @@ -0,0 +1,198 @@ +#pragma once +#include +#include +#include + +namespace clio { + +inline std::string name_to_string(uint64_t name) { + static const char* charmap = ".12345abcdefghijklmnopqrstuvwxyz"; + std::string str(13, '.'); + + uint64_t tmp = name; + for (uint32_t i = 0; i <= 12; ++i) { + char c = charmap[tmp & (i == 0 ? 0x0f : 0x1f)]; + str[12 - i] = c; + tmp >>= (i == 0 ? 4 : 5); + } + + const auto last = str.find_last_not_of('.'); + return str.substr(0, last + 1); +} + +inline constexpr uint64_t char_to_name_digit(char c) { + if (c >= 'a' && c <= 'z') + return (c - 'a') + 6; + if (c >= '1' && c <= '5') + return (c - '1') + 1; + return 0; +} + +inline constexpr uint64_t string_to_name(const char* str, int size) { + uint64_t name = 0; + int i = 0; + for (; i < size && i < 12; ++i) name |= (char_to_name_digit(str[i]) & 0x1f) << (64 - 5 * (i + 1)); + if (i < size) + name |= char_to_name_digit(str[i]) & 0x0F; + return name; +} + +inline constexpr uint64_t string_to_name(const char* str) { + int len = 0; + while (str[len]) ++len; + return string_to_name(str, len); +} + +inline constexpr uint64_t string_to_name(std::string_view str) { return string_to_name(str.data(), str.size()); } + +struct name { + enum class raw : uint64_t {}; + uint64_t value = 0; + + constexpr name() = default; + constexpr explicit name(uint64_t value) : value{ value } {} + constexpr explicit name(name::raw value) : value{ static_cast(value) } {} + constexpr explicit name(std::string_view str) { + if( not string_to_name_strict( value, str ) ) { + // TODO: throw_error + } + } + + + constexpr name(const name&) = default; + + constexpr operator raw() const { return static_cast(value); } + explicit operator std::string() const { return name_to_string(value); } + std::string to_string() const { return std::string(*this); } + /** + * Explicit cast to bool of the uint64_t value of the name + * + * @return Returns true if the name is set to the default value of 0 else true. + */ + constexpr explicit operator bool() const { return value != 0; } + + /** + * Converts a %name Base32 symbol into its corresponding value + * + * @param c - Character to be converted + * @return constexpr char - Converted value + */ + static constexpr uint8_t char_to_value(char c) { + if (c == '.') + return 0; + else if (c >= '1' && c <= '5') + return (c - '1') + 1; + else if (c >= 'a' && c <= 'z') + return (c - 'a') + 6; + return 0; // control flow will never reach here; just added to suppress warning + } + + /** + * Returns the length of the %name + */ + constexpr uint8_t length() const { + constexpr uint64_t mask = 0xF800000000000000ull; + + if (value == 0) + return 0; + + uint8_t l = 0; + uint8_t i = 0; + for (auto v = value; i < 13; ++i, v <<= 5) { + if ((v & mask) > 0) { + l = i; + } + } + + return l + 1; + } + + /** + * Returns the suffix of the %name + */ + constexpr name suffix() const { + uint32_t remaining_bits_after_last_actual_dot = 0; + uint32_t tmp = 0; + for (int32_t remaining_bits = 59; remaining_bits >= 4; + remaining_bits -= 5) { // Note: remaining_bits must remain signed integer + // Get characters one-by-one in name in order from left to right (not including the 13th character) + auto c = (value >> remaining_bits) & 0x1Full; + if (!c) { // if this character is a dot + tmp = static_cast(remaining_bits); + } else { // if this character is not a dot + remaining_bits_after_last_actual_dot = tmp; + } + } + + uint64_t thirteenth_character = value & 0x0Full; + if (thirteenth_character) { // if 13th character is not a dot + remaining_bits_after_last_actual_dot = tmp; + } + + if (remaining_bits_after_last_actual_dot == + 0) // there is no actual dot in the %name other than potentially leading dots + return name{ value }; + + // At this point remaining_bits_after_last_actual_dot has to be within the range of 4 to 59 (and restricted to + // increments of 5). + + // Mask for remaining bits corresponding to characters after last actual dot, except for 4 least significant bits + // (corresponds to 13th character). + uint64_t mask = (1ull << remaining_bits_after_last_actual_dot) - 16; + uint32_t shift = 64 - remaining_bits_after_last_actual_dot; + + return name{ ((value & mask) << shift) + (thirteenth_character << (shift - 1)) }; + } + + /** + * Returns the prefix of the %name + */ + constexpr name prefix() const { + uint64_t result = value; + bool not_dot_character_seen = false; + uint64_t mask = 0xFull; + + // Get characters one-by-one in name in order from right to left + for (int32_t offset = 0; offset <= 59;) { + auto c = (value >> offset) & mask; + + if (!c) { // if this character is a dot + if (not_dot_character_seen) { // we found the rightmost dot character + result = (value >> offset) << offset; + break; + } + } else { + not_dot_character_seen = true; + } + + if (offset == 0) { + offset += 4; + mask = 0x1Full; + } else { + offset += 5; + } + } + + return name{ result }; + } +}; +CLIO_REFLECT( name, value ) + + +template +void from_json(name& obj, S& stream) { + auto r = stream.get_string(); + obj = name(hash_name(r)); +} + +template +void to_json(const name& obj, S& stream) { + to_json(name_to_string(obj.value), stream); +} + +inline namespace literals { + inline constexpr name operator""_n(const char* s, size_t) { return name( std::string_view(s) ); } + inline constexpr name operator""_h(const char* s, size_t) { return name( hash_name(s) ); } +} // namespace literals + +} // namespace clio diff --git a/libraries/clio/include/clio/powers.h b/libraries/clio/include/clio/powers.h new file mode 100644 index 000000000..d2acf6707 --- /dev/null +++ b/libraries/clio/include/clio/powers.h @@ -0,0 +1,77 @@ + +/// From https://github.com/night-shift/fpconv +/// Boost Software License 1.0 +/// See accompanying license file + +#pragma once + +#include + +#define npowers 87 +#define steppowers 8 +#define firstpower -348 /* 10 ^ -348 */ + +#define expmax -32 +#define expmin -60 + +typedef struct Fp { + uint64_t frac; + int exp; +} Fp; + +static constexpr Fp powers_ten[] = { + { 18054884314459144840U, -1220 }, { 13451937075301367670U, -1193 }, { 10022474136428063862U, -1166 }, + { 14934650266808366570U, -1140 }, { 11127181549972568877U, -1113 }, { 16580792590934885855U, -1087 }, + { 12353653155963782858U, -1060 }, { 18408377700990114895U, -1034 }, { 13715310171984221708U, -1007 }, + { 10218702384817765436U, -980 }, { 15227053142812498563U, -954 }, { 11345038669416679861U, -927 }, + { 16905424996341287883U, -901 }, { 12595523146049147757U, -874 }, { 9384396036005875287U, -847 }, + { 13983839803942852151U, -821 }, { 10418772551374772303U, -794 }, { 15525180923007089351U, -768 }, + { 11567161174868858868U, -741 }, { 17236413322193710309U, -715 }, { 12842128665889583758U, -688 }, + { 9568131466127621947U, -661 }, { 14257626930069360058U, -635 }, { 10622759856335341974U, -608 }, + { 15829145694278690180U, -582 }, { 11793632577567316726U, -555 }, { 17573882009934360870U, -529 }, + { 13093562431584567480U, -502 }, { 9755464219737475723U, -475 }, { 14536774485912137811U, -449 }, + { 10830740992659433045U, -422 }, { 16139061738043178685U, -396 }, { 12024538023802026127U, -369 }, + { 17917957937422433684U, -343 }, { 13349918974505688015U, -316 }, { 9946464728195732843U, -289 }, + { 14821387422376473014U, -263 }, { 11042794154864902060U, -236 }, { 16455045573212060422U, -210 }, + { 12259964326927110867U, -183 }, { 18268770466636286478U, -157 }, { 13611294676837538539U, -130 }, + { 10141204801825835212U, -103 }, { 15111572745182864684U, -77 }, { 11258999068426240000U, -50 }, + { 16777216000000000000U, -24 }, { 12500000000000000000U, 3 }, { 9313225746154785156U, 30 }, + { 13877787807814456755U, 56 }, { 10339757656912845936U, 83 }, { 15407439555097886824U, 109 }, + { 11479437019748901445U, 136 }, { 17105694144590052135U, 162 }, { 12744735289059618216U, 189 }, + { 9495567745759798747U, 216 }, { 14149498560666738074U, 242 }, { 10542197943230523224U, 269 }, + { 15709099088952724970U, 295 }, { 11704190886730495818U, 322 }, { 17440603504673385349U, 348 }, + { 12994262207056124023U, 375 }, { 9681479787123295682U, 402 }, { 14426529090290212157U, 428 }, + { 10748601772107342003U, 455 }, { 16016664761464807395U, 481 }, { 11933345169920330789U, 508 }, + { 17782069995880619868U, 534 }, { 13248674568444952270U, 561 }, { 9871031767461413346U, 588 }, + { 14708983551653345445U, 614 }, { 10959046745042015199U, 641 }, { 16330252207878254650U, 667 }, + { 12166986024289022870U, 694 }, { 18130221999122236476U, 720 }, { 13508068024458167312U, 747 }, + { 10064294952495520794U, 774 }, { 14996968138956309548U, 800 }, { 11173611982879273257U, 827 }, + { 16649979327439178909U, 853 }, { 12405201291620119593U, 880 }, { 9242595204427927429U, 907 }, + { 13772540099066387757U, 933 }, { 10261342003245940623U, 960 }, { 15290591125556738113U, 986 }, + { 11392378155556871081U, 1013 }, { 16975966327722178521U, 1039 }, { 12648080533535911531U, 1066 } +}; + +static inline constexpr Fp find_cachedpow10(int exp, int* k) { + const double one_log_ten = 0.30102999566398114; + + int approx = -(exp + npowers) * one_log_ten; + int idx = (approx - firstpower) / steppowers; + + while (1) { + int current = exp + powers_ten[idx].exp + 64; + + if (current < expmin) { + idx++; + continue; + } + + if (current > expmax) { + idx--; + continue; + } + + *k = (firstpower + idx * steppowers); + + return powers_ten[idx]; + } +} diff --git a/libraries/clio/include/clio/protobuf/any.hpp b/libraries/clio/include/clio/protobuf/any.hpp new file mode 100644 index 000000000..75823b8bd --- /dev/null +++ b/libraries/clio/include/clio/protobuf/any.hpp @@ -0,0 +1,143 @@ +#pragma once +#include +#include +#include + +namespace clio { + namespace protobuf { + enum wire_type_enum { + varint = 0, + fixed64 = 1, + buffer = 2, + fixed32 = 5 + }; + + struct entry { + typedef std::variant value_type; + + entry(){} + template + entry( uint32_t n, T&& v ) + :number(n),value( std::forward(v) ){} + + uint32_t number; + value_type value; + }; + CLIO_REFLECT( entry, number, value ) + + /** + * This class can be used to hold the + * deserialized contents of any protobuf stream + */ + struct any { + std::vector< entry > members; + + void add( uint32_t field, const std::string& s ) { + members.push_back( entry{ field, bytes{ std::vector(s.begin(), s.end()) } } ); + } + template::value > > + void add( uint32_t field, const std::vector& s ) { + members.push_back( entry{ field, + bytes{ std::vector( (char*)s.data(), (char*)(s.data()+s.size())) } } ); + } + void add( uint32_t field, std::vector&& s ) { + members.push_back( entry{ field, bytes{ std::move(s) } } ); + } + void add( uint32_t field, varuint32 s ) { + members.push_back( entry{ field, s } ); + } + void add( uint32_t field, uint64_t s ) { + members.push_back( entry{ field, static_cast(s) } ); + } + void add( uint32_t field, int64_t s ) { + members.push_back( entry{ field, s } ); + } + void add( uint32_t field, int32_t s ) { + members.push_back( entry{ field, s } ); + } + void add( uint32_t field, int16_t s ) { + members.push_back( entry{ field, varuint32(s) } ); + } + void add( uint32_t field, uint16_t s ) { + members.push_back( entry{ field, varuint32(s) } ); + } + void add( uint32_t field, int8_t s ) { + members.push_back( entry{ field, varuint32(s) } ); + } + void add( uint32_t field, uint8_t s ) { + members.push_back( entry{ field, varuint32(s) } ); + } + void add( uint32_t field, bool s ) { + members.push_back( entry{ field, varuint32(s) } ); + } + void add( uint32_t field, char s ) { + members.push_back( entry{ field, varuint32(s) } ); + } + void add( uint32_t field, double s ) { + members.push_back( entry{ field, *reinterpret_cast(&s) } ); + } + void add( uint32_t field, float s ) { + members.push_back( entry{ field, *reinterpret_cast(&s) } ); + } + }; + CLIO_REFLECT( any, members ) + + + template + void to_bin( const any& a, Stream& s ) { + for( const auto& e : a.members ) { + uint32_t key = e.number << 3; + key |= e.value.index() == 3 ? wire_type_enum::fixed32 : e.value.index(); + varuint32_to_bin( key, s ); + + std::visit( [&]( const auto& v ){ + to_bin( v, s ); + }, e.value ); + } + } + + template + void from_bin( any& a, Stream& s ) { + while( s.remaining() ) { + uint32_t key = 0; + varuint32_from_bin( key, s ); + wire_type_enum type = wire_type_enum(uint8_t(key) & 0x07); + uint32_t number = key >> 3; + + switch( type ) { + case wire_type_enum::varint: { + varuint32 val; + from_bin( val, s ); + a.members.push_back( {number, val} ); + } break; + case wire_type_enum::fixed64: { + int64_t val; + from_bin( val, s ); + a.members.push_back( {number, val} ); + } break; + case wire_type_enum::buffer: { + bytes val; + from_bin( val, s ); + a.members.emplace_back( number, std::move(val) ); + } break; + case wire_type_enum::fixed32: { + int32_t val; + from_bin( val, s ); + a.members.push_back( {number, val} ); + } break; + } + } + } + + } /// namespace protobuf + + template + void to_protobuf( const protobuf::any& a, Stream& s ) { + to_bin( a, s ); + } + template + void from_protobuf( protobuf::any& a, Stream& s ) { + from_bin( a, s ); + } + +} /// namespace clio diff --git a/libraries/clio/include/clio/protobuf/json.hpp b/libraries/clio/include/clio/protobuf/json.hpp new file mode 100644 index 000000000..8ddd585c7 --- /dev/null +++ b/libraries/clio/include/clio/protobuf/json.hpp @@ -0,0 +1,172 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace clio { + + namespace protobuf { + + struct json_field_query; + struct json_query { + std::vector fields; + }; + struct json_field_query { + std::string name; + json::any args; /// args as json encoded tuple, or named object + json_query filter; /// apply to result + }; + CLIO_REFLECT( json_field_query, name, args, filter ) + CLIO_REFLECT( json_query, fields ) + + + /** + * Given a json description of the query and an assumed reflected type (used to interpret names to numbers), + * this function will convert the JSON-query into a protobuf-style query. + */ + template + protobuf::query from_json_query( const json_query& jq ) { + protobuf::query result; + result.fields.reserve( jq.fields.size() ); + + if constexpr( reflect::is_struct ) { + for( const auto& field : jq.fields ) { + reflect::for_each( [&](const meta& ref, auto mptr ) { + if constexpr( not std::is_member_function_pointer_v< decltype(mptr) > ) { + if( ref.name == field.name ) { + if( field.filter.fields.size() ) { + result.fields.push_back({ + .number = ref.number, + .filter = from_json_query(nullptr)->*mptr)>>( field.filter ) + }); + } + else { + result.fields.push_back( field_query{ ref.number } ); + } + } + } else { /// member function ptr + if( ref.name == field.name ) { + /// TODO: do a direct conversion from json::any to pb bytes instead of + /// any->json_string->native->pb + using args_tuple_type = decltype( args_as_tuple( mptr ) ); + std::string json = convert_to_json( field.args ); + auto args_value = clio::from_json(std::move(json)); + + if( field.filter.fields.size() ) { + result.fields.push_back({ + .number = ref.number, + .args = to_protobuf( args_value ), + .filter = from_json_query< decltype( result_of(mptr) )>( field.filter ) + }); + } else { + result.fields.push_back( field_query{ + .number = ref.number, + .args = to_protobuf( args_value ) + }); + } + } + } + }); + } + } + return result; + } + + /** + * Given a protobuf query and a type, convert it into a friendly JSON Query + */ + template + json_query to_json_query( const protobuf::query& pbuf ) { + json_query result; + + if constexpr( reflect::is_struct ) { + for( const auto& entry : pbuf.fields ) { + reflect::for_each( [&](const meta& ref, auto mptr ) { + if constexpr( not std::is_member_function_pointer_v< decltype(mptr) > ) { + if( ref.number == entry.number.value ) { + if( entry.filter.fields.size() ) { + result.fields.push_back({ + .name = ref.name, + .filter = to_json_query(nullptr)->*mptr)>>( entry.filter ) + }); + } + else { + result.fields.push_back( { ref.name} ); + } + } + } else { /// member function ptr + if( ref.number == entry.number.value ) { + /// TODO: do a direct conversion from PB to json::any instead of + /// pb->native->json_string->any + using args_tuple_type = decltype( args_as_tuple( mptr ) ); + auto pbargs = from_protobuf( entry.args.data ); + auto jargs = to_json( pbargs ); + + if( entry.filter.fields.size() ) { + result.fields.push_back({ + .name = ref.name, + .args = from_json( jargs ), + .filter = to_json_query< decltype( result_of(mptr) )>( entry.filter ) + }); + } else { + result.fields.push_back( { + .name = ref.name, + .args = from_json( jargs ) + }); + } + } + } + }); + } + } + return result; + } + + } /// namespace protobuf + template + void to_json(const protobuf::json_query& s, S& stream) { + to_json(s.fields, stream); + } + template + void from_json( protobuf::json_query& s, S& stream) { + from_json(s.fields, stream); + } + template + void to_json(const protobuf::json_field_query& s, S& stream) { + stream.write('{'); + increase_indent(stream); + write_newline(stream); + stream.write("\"name\""); + write_colon(stream); + to_json( s.name, stream ); + if( auto* a = s.args.get_if() ) { + if( a->size() ) { + stream.write(','); + write_newline(stream); + stream.write("\"args\""); + write_colon(stream); + to_json( s.args, stream ); + } + } + else if( not s.args.get_if() ) { + stream.write(','); + write_newline(stream); + stream.write("\"args\""); + write_colon(stream); + to_json( s.args, stream ); + } + if( s.filter.fields.size() ) { + stream.write(','); + write_newline(stream); + stream.write("\"filter\""); + write_colon(stream); + to_json( s.filter, stream ); + } + decrease_indent(stream); + write_newline(stream); + stream.write('}'); + } + +} /// clio diff --git a/libraries/clio/include/clio/protobuf/query.hpp b/libraries/clio/include/clio/protobuf/query.hpp new file mode 100644 index 000000000..7147172e1 --- /dev/null +++ b/libraries/clio/include/clio/protobuf/query.hpp @@ -0,0 +1,201 @@ +#pragma once +#include +#include + +namespace clio { + + namespace protobuf { + + struct field_query; + struct variant_query; + + struct variant_query { + varuint32 type; + std::vector fields; + }; + + struct query { + std::vector fields; + std::vector variant_fields; /// only apply these fields when the type has them + }; + + struct field_query { + varuint32 number; + bytes args; /// args as protobuf encoded tuple + query filter; /// apply to result + }; + + CLIO_REFLECT( variant_query, type, fields ) + CLIO_REFLECT( field_query, number, args, filter ) + CLIO_REFLECT( query, fields, variant_fields ) + + + + template + any dispatch( T&& obj, const query& q ) { + any result; + result.members.reserve( q.fields.size() ); + + for( const auto& field : q.fields ) { + reflect::get( int64_t(field.number), [&]( auto mptr ){ + using member_ptr_type = decltype(mptr); + if constexpr ( std::is_member_function_pointer_v ) { + using return_type = decltype( result_of(mptr) ); + using param_type = decltype( args_as_tuple(mptr) ); + // std::cerr<< boost::core::demangle(typeid(param_type).name()) <<"\n"; + + param_type params; + input_stream in( field.args.data.data(), field.args.data.size() ); + (void) from_protobuf_object( params, in ); + std::apply( [&]( auto... args ){ + if constexpr( is_std_variant::value ) { + /// TODO: + } else if constexpr( not reflect::is_struct ) + result.add( field.number, (obj.*mptr)( args... ) ); + else { + any field_data = dispatch( (obj.*mptr)( args...) , field.filter ); + result.add( field.number, clio::to_bin( field_data ) ); + } + }, params ); + + } else if constexpr ( not std::is_member_function_pointer_v ) { + using member_type = std::decay_t; + + if constexpr( is_std_tuple::value ) { + /// TODO: + } + else if constexpr( is_std_variant::value ) { + /// TODO: + } + else if constexpr ( is_std_vector::value ) + { + using value_type = typename is_std_vector::value_type; + if constexpr( is_std_variant::value ) { + /// TODO: + } else if constexpr ( std::is_arithmetic_v< value_type > ) { + result.add( field.number, obj.*mptr ); + } else if constexpr( reflect::is_struct ) { + for( const auto& item : obj.*mptr ) { + any field_data = dispatch( item, field.filter ); + result.add( field.number, clio::to_bin( field_data ) ); + } + } else { + for( const auto& item : obj.*mptr ) { + result.add( field.number, item ); + } + } + } + else if constexpr ( not reflect::is_struct ) + { + result.add( field.number, obj.*mptr ); + } + else + { + any field_data = dispatch( obj.*mptr, field.filter ); + result.add( field.number, clio::to_bin( field_data ) ); + } + + } + }); + } + return result; + } + + struct query_proxy { + protobuf::query q; + + template + query_proxy& call( Args&&... args ) { + return *this; + } + template + void get() { + } + + template + void get()const { + } + + template + query_proxy& call( const meta& ref, Mptr mptr, Filter&& filter ) { + using result_type = decltype( result_of(mptr) ); + using args_tuple = decltype(args_as_tuple( mptr )); + + if constexpr ( std::tuple_size::value == 1 ) { + auto params = args_tuple( std::forward(filter) ); + q.fields.push_back( { + .number = ref.number, + .args = to_protobuf( params ), + }); + } else { + typename reflect::template proxy rp; + filter( rp ); + q.fields.push_back( { + .number = ref.number, + .filter = std::move(rp->q) + }); + } + return *this; + } + + template + query_proxy& call( const meta& ref, Mptr mptr ) { + q.fields.push_back( { + .number = ref.number, + }); + return *this; + } + + + template + query_proxy& call( const meta& ref, Mptr mptr, Filter&& filter, Args&&... args ) { + using result_type = decltype( result_of(mptr) ); + using args_tuple = decltype(args_as_tuple( mptr )); + /** test to see whether the user passed a filter by comparing the number of args, + * if the number of var args equals the number of paramters then a filter was passed, + * otherwise the Filter is being interpreted as the first argument. This means that in + * order to use a filter you must pass all args, if you don't want a filter on the result + * then you can skip trailing args. + */ + if constexpr ( std::tuple_size::value == sizeof...(Args) ) { + auto params = args_tuple( std::forward(args)... ); + + if constexpr( reflect::is_struct ) { + typename reflect::template proxy rp; + filter( rp ); + q.fields.push_back( { + .number = ref.number, + .args = to_protobuf( params ), + .filter = std::move(rp->q) + }); + } else { + q.fields.push_back( { + .number = ref.number, + .args = to_protobuf( params ), + }); + } + } else { + auto params = args_tuple( std::forward(filter), std::forward(args)... ); + + if constexpr( reflect::is_struct ) { + typename reflect::template proxy rp; + q.fields.push_back( { + .number = ref.number, + .args = to_protobuf( params ), + .filter = std::move(rp->q) + }); + } else { + q.fields.push_back( { + .number = ref.number, + .args = to_protobuf( params ), + }); + } + } + return *this; + } + }; + + + } // namespace protobuf + +} /// namespace clio diff --git a/libraries/clio/include/clio/protobuf/schema.hpp b/libraries/clio/include/clio/protobuf/schema.hpp new file mode 100644 index 000000000..d19575dab --- /dev/null +++ b/libraries/clio/include/clio/protobuf/schema.hpp @@ -0,0 +1,187 @@ +#pragma once +#include + +namespace clio { + + /** + * Writes a protobuf schem file to stream given a schema object + */ + template + void to_protobuf_schema( const schema& sch, S& stream ) { + + stream.write( "syntax = \"proto3\";\n" ); + + auto convert_variant_name = []( string name ) { + for( auto& c : name ) { + if( c == '|' ) c = '_'; + if( c == '?' ) c = 'O'; + if( c == '[' ) c = '_'; + if( c == ']' ) c = 'a'; + } + return "variant_" + name.substr( 0, name.size()-1); + }; + auto convert_tuple_name = []( string name ) { + for( auto& c : name ) { + if( c == '&' ) c = '_'; + if( c == '?' ) c = 'O'; + if( c == '[' ) c = '_'; + if( c == ']' ) c = 'a'; + } + return "tuple_" + name.substr( 0, name.size()-1); + }; + + auto convert_map_name = []( string name ) { + /* //TODO... + for( auto& c : name ) { + if( c == '&' ) c = '_'; + if( c == '?' ) c = 'O'; + if( c == '[' ) c = '_'; + if( c == ']' ) c = 'a'; + } + */ + return name; + }; + + auto decay_type = [&]( const std::string& str, bool& is_repeated ) -> std::string { + if( str.back() == ']' ) { + is_repeated = true; + return str.substr( 0, str.size()-2); + } + else if( str.back() == '?' ) return str.substr( 0, str.size()-1); + else if( str.back() == '|' ) return convert_variant_name( str ); + else if( str.back() == '&' ) return convert_tuple_name( str ); + else if( str.back() == '>' ) return convert_map_name( str ); + else return str; + }; + + auto convert_to_pb_type = [&]( const string& str, bool& is_repeated, bool& is_prim ) -> std::string { + auto decay = decay_type( str, is_repeated ); + is_prim = true; + if( decay == "int16_t" ) return "int32"; + else if( decay == "char" ) return "int32"; + else if( decay == "int8_t" ) return "int32"; + else if( decay == "uint16_t" ) return "int32"; + else if( decay == "uint8_t" ) return "int32"; + else if( decay == "int32_t" ) return "sfixed32"; + else if( decay == "uint32_t" ) return "fixed32"; + else if( decay == "int64_t" ) return "sfixed64"; + else if( decay == "uint64_t" ) return "fixed64"; + else if( decay == "bool" ) return "bool"; + else if( decay == "double" ) return decay; + else if( decay == "float" ) return decay; + is_prim = false; + return decay; + }; + + for( const auto& item : sch.types ) { + std::visit( [&]( auto i ) { + if constexpr ( std::is_same_v ) { + stream.write( "message ", 8 ); + stream.write( item.first ); + stream.write( " {\n" ); + for( auto mem : i.members ) { + bool is_repeated = false; + bool is_prim = false; + auto pbt = convert_to_pb_type( mem.type, is_repeated, is_prim ); + stream.write( " " ); + if( is_repeated ) { + stream.write( "repeated " ); + } + stream.write( pbt ); + stream.write( " " ); + stream.write( mem.name.c_str() ); + stream.write( " = " ); + stream.write( std::to_string( mem.number ).c_str() ); + if( is_repeated && is_prim ) { + stream.write( " [packed=true]" ); + } + stream.write( ";\n" ); + } + /* + for( auto mem : i.methods ) { + bool is_repeated = false; + bool is_prim = false; + auto pbt = convert_to_pb_type( mem.return_type, is_repeated, is_prim ); + stream.write( " " ); + stream.write( "repeated " ); + stream.write( pbt ); + stream.write( " " ); + stream.write( mem.name.c_str() ); + stream.write( " = " ); + stream.write( std::to_string( mem.number ).c_str() ); + stream.write( ";\n" ); + }*/ + stream.write( "}\n" ); + } else if constexpr ( std::is_same_v ) { + stream.write( "message " ); + if( item.first.back() == '|' ) { + stream.write( convert_variant_name(item.first) ); + } else { + stream.write( item.first ); + } + + stream.write( " {\n" ); + stream.write( " oneof which {\n" ); + + for( uint32_t idx = 0; idx < i.types.size(); ++idx ) { + bool is_repeated = false; + bool is_prim = false; + auto pbt = convert_to_pb_type( i.types[idx], is_repeated, is_prim ); + + stream.write( " " ); + stream.write( pbt ); + stream.write( " " ); + stream.write( pbt ); + stream.write( "_value = " ); + stream.write( std::to_string(idx+1)); + stream.write( ";\n" ); + } + stream.write( " }\n" ); + stream.write( "}\n" ); + } else if constexpr ( std::is_same_v ) { + stream.write( "message " ); + + if( item.first.back() == '&' ) { + stream.write( convert_tuple_name(item.first) ); + } else { + stream.write( item.first ); + } + + stream.write( " {\n" ); + for( uint32_t idx = 0; idx < i.types.size(); ++idx ) { + stream.write( " " ); + bool is_repeated = false; + bool is_prim = false; + auto pbt = convert_to_pb_type( i.types[idx], is_repeated, is_prim ); + if( is_repeated ) { + stream.write( "repeated " ); + } + stream.write( pbt ); + stream.write( " _" ); + stream.write( std::to_string(idx+1) ); + stream.write( " = " ); + stream.write( std::to_string(idx+1)); + if( is_repeated && is_prim ) { + stream.write( " [packed=true]" ); + } + stream.write( ";\n" ); + } + stream.write( "}\n" ); + } + }, item.second ); + } + } + inline std::string to_protobuf_schema( const schema& s ) { + size_stream ss; + to_protobuf_schema(s, ss); + + std::string result(ss.size, 0); + fixed_buf_stream fbs(result.data(), result.size()); + to_protobuf_schema(s, fbs); + + if (fbs.pos != fbs.end) + throw_error(stream_error::underrun); + + return result; + } +} /// clio diff --git a/libraries/clio/include/clio/reflect.hpp b/libraries/clio/include/clio/reflect.hpp new file mode 100644 index 000000000..705aa12d7 --- /dev/null +++ b/libraries/clio/include/clio/reflect.hpp @@ -0,0 +1,388 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace clio { + + template + std::tuple...> args_as_tuple( R (C::*)(Args...) ); + template + std::tuple...> args_as_tuple( R (C::*)(Args...)const ); + + template + R result_of( R (C::*)(Args...)const ); + template + R result_of( R (C::*)(Args...) ); + + template + constexpr R result_of_member( R (C::*) ); + template + constexpr C class_of_member( R (C::*) ); + + template + void result_of_member( R (C::*)(Args...)const ); + template + void result_of_member( R (C::*)(Args...) ); + + + struct meta { + const char* name; + int32_t number; + std::initializer_list param_names; + }; + + #define CLIO_REFLECT_ARGS_INTERNAL( r, OP, i, PARAM) \ + BOOST_PP_COMMA_IF(i) BOOST_PP_STRINGIZE(PARAM) + + #define CLIO_REFLECT_ARGS_HELPER( METHOD, PARAM_NAMES ) \ + BOOST_PP_SEQ_FOR_EACH_I( CLIO_REFLECT_ARGS_INTERNAL, METHOD, PARAM_NAMES ) + + + #define CLIO_REFLECT_FILTER_PARAMS(NAME, IDX, ...) { CLIO_REFLECT_ARGS_HELPER( METHOD, BOOST_PP_VARIADIC_TO_SEQ( __VA_ARGS__ ) ) } + #define CLIO_REFLECT_FILTER_NAME(NAME, IDX, ...) NAME + #define CLIO_REFLECT_FILTER_NAME_STR(NAME, IDX, ...) BOOST_PP_STRINGIZE(NAME) + #define CLIO_REFLECT_FILTER_IDX(NAME, IDX, ...) IDX + + + #define CLIO_REFLECT_FOREACH_PB_INTERNAL( r, OP, member) \ + lambda( clio::meta{ .number = CLIO_REFLECT_FILTER_IDX member, \ + .name = CLIO_REFLECT_FILTER_NAME_STR member, \ + .param_names = CLIO_REFLECT_FILTER_PARAMS member }, \ + &OP::CLIO_REFLECT_FILTER_NAME member);\ + + + #define CLIO_REFLECT_FOREACH_INTERNAL( r, OP, i, member) \ + (void)lambda( clio::meta{ \ + .name = BOOST_PP_STRINGIZE(member), \ + .number = i+1 , \ + }, \ + &OP::member);\ + + #define CLIO_REFLECT_MEMBER_BY_STR_INTERNAL( r, OP, member) \ + if( BOOST_PP_STRINGIZE(member) == m ) { \ + (void)lambda( &OP::member ); return true; \ + } + + #define CLIO_REFLECT_MEMBER_BY_STR_PB_INTERNAL( r, OP, member) \ + if( CLIO_REFLECT_FILTER_NAME_STR member == m ) { \ + (void)lambda( &OP::CLIO_REFLECT_FILTER_NAME member ); return true; \ + } + + + #define CLIO_REFLECT_MEMBER_BY_IDX_PB_INTERNAL( r, OP, member) \ + case CLIO_REFLECT_FILTER_IDX member: (void)lambda( &OP:: CLIO_REFLECT_FILTER_NAME member); return true; + +#define CLIO_REFLECT_PROXY_MEMBER_BY_IDX_INTERNAL( r, OP, I, member ) \ + template \ + auto member( Args&&... args ) { \ + return proxy___.call( clio::meta{ .number = I+1, \ + .name = BOOST_PP_STRINGIZE(member)\ + }, \ + &OP::member, std::forward(args)... ); \ + } \ + + +#define CLIO_REFLECT_MPROXY_MEMBER_BY_IDX_INTERNAL( r, OP, I, member ) \ + clio::member_proxy member; + +#define CLIO_REFLECT_MPROXY_MEMBER_BY_IDX_HELPER( QUERY_CLASS, MEMBER_IDXS ) \ + BOOST_PP_SEQ_FOR_EACH_I( CLIO_REFLECT_MPROXY_MEMBER_BY_IDX_INTERNAL, QUERY_CLASS, MEMBER_IDXS ) + + +#define CLIO_REFLECT_PROXY_MEMBER_BY_PB_INTERNAL( r, OP, member ) \ + template \ + auto CLIO_REFLECT_FILTER_NAME member ( Args&&... args ) { \ + return proxy___.call( clio::meta{ .number = CLIO_REFLECT_FILTER_IDX member, \ + .name = CLIO_REFLECT_FILTER_NAME_STR member, \ + .param_names = CLIO_REFLECT_FILTER_PARAMS member }, \ + &OP::CLIO_REFLECT_FILTER_NAME member, std::forward(args)... ); \ + } + + + #define CLIO_REFLECT_FOREACH_MEMBER_HELPER( QUERY_CLASS, MEMBERS ) \ + BOOST_PP_SEQ_FOR_EACH_I( CLIO_REFLECT_FOREACH_INTERNAL, QUERY_CLASS, MEMBERS ) + + #define CLIO_REFLECT_MEMBER_BY_STR_HELPER( QUERY_CLASS, MEMBERS ) \ + BOOST_PP_SEQ_FOR_EACH( CLIO_REFLECT_MEMBER_BY_STR_INTERNAL, QUERY_CLASS, MEMBERS ) + + + #define CLIO_REFLECT_MEMBER_BY_IDX_I_INTERNAL( r, OP, I, member) \ + case I+1: (void)lambda( &OP::member ); return true; + + #define CLIO_REFLECT_MEMBER_BY_IDX_HELPER( QUERY_CLASS, MEMBER_IDXS ) \ + BOOST_PP_SEQ_FOR_EACH_I( CLIO_REFLECT_MEMBER_BY_IDX_I_INTERNAL, QUERY_CLASS, MEMBER_IDXS ) + + + #define CLIO_REFLECT_MEMBER_BY_NAME_I_INTERNAL( r, OP, I, member) \ + case clio::hash_name(BOOST_PP_STRINGIZE(member)): (void)lambda( &OP::member ); return true; + + #define CLIO_REFLECT_MEMBER_BY_NAME_HELPER( QUERY_CLASS, MEMBER_NAMES ) \ + BOOST_PP_SEQ_FOR_EACH_I( CLIO_REFLECT_MEMBER_BY_NAME_I_INTERNAL, QUERY_CLASS, MEMBER_NAMES ) + + #define CLIO_REFLECT_MEMBER_TYPE_BY_IDX_INTERNAL( r, OP, I, member ) \ + BOOST_PP_COMMA_IF( I ) std::decay_t< decltype(clio::result_of_member(&OP::member))> + + + #define CLIO_REFLECT_MEMBER_TYPE_IDX_HELPER( QUERY_CLASS, MEMBER_IDXS ) \ + BOOST_PP_SEQ_FOR_EACH_I( CLIO_REFLECT_MEMBER_TYPE_BY_IDX_INTERNAL, QUERY_CLASS, MEMBER_IDXS ) + + + #define CLIO_REFLECT_FOREACH_MEMBER_PB_HELPER( QUERY_CLASS, MEMBERS ) \ + BOOST_PP_SEQ_FOR_EACH( CLIO_REFLECT_FOREACH_PB_INTERNAL, QUERY_CLASS, MEMBERS ) + + #define CLIO_REFLECT_MEMBER_INDEX_HELPER( QUERY_CLASS, MEMBER_IDXS ) \ + BOOST_PP_SEQ_FOR_EACH_I( CLIO_REFLECT_MEMBER_PB_INTERNAL, QUERY_CLASS, MEMBER_IDXS ) + + #define CLIO_REFLECT_MEMBER_BY_STR_PB_HELPER( QUERY_CLASS, MEMBER_IDXS ) \ + BOOST_PP_SEQ_FOR_EACH( CLIO_REFLECT_MEMBER_BY_STR_PB_INTERNAL, QUERY_CLASS, MEMBER_IDXS ) + + #define CLIO_REFLECT_MEMBER_BY_IDX_PB_HELPER( QUERY_CLASS, MEMBER_IDXS ) \ + BOOST_PP_SEQ_FOR_EACH( CLIO_REFLECT_MEMBER_BY_IDX_PB_INTERNAL, QUERY_CLASS, MEMBER_IDXS ) + +#define CLIO_REFLECT_PROXY_MEMBER_BY_IDX_HELPER( QUERY_CLASS, MEMBER_IDXS ) \ + BOOST_PP_SEQ_FOR_EACH_I( CLIO_REFLECT_PROXY_MEMBER_BY_IDX_INTERNAL, QUERY_CLASS, MEMBER_IDXS ) + +#define CLIO_REFLECT_PROXY_MEMBER_BY_PB_HELPER( QUERY_CLASS, MEMBER_IDXS ) \ + BOOST_PP_SEQ_FOR_EACH( CLIO_REFLECT_PROXY_MEMBER_BY_PB_INTERNAL, QUERY_CLASS, MEMBER_IDXS ) + + + + #define CLIO_REFLECT_PARAMS_BY_IDX_PB_INTERNAL( r, OP, i, member) \ + if constexpr ( std::is_member_function_pointer_v ) \ + return OP:: BOOST_PP_CAT( CLIO_REFLECT_FILTER_NAME member ,___PARAM_NAMES); + + #define CLIO_REFLECT_PARAMS_BY_IDX_PB_HELPER( QUERY_CLASS, MEMBER_IDXS ) \ + BOOST_PP_SEQ_FOR_EACH_I( CLIO_REFLECT_PARAMS_BY_IDX_PB_INTERNAL, QUERY_CLASS, MEMBER_IDXS ) + + + /* + template \ + struct proxy { \ + template \ + proxy( Args&&... args ):proxy___( std::forward(args)... ){}\ + ProxyObject* operator->(){ return &proxy___; } \ + const ProxyObject* operator->()const{ return &proxy___; } \ + ProxyObject& operator*(){ return proxy___; } \ + const ProxyObject& operator*()const{ return proxy___; } \ + CLIO_REFLECT_PROXY_MEMBER_BY_IDX_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + private: \ + ProxyObject proxy___; \ + }; \ + */ + + +#define CLIO_REFLECT(QUERY_CLASS, ...) \ + CLIO_REFLECT_TYPENAME( QUERY_CLASS ) \ + struct reflect_impl_##QUERY_CLASS { \ + static constexpr bool is_defined = true; \ + static constexpr bool is_struct = true; \ + static inline constexpr const char* name() { return BOOST_PP_STRINGIZE(QUERY_CLASS); } \ + typedef std::tuple< \ + CLIO_REFLECT_MEMBER_TYPE_IDX_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) > struct_tuple_type; \ + template constexpr inline static void for_each(L&& lambda) { \ + CLIO_REFLECT_FOREACH_MEMBER_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + } \ + template inline static bool get(const std::string_view& m, L&& lambda) { \ + CLIO_REFLECT_MEMBER_BY_STR_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + return false; \ + } \ + template inline static bool get(int64_t m, L&& lambda) { \ + switch (m) { \ + CLIO_REFLECT_MEMBER_BY_IDX_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + } \ + return false; \ + } \ + template inline static bool get_by_name(uint64_t n, L&& lambda) { \ + switch (n) { \ + CLIO_REFLECT_MEMBER_BY_NAME_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + } \ + return false; \ + } \ + template \ + struct proxy { \ + private: \ + ProxyObject proxy___; \ + public: \ + template \ + proxy( Args&&... args ):proxy___( std::forward(args)... ){}\ + ProxyObject* operator->(){ return &proxy___; } \ + const ProxyObject* operator->()const{ return &proxy___; } \ + ProxyObject& operator*(){ return proxy___; } \ + const ProxyObject& operator*()const{ return proxy___; } \ + CLIO_REFLECT_MPROXY_MEMBER_BY_IDX_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + }; \ + }; \ + reflect_impl_##QUERY_CLASS get_reflect_impl(const QUERY_CLASS&); + + +#define CLIO_REFLECT_PB(QUERY_CLASS, ...) \ + CLIO_REFLECT_TYPENAME( QUERY_CLASS ) \ + struct reflect_impl_##QUERY_CLASS { \ + static constexpr bool is_defined = true; \ + static constexpr bool is_struct = true; \ + static inline constexpr const char* name() { return BOOST_PP_STRINGIZE(QUERY_CLASS); } \ + template inline static void for_each(L&& lambda) { \ + CLIO_REFLECT_FOREACH_MEMBER_PB_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + } \ + template inline static bool get(const std::string_view& m, L&& lambda) { \ + CLIO_REFLECT_MEMBER_BY_STR_PB_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + return false; \ + } \ + template inline static bool get(int64_t m, L&& lambda) { \ + switch (m) { \ + CLIO_REFLECT_MEMBER_BY_IDX_PB_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + } \ + return false; \ + } \ + }; \ + reflect_impl_##QUERY_CLASS get_reflect_impl(const QUERY_CLASS&); \ + + +#define CLIO_REFLECT_TEMPLATE_OBJECT(QUERY_CLASS, TPARAM, ...) \ + template struct reflect_impl_##QUERY_CLASS { \ + static constexpr bool is_defined = true; \ + static constexpr bool is_struct = true; \ + static inline const char* name() { return BOOST_PP_STRINGIZE(QUERY_CLASS) "<" BOOST_PP_STRINGIZE(TPARAM) ">"; } \ + template inline static void for_each(L&& lambda) { \ + CLIO_REFLECT_FOREACH_MEMBER_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + } \ + template inline static void get(const std::string_view& m, L&& lambda) { \ + CLIO_REFLECT_MEMBER_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + return false; \ + } \ + template inline static void get(int64_t m, L&& lambda) { \ + switch (m) { \ + CLIO_REFLECT_MEMBER_INDEX_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + } \ + return false; \ + } \ + using tuple_type = std::tuple< \ + CLIO_REFLECT_MEMBER_TYPE_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) > \ + }; \ + template reflect_impl_##QUERY_CLASS get_reflect_impl(const QUERY_CLASS&); \ + constexpr const char* get_type_name( QUERY_CLASS* ) { return reflect_impl_##QUERY_CLASS :: name(); } + + template + struct reflect_undefined { + static constexpr bool is_defined = false; + static constexpr bool is_struct = false; + template + static void get( const std::string_view& m, L&& lambda ); + }; + + + + /* + using std::string; + CLIO_REFLECT_TYPENAME( int32_t ) + CLIO_REFLECT_TYPENAME( int64_t ) + CLIO_REFLECT_TYPENAME( int16_t ) + CLIO_REFLECT_TYPENAME( int8_t ) + CLIO_REFLECT_TYPENAME( uint32_t ) + CLIO_REFLECT_TYPENAME( uint64_t ) + CLIO_REFLECT_TYPENAME( uint16_t ) + CLIO_REFLECT_TYPENAME( uint8_t ) + CLIO_REFLECT_TYPENAME( float ) + CLIO_REFLECT_TYPENAME( double ) + CLIO_REFLECT_TYPENAME( char ) + CLIO_REFLECT_TYPENAME( bool ) + CLIO_REFLECT_TYPENAME( string ) + */ + + + template + reflect_undefined get_reflect_impl(const QueryClass&); + + + template + using reflect = std::decay_t()))>; + + template + struct is_std_vector : std::false_type {}; + + template + struct is_std_vector> : std::true_type { using value_type = T; }; + + template + struct is_std_optional : std::false_type {}; + + template + struct is_std_optional> : std::true_type { using value_type = T; }; + + template + struct is_std_variant : std::false_type {}; + + template + struct is_std_variant> : std::true_type { + static std::string name() { + return get_variant_typename(); + } + template + static std::string get_variant_typename() { + if constexpr ( sizeof...(Rest) > 0 ) + return std::string( get_type_name() ) + "|" + get_variant_typename(); + else + return std::string( get_type_name() ) + "|"; + } + using alts_as_tuple = std::tuple; + }; + + template + struct is_std_tuple : std::false_type {}; + + template + struct is_std_tuple> : std::true_type { + static std::string name() { + return get_tuple_typename(); + } + template + static std::string get_tuple_typename() { + if constexpr ( sizeof...(Rest) > 0 ) + return std::string( get_type_name() ) + "&" + get_tuple_typename(); + else + return std::string( get_type_name() ) + "&"; + } + }; + + + template + struct is_std_map : std::false_type {}; + + template + struct is_std_map> : std::true_type { + static const std::string& name() { + static std::string n = std::string("map<") + get_type_name() +"," + get_type_name() + ">"; + return n; + } + }; + + +} /// namespace clio + +namespace std { + namespace { + CLIO_REFLECT_TYPENAME( string ) + } +} + diff --git a/libraries/clio/include/clio/schema.hpp b/libraries/clio/include/clio/schema.hpp new file mode 100644 index 000000000..e1acc6236 --- /dev/null +++ b/libraries/clio/include/clio/schema.hpp @@ -0,0 +1,288 @@ +#pragma once +#include +#include +#include +#include +#include +#include + +namespace clio { + using std::string; + using std::vector; + + enum scalar_type { + int64_type, + int32_type, + int16_type, + int8_type, + uint64_type, + uint32_type, + uint16_type, + uint8_type, + varuint32_type, + double_type, + float_type, + string_type, + bool_type + }; + + using type_name = std::string; + + struct variant_type { + vector types; + }; + + CLIO_REFLECT( variant_type, types ) + + struct tuple_type { + vector types; + }; + CLIO_REFLECT( tuple_type, types ) + + struct vector_type { + type_name type; + }; + CLIO_REFLECT( vector_type, type ) + + + struct enum_type { + scalar_type value_type; + struct enum_value { + string name; + int64_t value; + }; + vector values; + }; + using enum_value = enum_type::enum_value; + CLIO_REFLECT( enum_value, name, value ) + CLIO_REFLECT( enum_type, values ) + + /* + struct function_type { + struct parameter { + type_name type; + string name; + }; + vector args; + type_name result; + }; + using function_parameter = function_type::parameter; + CLIO_REFLECT( function_parameter, type, name ) + CLIO_REFLECT( function_type, args, result ) + */ + + struct object_type { + + + struct member { + string name; + type_name type; + int32_t number; + + template + void set_params( std::initializer_list names, R (T::*method)(Args...) ) { + if constexpr ( sizeof...(Args) > 0 ) + push_param( names.begin(), names.end() ); + } + template + void set_params( std::initializer_list names, R (T::*method)(Args...)const ) { + if constexpr ( sizeof...(Args) > 0 ) + push_param(names.begin(), names.end()); + } + + struct param { + string name; + type_name type; + }; + std::vector params; + + private: + template + void push_param( std::initializer_list::iterator begin, + std::initializer_list::iterator end ) { + params.push_back( { + .name = begin != end ? *begin : "", + .type = get_type_name>() + } ); + if constexpr ( sizeof...(Args) > 0 ) + push_param( begin != end ? ++begin : end, end ); + } + }; + + const member* get_member_by_name( const std::string_view& n )const { + for( const auto& m : members ) + if( m.name == n ) return &m; + return nullptr; + + } + + const member* get_member_by_number( uint32_t n )const { + for( const auto& m : members ) + if( m.number == n ) return &m; + return nullptr; + } + + vector members; + }; + + using object_member = object_type::member; + using object_method_param = object_type::member::param; + CLIO_REFLECT( object_method_param, name, type ) + CLIO_REFLECT( object_member, name, type, number, params ) + CLIO_REFLECT( object_type, members ) + + struct typedef_type { + type_name type; + }; + CLIO_REFLECT( typedef_type, type ) + + using schema_type = std::variant< + object_type, + variant_type, + tuple_type, + vector_type, + enum_type, + typedef_type>; + CLIO_REFLECT_TYPENAME( schema_type ) + + + + struct schema { + std::map types; + + std::optional get_type( const std::string& name )const { + auto itr = types.find(name); + if( itr == types.end() ) return {}; + return itr->second; + } + std::optional get_object( const string& name )const { + auto itr = types.find(name); + if( itr == types.end() ) return {}; + auto objptr = std::get_if(&itr->second); + if( objptr ) + return *objptr; + return {}; + } + std::optional get_vector( const string& name )const { + auto itr = types.find(name); + if( itr == types.end() ) return {}; + auto objptr = std::get_if(&itr->second); + if( objptr ) + return *objptr; + return {}; + } + template + bool visit_type( const std::string& type, Visitor&& v ) { + auto itr = types.find( type ); + if( itr == types.end() ) return false; + std::visit( std::forward(v), itr->second ); + return true; + } + + template + bool add_type( schema_type t, L&& on_generate ) { + auto tn = get_type_name(); + if( types.find( tn ) == types.end() ) { + on_generate( static_cast(nullptr) ); + types[tn] = t; + return true; + } + return false; + } + + template + void generate_map( const std::map*, L&& on_generate ) { + generate( on_generate ); + } + + template + void generate_variant( const std::tuple* v, L&& on_generate ) { + if( add_type>( tuple_type(), on_generate ) ) { + tuple_type vt; + generate_helper(vt, on_generate ); + types[get_type_name>()] = vt; + } + } + + template + void generate_variant( const std::variant* v, L&& on_generate ) { + auto tn = get_type_name>(); + if( add_type>( variant_type(), on_generate ) ) + { + variant_type vt; + generate_helper(vt, on_generate); + types[tn] = vt; + } + } + + template + void generate_helper( OT& vt, L&& on_generate ) { + generate( on_generate ); + vt.types.push_back( get_type_name() ); + if constexpr ( sizeof...(Args) > 0 ) { + generate_helper( vt, on_generate ); + } + } + + template + void generate() { + generate( [](auto){} ); + } + + template + void generate( L on_generate ) { + if constexpr ( std::is_same_v> ){} + else if constexpr ( is_std_map::value ) { + generate_map( (const T*)nullptr, on_generate ); + } + else if constexpr ( is_std_tuple::value ) { + generate_variant( (const T*)nullptr, on_generate ); + } + else if constexpr ( is_std_variant::value ) { + generate_variant( (const T*)nullptr, on_generate ); + } + else if constexpr ( is_std_optional::value ) { + generate::value_type>( on_generate ); + } + else if constexpr ( is_std_vector::value ) { + using value_type = typename is_std_vector::value_type; + if( add_type( vector_type{ get_type_name() }, on_generate ) ) { + generate( on_generate ); + } + } + else if constexpr ( reflect::is_struct ) { + auto n = get_type_name(); + if( add_type( object_type(), on_generate ) ) { + object_type ot; + reflect::for_each( [&]( const meta& r, auto m ) { + if constexpr( not std::is_member_function_pointer_v ) { + using member_type = std::decay_t*>(nullptr)->*m )>; + generate( on_generate ); + + auto tn = get_type_name(); + ot.members.push_back( { .name = r.name, .type = tn, .number = r.number } ); + } else { + using member_type = decltype(result_of( m )); + generate( on_generate ); + auto tn = get_type_name(); + ot.members.push_back( { .name = r.name, .type = tn, .number = r.number } ); + ot.members.back().set_params( r.param_names, m ); + } + }); + types[n] = ot; + } + } + } /// generate + }; + + CLIO_REFLECT( schema, types ) + + +} /// clio + +namespace std { + namespace { + using clio::schema_type; + CLIO_REFLECT_TYPENAME( schema_type ) + } +} diff --git a/libraries/clio/include/clio/stream.hpp b/libraries/clio/include/clio/stream.hpp new file mode 100644 index 000000000..726591d10 --- /dev/null +++ b/libraries/clio/include/clio/stream.hpp @@ -0,0 +1,323 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace clio { +enum class stream_error { + no_error, + overrun, + underrun, + float_error, + varuint_too_big, + invalid_varuint_encoding, + bad_variant_index, + invalid_asset_format, + array_size_mismatch, + invalid_name_char, + invalid_name_char13, + name_too_long, + json_writer_error, // !!! +}; // stream_error +} // namespace clio + +namespace std { +template <> +struct is_error_code_enum : true_type {}; +} // namespace std + +namespace clio { + +class stream_error_category_type : public std::error_category { + public: + const char* name() const noexcept override final { return "ConversionError"; } + + std::string message(int c) const override final { + switch (static_cast(c)) { + // clang-format off + case stream_error::no_error: return "No error"; + case stream_error::overrun: return "Stream overrun"; + case stream_error::underrun: return "Stream underrun"; + case stream_error::float_error: return "Float error"; + case stream_error::varuint_too_big: return "Varuint too big"; + case stream_error::invalid_varuint_encoding: return "Invalid varuint encoding"; + case stream_error::bad_variant_index: return "Bad variant index"; + case stream_error::invalid_asset_format: return "Invalid asset format"; + case stream_error::array_size_mismatch: return "T[] size and unpacked size don't match"; + case stream_error::invalid_name_char: return "character is not in allowed character set for names"; + case stream_error::invalid_name_char13: return "thirteenth character in name cannot be a letter that comes after j"; + case stream_error::name_too_long: return "string is too long to be a valid name"; + case stream_error::json_writer_error: return "Error writing json"; + // clang-format on + + default: return "unknown"; + } + } +}; // stream_error_category_type + +inline const stream_error_category_type& stream_error_category() { + static stream_error_category_type c; + return c; +} + +inline std::error_code make_error_code(stream_error e) { return { static_cast(e), stream_error_category() }; } + + +template +constexpr bool has_bitwise_serialization() { + if constexpr (std::is_arithmetic_v) { + return true; + } else if constexpr (std::is_enum_v) { + static_assert(!std::is_convertible_v>, "Serializing unscoped enum"); + return true; + } else { + return false; + } +} + +template +struct small_buffer { + char data[max_size]; + char* pos{ data }; + + void reverse() { std::reverse(data, pos); } +}; + +struct string_stream { + std::string& data; + string_stream(std::string& data) : data(data) {} + + void write(char ch) { + data += ch; + } + + void write(const void* src, size_t size) { + auto s = reinterpret_cast(src); + data.insert(data.end(), s, s + size); + } + + template + void write(const char (&src)[size]) { + write(src, size-1); + } + + template + void write_raw(const T& v) { + write(&v, sizeof(v)); + } + + void write(const std::string& v) { + write(v.c_str(), v.size()); + } +}; + + +struct vector_stream { + std::vector& data; + vector_stream(std::vector& data) : data(data) {} + + void write(char ch) { + data.push_back(ch); + } + + void write(const void* src, size_t size) { + auto s = reinterpret_cast(src); + data.insert(data.end(), s, s + size); + } + + template + void write(const char (&src)[size]) { + write(src, size-1); + } + + template + void write_raw(const T& v) { + write(&v, sizeof(v)); + } + + void write(const std::string& v) { + write(v.c_str(), v.size()); + } +}; + +struct fixed_buf_stream { + char* begin; + char* pos; + char* end; + + fixed_buf_stream(char* pos, size_t size) : begin(pos), pos{ pos }, end{ pos + size } {} + + void write(char ch) { + if (pos >= end) + throw_error(stream_error::overrun); + *pos++ = ch; + } + + void write(const void* src, size_t size) { + if (pos + size > end) + throw_error(stream_error::overrun); + memcpy(pos, src, size); + pos += size; + } + + template + void write(const char (&src)[size]) { + write(src, size-1); + } + + template + void write_raw(const T& v) { + write(&v, sizeof(v)); + } + + void write(const std::string& v) { + write(v.c_str(), v.size()); + } + + void skip( int32_t s ) { + if( (pos + s > end) or (pos + s < begin) ) + throw_error( stream_error::overrun ); + pos += s; + } + auto get_pos()const { return pos; } + + size_t remaining() { return end - pos; } +}; + +struct size_stream { + size_t size = 0; + + size_t get_pos()const { return size; } + void write(char ch) { + ++size; + } + + void write(const void* src, size_t size) { + this->size += size; + } + + template + void write(const char (&src)[size]) { + this->size += size-1; + } + + template + void write_raw(const T& v) { + size += sizeof(v); + } + + void write(const std::string& v) { + write(v.c_str(), v.size()); + } + + void skip( int32_t s ) { + size += s; + } +}; + +template +void increase_indent(S&) { } + +template +void decrease_indent(S&) { } + +template +void write_colon(S& s) { + s.write(':'); +} + +template +void write_newline(S&) { +} + +template +struct pretty_stream : Base { + using Base::Base; + int indent_size = 4; + std::vector current_indent; +}; + +template +void increase_indent(pretty_stream& s) { + s.current_indent.resize(s.current_indent.size() + s.indent_size, ' '); +} + +template +void decrease_indent(pretty_stream& s) { + if (s.current_indent.size() < s.indent_size) + throw_error(stream_error::overrun); + s.current_indent.resize(s.current_indent.size() - s.indent_size); +} + +template +void write_colon(pretty_stream& s) { + s.write(": ", 2); +} + +template +void write_newline(pretty_stream& s) { + s.write('\n'); + s.write(s.current_indent.data(), s.current_indent.size()); +} + +struct input_stream { + const char* pos; + const char* end; + + input_stream() : pos{ nullptr }, end{ nullptr } {} + input_stream(const char* pos, size_t size) : pos{ pos }, end{ pos + size } { + if ( size < 0 ) + throw_error( stream_error::overrun ); + } + input_stream(const char* pos, const char* end) : pos{ pos }, end{ end } { + if ( end < pos ) + throw_error( stream_error::overrun ); + } + input_stream(const std::vector& v) : pos{ v.data() }, end{ v.data() + v.size() } {} + input_stream(std::string_view v) : pos{ v.data() }, end{ v.data() + v.size() } {} + input_stream(const input_stream&) = default; + + input_stream& operator=(const input_stream&) = default; + + size_t remaining() { return end - pos; } + + void check_available(size_t size) { + if (size > size_t(end - pos)) + throw_error(stream_error::overrun); + } + + auto get_pos()const { return pos; } + + void read(void* dest, size_t size) { + if (size > size_t(end - pos)) + throw_error( stream_error::overrun ); + memcpy(dest, pos, size); + pos += size; + } + + template + void read_raw(T& dest) { + read(&dest, sizeof(dest)); + } + + void skip(size_t size) { + if (size > size_t(end - pos)) + throw_error(stream_error::overrun); + pos += size; + } + + void read_reuse_storage(const char*& result, size_t size) { + if (size > size_t(end - pos)) + throw_error( stream_error::overrun ); + result = pos; + pos += size; + } +}; + +} // namespace clio diff --git a/libraries/clio/include/clio/to_bin.hpp b/libraries/clio/include/clio/to_bin.hpp new file mode 100644 index 000000000..c28ee6f38 --- /dev/null +++ b/libraries/clio/include/clio/to_bin.hpp @@ -0,0 +1,172 @@ +#pragma once +#include +#include +#include +#include + +namespace clio { + +template +std::vector to_bin(const T& t); + +template +void to_bin(std::string_view sv, S& stream); + +template +void to_bin(const std::string& s, S& stream); + +template +void to_bin(const std::vector& obj, S& stream); + +template +void to_bin(const std::optional& obj, S& stream); + +template +void to_bin(const std::variant& obj, S& stream); + +template +void to_bin(const std::tuple& obj, S& stream); + +template +void to_bin(const T& obj, S& stream); + +template +void varuint32_to_bin(uint64_t val, S& stream) { + if (val >> 32) { + /// TODO throw error return stream_error::varuint_too_big; + return; + } + do { + uint8_t b = val & 0x7f; + val >>= 7; + b |= ((val > 0) << 7); + stream.write(b); + } while (val); +} + +// !!! temp +inline void push_varuint32(std::vector& bin, uint32_t v) { + vector_stream st{ bin }; + varuint32_to_bin(v, st); +} + +template +void to_bin(std::string_view sv, S& stream) { + varuint32_to_bin(sv.size(), stream); + stream.write(sv.data(), sv.size()); +} + +template +void to_bin(const std::string& s, S& stream) { + to_bin(std::string_view{ s }, stream); +} + +template +void to_bin_range(const T& obj, S& stream) { + varuint32_to_bin(obj.size(), stream); + for (auto& x : obj) { to_bin(x, stream); } +} + +template +void to_bin(const T (&obj)[N], S& stream) { + varuint32_to_bin(N, stream); + if constexpr (has_bitwise_serialization()) { + stream.write(reinterpret_cast(&obj), N * sizeof(T)); + } else { + for (auto& x : obj) { to_bin(x, stream); } + } +} + +template +void to_bin(const std::vector& obj, S& stream) { + varuint32_to_bin(obj.size(), stream); + if constexpr (has_bitwise_serialization()) { + stream.write(reinterpret_cast(obj.data()), obj.size() * sizeof(T)); + } else { + for (auto& x : obj) { to_bin(x, stream); } + } +} + +template +void to_bin(const std::variant& obj, S& stream) { + varuint32_to_bin(obj.index(), stream); + std::visit([&](auto& x) { to_bin(x, stream); }, obj); +} + + +template +void to_bin(const input_stream& obj, S& stream) { + varuint32_to_bin(obj.end - obj.pos, stream); + stream.write(obj.pos, obj.end - obj.pos); +} + +template +void to_bin(const std::pair& obj, S& stream) { + to_bin(obj.first, stream); + return to_bin(obj.second, stream); +} + +template +void to_bin(const std::optional& obj, S& stream) { + to_bin(obj.has_value(), stream); + if (obj) to_bin(*obj, stream); +} + + +template +void to_bin_tuple(const T& obj, S& stream) { + if constexpr (i < std::tuple_size_v) { + to_bin(std::get(obj), stream); + to_bin_tuple(obj, stream); + } +} + +template +void to_bin(const std::tuple& obj, S& stream) { + return to_bin_tuple<0>(obj, stream); +} + +template +void to_bin(const std::array& obj, S& stream) { + for (const T& elem : obj) { to_bin(elem, stream); } +} + +template +void to_bin(const T& obj, S& stream) { + if constexpr (has_bitwise_serialization()) { + stream.write(reinterpret_cast(&obj), sizeof(obj)); + } else { + + reflect::for_each( [&]( const clio::meta&, auto m ) { + if constexpr( not std::is_member_function_pointer_v ) { + to_bin( obj.*m, stream ); + } + }); + } +} + +template +void convert_to_bin(const T& t, std::vector& bin) { + size_stream ss; + to_bin(t, ss); + + auto orig_size = bin.size(); + bin.resize(orig_size + ss.size); + fixed_buf_stream fbs(bin.data() + orig_size, ss.size); + to_bin(t, fbs); + + /** TODO maybe throw + if (fbs.pos != fbs.end) + return stream_error::underrun; + */ +} + +template +std::vector to_bin(const T& t) { + std::vector result; + convert_to_bin(t, result); + return result; +} + + +} // clio diff --git a/libraries/clio/include/clio/to_bin/deque.hpp b/libraries/clio/include/clio/to_bin/deque.hpp new file mode 100644 index 000000000..e20952671 --- /dev/null +++ b/libraries/clio/include/clio/to_bin/deque.hpp @@ -0,0 +1,12 @@ +#pragma once +#include +#include + +namespace clio { + + template + result to_bin(const std::deque& obj, S& stream) { + return to_bin_range(obj, stream); + } + +} // clio diff --git a/libraries/clio/include/clio/to_bin/list.hpp b/libraries/clio/include/clio/to_bin/list.hpp new file mode 100644 index 000000000..367a124ba --- /dev/null +++ b/libraries/clio/include/clio/to_bin/list.hpp @@ -0,0 +1,12 @@ +#pragma once +#include +#include + +namespace clio { + + template + result to_bin(const std::list& obj, S& stream) { + return to_bin_range(obj, stream); + } + +} // clio diff --git a/libraries/clio/include/clio/to_bin/map.hpp b/libraries/clio/include/clio/to_bin/map.hpp new file mode 100644 index 000000000..ef436d382 --- /dev/null +++ b/libraries/clio/include/clio/to_bin/map.hpp @@ -0,0 +1,12 @@ +#pragma once +#include +#include + +namespace clio { + + template + result to_bin(const std::map& obj, S& stream) { + return to_bin_range(obj, stream); + } + +} // clio diff --git a/libraries/clio/include/clio/to_bin/set.hpp b/libraries/clio/include/clio/to_bin/set.hpp new file mode 100644 index 000000000..4afbe76ee --- /dev/null +++ b/libraries/clio/include/clio/to_bin/set.hpp @@ -0,0 +1,12 @@ +#pragma once +#include +#include + +namespace clio { + + template + result to_bin(const std::set& obj, S& stream) { + return to_bin_range(obj, stream); + } + +} // clio diff --git a/libraries/clio/include/clio/to_bin/varint.hpp b/libraries/clio/include/clio/to_bin/varint.hpp new file mode 100644 index 000000000..733e14f16 --- /dev/null +++ b/libraries/clio/include/clio/to_bin/varint.hpp @@ -0,0 +1,17 @@ +#pragma once +#include +#include + +namespace clio { + +template +void to_bin(const varuint32& obj, S& stream) { + varuint32_to_bin(obj.value, stream); +} + +template +void to_bin(const varint32& obj, S& stream) { + varuint32_to_bin((uint32_t(obj.value) << 1) ^ uint32_t(obj.value >> 31), stream); +} + +} /// namespace clio diff --git a/libraries/clio/include/clio/to_json.hpp b/libraries/clio/include/clio/to_json.hpp new file mode 100644 index 000000000..0d172d174 --- /dev/null +++ b/libraries/clio/include/clio/to_json.hpp @@ -0,0 +1,283 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace clio { + +inline constexpr char hex_digits[] = "0123456789ABCDEF"; + +// Adaptors for rapidjson +struct stream_adaptor { + stream_adaptor(const char* src, int sz) { + int chars = std::min(sz, 4); + memcpy(buf, src, chars); + memset(buf + chars, 0, 4 - chars); + } + void Put(char ch) {} + char Take() { return buf[idx++]; } + char buf[4]; + int idx = 0; +}; + + +// Replaces any invalid utf-8 bytes with ? +template +void to_json(std::string_view sv, S& stream) { + stream.write('"'); + auto begin = sv.begin(); + auto end = sv.end(); + while (begin != end) { + auto pos = begin; + while (pos != end && *pos != '"' && *pos != '\\' && (unsigned char)(*pos) >= 32 && *pos != 127) ++pos; + while (begin != pos) { + stream_adaptor s2(begin, static_cast(pos - begin)); + if (rapidjson::UTF8<>::Validate(s2, s2)) { + stream.write(begin, s2.idx); + begin += s2.idx; + } else { + ++begin; + stream.write('?'); + } + } + if (begin != end) { + if (*begin == '"') { + stream.write("\\\"", 2); + } else if (*begin == '\\') { + stream.write("\\\\", 2); + } else { + stream.write("\\u00", 4); + stream.write(hex_digits[(unsigned char)(*begin) >> 4]); + stream.write(hex_digits[(unsigned char)(*begin) & 15]); + } + ++begin; + } + } + stream.write('"'); +} + +template +void to_json(const std::string& s, S& stream) { + return to_json(std::string_view{ s }, stream); +} + +template +void to_json(const char* s, S& stream) { + return to_json(std::string_view{ s }, stream); +} + +template +void to_json(bool value, S& stream) { + if (value) + return stream.write("true", 4); + else + return stream.write("false", 5); +} + +template +void int_to_json(T value, S& stream) { + auto uvalue = std::make_unsigned_t(value); + small_buffer::digits10 + 4> b; + bool neg = value < 0; + if (neg) + uvalue = -uvalue; + if (sizeof(T) > 4) + *b.pos++ = '"'; + do { + *b.pos++ = '0' + (uvalue % 10); + uvalue /= 10; + } while (uvalue); + if (neg) + *b.pos++ = '-'; + if (sizeof(T) > 4) + *b.pos++ = '"'; + b.reverse(); + return stream.write(b.data, b.pos - b.data); +} + +template +void fp_to_json(double value, S& stream) { + // fpconv is not quite consistent with javascript for nans and infinities + if (value == std::numeric_limits::infinity()) { + return stream.write("\"Infinity\"", 10); + } else if (value == -std::numeric_limits::infinity()) { + return stream.write("\"-Infinity\"", 11); + } else if (std::isnan(value)) { + return stream.write("\"NaN\"", 5); + } + small_buffer<24> b; // fpconv_dtoa generates at most 24 characters + int n = fpconv_dtoa(value, b.pos); + if (n <= 0) + throw_error(stream_error::float_error); + b.pos += n; + stream.write(b.data, b.pos - b.data); +} + +// clang-format off +template void to_json(uint8_t value, S& stream) { return int_to_json(value, stream); } +template void to_json(uint16_t value, S& stream) { return int_to_json(value, stream); } +template void to_json(uint32_t value, S& stream) { return int_to_json(value, stream); } +template void to_json(uint64_t value, S& stream) { return int_to_json(value, stream); } +template void to_json(unsigned __int128 value, S& stream) { return int_to_json(value, stream); } +template void to_json(int8_t value, S& stream) { return int_to_json(value, stream); } +template void to_json(int16_t value, S& stream) { return int_to_json(value, stream); } +template void to_json(int32_t value, S& stream) { return int_to_json(value, stream); } +template void to_json(int64_t value, S& stream) { return int_to_json(value, stream); } +template void to_json(__int128 value, S& stream) { return int_to_json(value, stream); } +template void to_json(double value, S& stream) { return fp_to_json(value, stream); } +template void to_json(float value, S& stream) { return fp_to_json(value, stream); } +// clang-format on + +template +void to_json(const std::vector& obj, S& stream) { + stream.write('['); + bool first = true; + for (auto& v : obj) { + if (first) { + increase_indent(stream); + } else { + stream.write(','); + } + write_newline(stream); + first = false; + to_json(v, stream); + } + if (!first) { + decrease_indent(stream); + write_newline(stream); + } + stream.write(']'); +} + +template +void to_json(const std::optional& obj, S& stream) { + if (obj) { + return to_json(*obj, stream); + } else { + return stream.write("null", 4); + } +} + + + +template +void to_json(const std::variant& obj, S& stream) { + stream.write('['); + std::visit( + [&](const auto& t) { to_json( get_type_name>(), stream); }, obj); + stream.write(','); + std::visit([&](auto& x) { return to_json(x, stream); }, obj); + stream.write(']'); +} + +template +void to_json(const std::tuple& obj, S& stream) { + stream.write('['); + if( sizeof...(T) == 0 ) + return stream.write(']'); + increase_indent(stream); + + tuple_for_each( obj, [&]( int idx, auto& item ) { + write_newline(stream); + if( idx ) { stream.write(',');} + to_json( item, stream ); + }); + decrease_indent(stream); + write_newline(stream); + return stream.write(']'); +} + + +template +void to_json(const T& t, S& stream) { + if constexpr ( not reflect::is_defined ) { + stream.write('"'); + std::string str(t); + stream.write( str.data(), str.size() ); + stream.write('"'); + } else { + bool first = true; + stream.write('{'); + reflect::for_each([&]( const clio::meta& ref, auto&& member) { + if constexpr ( not std::is_member_function_pointer_v> ) { + auto addfield = [&]() { + if (first) { + increase_indent(stream); + first = false; + } else { + stream.write(','); + } + write_newline(stream); + to_json(ref.name, stream); + write_colon(stream); + to_json( t.*member, stream); + }; + + using member_type = std::decay_t; + if constexpr ( not is_std_optional::value ) { + addfield(); + } else { + if( !!(t.*member) ) + addfield(); + } + } + }); + if (!first) { + decrease_indent(stream); + write_newline(stream); + } + stream.write('}'); + } +} + +template +void to_json_hex(const char* data, size_t size, S& stream) { + stream.write('"'); + for (size_t i = 0; i < size; ++i) { + unsigned char byte = data[i]; + stream.write(hex_digits[byte >> 4]); + stream.write(hex_digits[byte & 15]); + } + stream.write('"'); +} + +template +std::string convert_to_json(const T& t) { + size_stream ss; + to_json(t, ss); + + std::string result(ss.size, 0); + fixed_buf_stream fbs(result.data(), result.size()); + to_json(t, fbs); + + if (fbs.pos == fbs.end) + return result; + else + throw_error(stream_error::underrun); +} + +template +std::string to_json(const T& t) { + return convert_to_json(t); +} + +template +std::string format_json(const T& t) { + pretty_stream ss; + to_json(t, ss); + std::string result(ss.size, 0); + pretty_stream fbs(result.data(), result.size()); + to_json(t, fbs); + if (fbs.pos != fbs.end) + throw_error(stream_error::underrun); + return result; +} + +} // namespace eosio diff --git a/libraries/clio/include/clio/to_json/map.hpp b/libraries/clio/include/clio/to_json/map.hpp new file mode 100644 index 000000000..ebbb48499 --- /dev/null +++ b/libraries/clio/include/clio/to_json/map.hpp @@ -0,0 +1,28 @@ +#pragma once +#include +#include + +namespace clio { +template +void to_json( const std::map& m, S& stream) { + stream.write('{'); + if( m.size() == 0 ) { + return stream.write('}'); + } + increase_indent(stream); + bool not_first = false; + for( const auto& p : m ) { + if( not_first ) { + stream.write(','); + } + write_newline(stream); + to_json( p.first, stream ); + write_colon(stream); + to_json( p.second, stream ); + not_first = true; + } + decrease_indent(stream); + write_newline(stream); + return stream.write('}'); +} +} /// namespace clio diff --git a/libraries/clio/include/clio/to_json/varint.hpp b/libraries/clio/include/clio/to_json/varint.hpp new file mode 100644 index 000000000..f679897e5 --- /dev/null +++ b/libraries/clio/include/clio/to_json/varint.hpp @@ -0,0 +1,17 @@ +#pragma once +#include +#include + +namespace clio { + +template +void to_json(const varuint32& obj, S& stream) { + to_json(obj.value, stream); +} + +template +void to_json(const varint32& obj, S& stream) { + to_json(obj.value, stream); +} + +} // namespace clio diff --git a/libraries/clio/include/clio/to_key.hpp b/libraries/clio/include/clio/to_key.hpp new file mode 100644 index 000000000..e91c09b8a --- /dev/null +++ b/libraries/clio/include/clio/to_key.hpp @@ -0,0 +1,303 @@ +#pragma once + +//#include +//#include +//#include +//#include +#include +#include +#include +#include +#include +#include + +#include + +namespace clio { + +template +void to_key(const std::tuple& obj, S& stream); + +// to_key defines a conversion from a type to a sequence of bytes whose lexicograpical +// ordering is the same as the ordering of the original type. +// +// For any two objects of type T, a and b: +// +// - key(a) < key(b) iff a < b +// - key(a) is not a prefix of key(b) +// +// Overloads of to_key for user-defined types can be found by Koenig lookup. +// +// Abieos provides specializations of to_key for the following types +// - std::string and std::string_view +// - std::vector, std::list, std::deque +// - std::tuple +// - std::array +// - std::optional +// - std::variant +// - Arithmetic types +// - Scoped enumeration types +// - Reflected structs +// - All smart-contract related types defined by abieos +template +void to_key(const T& obj, S& stream); + +template +void to_key_tuple(const T& obj, S& stream) { + if constexpr (i < std::tuple_size_v) { + to_key(std::get(obj), stream); + to_key_tuple(obj, stream); + } +} + +template +void to_key(const std::tuple& obj, S& stream) { + return to_key_tuple<0>(obj, stream); +} + +template +void to_key(const std::array& obj, S& stream) { + for (const T& elem : obj) { to_key(elem, stream); } +} + +template +void to_key_optional(const bool* obj, S& stream) { + if (obj == nullptr) + return stream.write('\0'); + else if (!*obj) + return stream.write('\1'); + else + return stream.write('\2'); +} + +template +void to_key_optional(const T* obj, S& stream) { + if constexpr (has_bitwise_serialization() && sizeof(T) == 1) { + if (obj == nullptr) + return stream.write("\0", 2); + else { + char buf[1]; + fixed_buf_stream tmp_stream(buf, 1); + to_key(*obj, tmp_stream); + stream.write(buf[0]); + if (buf[0] == '\0') + stream.write('\1'); + } + } else { + if (obj) { + stream.write('\1'); + return to_key(*obj, stream); + } else { + return stream.write('\0'); + } + } +} + +template +void to_key(const std::pair& obj, S& stream) { + to_key(obj.first, stream); + return to_key(obj.second, stream); +} + +template +void to_key_range(const T& obj, S& stream) { + for (const auto& elem : obj) { to_key_optional(&elem, stream); } + return to_key_optional((decltype(&*std::begin(obj))) nullptr, stream); +} + +template +void to_key(const std::vector& obj, S& stream) { + for (const T& elem : obj) { to_key_optional(&elem, stream); } + return to_key_optional((const T*)nullptr, stream); +} + +/* +template +void to_key(const std::list& obj, S& stream) { + return to_key_range(obj, stream); +} + +template +void to_key(const std::deque& obj, S& stream) { + return to_key_range(obj, stream); +} + +template +void to_key(const std::set& obj, S& stream) { + return to_key_range(obj, stream); +} + +template +void to_key(const std::map& obj, S& stream) { + return to_key_range(obj, stream); +} +*/ + +template +void to_key(const std::optional& obj, S& stream) { + return to_key_optional(obj ? &*obj : nullptr, stream); +} + +// The first byte holds: +// 0-4 1's (number of additional bytes) 0 (terminator) bits +// +// The number is represented as big-endian using the low order +// bits of the first byte and all of the remaining bytes. +// +// Notes: +// - values must be encoded using the minimum number of bytes, +// as non-canonical representations will break the sort order. +template +void to_key_varuint32(std::uint32_t obj, S& stream) { + int num_bytes; + if (obj < 0x80u) { + num_bytes = 1; + } else if (obj < 0x4000u) { + num_bytes = 2; + } else if (obj < 0x200000u) { + num_bytes = 3; + } else if (obj < 0x10000000u) { + num_bytes = 4; + } else { + num_bytes = 5; + } + + stream.write( + static_cast(~(0xFFu >> (num_bytes - 1)) | (num_bytes == 5 ? 0 : (obj >> ((num_bytes - 1) * 8))))); + for (int i = num_bytes - 2; i >= 0; --i) { stream.write(static_cast((obj >> i * 8) & 0xFFu)); } +} + +// for non-negative values +// The first byte holds: +// 1 (signbit) 0-4 1's (number of additional bytes) 0 (terminator) bits +// The value is represented as big endian +// for negative values +// The first byte holds: +// 0 (signbit) 0-4 0's (number of additional bytes) 1 (terminator) bits +// The value is adjusted to be positive based on the range that can +// be represented with this number of bytes and then encoded as big endian. +// +// Notes: +// - negative values must sort before positive values +// - For negative value, numbers that need more bytes are smaller, hence +// the encoding of the width must be opposite the encoding used for +// non-negative values. +// - A 5-byte varint can represent values in $[-2^34, 2^34)$. In this case, +// the argument will be sign-extended. +template +void to_key_varint32(std::int32_t obj, S& stream) { + static_assert(std::is_same_v, "to_key for varint32 has been temporarily disabled"); + int num_bytes; + bool sign = (obj < 0); + if (obj < 0x40 && obj >= -0x40) { + num_bytes = 1; + } else if (obj < 0x2000 && obj >= -0x2000) { + num_bytes = 2; + } else if (obj < 0x100000 && obj >= -0x100000) { + num_bytes = 3; + } else if (obj < 0x08000000 && obj >= -0x08000000) { + num_bytes = 4; + } else { + num_bytes = 5; + } + + unsigned char width_field; + if (sign) { + width_field = 0x80u >> num_bytes; + } else { + width_field = 0x80u | ~(0xFFu >> num_bytes); + } + auto uobj = static_cast(obj); + unsigned char value_mask = (0xFFu >> (num_bytes + 1)); + unsigned char high_byte = (num_bytes == 5 ? (sign ? 0xFF : 0) : (uobj >> ((num_bytes - 1) * 8))); + stream.write(width_field | (high_byte & value_mask)); + for (int i = num_bytes - 2; i >= 0; --i) { stream.write(static_cast((uobj >> i * 8) & 0xFFu)); } +} + +template +void to_key(const std::variant& obj, S& stream) { + to_key_varuint32(static_cast(obj.index()), stream); + std::visit([&](const auto& item) { return to_key(item, stream); }, obj); +} + +template +void to_key(std::string_view obj, S& stream) { + for (char ch : obj) { + stream.write(ch); + if (ch == '\0') { + stream.write('\1'); + } + } + return stream.write("\0", 2); +} + +template +void to_key(const std::string& obj, S& stream) { + return to_key(std::string_view(obj), stream); +} + +template +void to_key(bool obj, S& stream) { + return stream.write(static_cast(obj ? 1 : 0)); +} + +template +UInt float_to_key(T value) { + static_assert(sizeof(T) == sizeof(UInt), "Expected unsigned int of the same size"); + UInt result; + std::memcpy(&result, &value, sizeof(T)); + UInt signbit = (static_cast(1) << (std::numeric_limits::digits - 1)); + UInt mask = 0; + if (result == signbit) + result = 0; + if (result & signbit) + mask = ~mask; + return result ^ (mask | signbit); +} + +template +void to_key(const T& obj, S& stream) { + if constexpr (std::is_floating_point_v) { + if constexpr (sizeof(T) == 4) { + return to_key(float_to_key(obj), stream); + } else { + static_assert(sizeof(T) == 8, "Unknown floating point type"); + return to_key(float_to_key(obj), stream); + } + } else if constexpr (std::is_integral_v) { + auto v = static_cast>(obj); + v -= static_cast>(std::numeric_limits::min()); + std::reverse(reinterpret_cast(&v), reinterpret_cast(&v + 1)); + return stream.write_raw(v); + } else if constexpr (std::is_enum_v) { + static_assert(!std::is_convertible_v>, "Serializing unscoped enum"); + return to_key(static_cast>(obj), stream); + } else { + clio::reflect::for_each( [&](const clio::meta&, auto member) { + if constexpr( std::is_member_pointer_v ) { + to_key( obj.*member, stream); + } + }); + } +} + +template +void convert_to_key(const T& t, std::vector& bin) { + size_stream ss; + to_key(t, ss); + auto orig_size = bin.size(); + bin.resize(orig_size + ss.size); + fixed_buf_stream fbs(bin.data() + orig_size, ss.size); + to_key(t, fbs); + if (fbs.pos != fbs.end) + throw_error(stream_error::underrun); +} + +template +std::vector convert_to_key(const T& t) { + std::vector result; + convert_to_key(t, result); + return result; +} + +} // namespace clio diff --git a/libraries/clio/include/clio/to_protobuf.hpp b/libraries/clio/include/clio/to_protobuf.hpp new file mode 100644 index 000000000..fa71080ee --- /dev/null +++ b/libraries/clio/include/clio/to_protobuf.hpp @@ -0,0 +1,152 @@ +#pragma once +#include +#include +#include + +namespace clio { + +template +void to_protobuf_object(const std::tuple& obj, S& stream); + +template +void to_protobuf_object(const std::variant& obj, S& stream); + +template +void to_protobuf_object(const std::vector& obj, S& stream); + +template +void to_protobuf_object(const T& obj, S& stream); + +template +struct wire_type { static constexpr const auto value = 2; }; + +#define WIRE_TYPE(X,T) \ +template<> struct wire_type { static constexpr const auto value = T; }; + + +template struct wire_type< std::variant> { static constexpr const auto value = 2; }; + +WIRE_TYPE( uint16_t, 0 ) +WIRE_TYPE( uint8_t, 0 ) +WIRE_TYPE( int16_t, 0 ) +WIRE_TYPE( int8_t, 0 ) +WIRE_TYPE( char, 0 ) +WIRE_TYPE( bool, 0 ) +WIRE_TYPE( varuint32, 0 ) +WIRE_TYPE( uint64_t, 1 ) +WIRE_TYPE( int64_t, 1 ) +WIRE_TYPE( double, 1 ) +WIRE_TYPE( uint32_t, 5 ) +WIRE_TYPE( int32_t, 5 ) +WIRE_TYPE( float, 5 ) +WIRE_TYPE( std::string, 2 ) +WIRE_TYPE( bytes, 2 ) + +/** + * Assumes key has been writen, then writes the rest... + */ +template +void to_protobuf_member( const T& obj, S& stream ) { + if constexpr( std::is_same_v< T, std::string> ) { + varuint32_to_bin( obj.size(), stream ); + stream.write( obj.data(), obj.size() ); + } else if constexpr( std::is_same_v< T, bytes> ) { + varuint32_to_bin( obj.data.size(), stream ); + stream.write( obj.data.data(), obj.data.size()); + } else if constexpr ( 5 == wire_type::value or 1 == wire_type::value ) { + to_bin( obj, stream ); + } else if constexpr ( 0 == wire_type::value ) { + varuint32_to_bin( obj, stream ); + } else if constexpr( reflect::is_struct || + is_std_vector::value || + is_std_tuple::value || + is_std_variant::value ) + { + size_stream ss; + to_protobuf_object(obj, ss); + varuint32_to_bin( ss.size, stream ); + to_protobuf_object(obj, stream); + } else { + T::to_protobuf_is_not_defined; /// used to generate useful compile error + } +} + +template +void write_protobuf_field( int field, const Member& member, Stream& stream ) { + if constexpr( is_std_vector< std::decay_t >::value ) { + if( member.size() == 0 ) return; + } + uint32_t key = (field << 3) | wire_type::value; + varuint32_to_bin( key, stream ); + to_protobuf_member( member, stream ); +} + +template +void to_protobuf_object(const std::variant& obj, S& stream) { + std::visit( [&](auto m){ + write_protobuf_field( obj.index()+1, m, stream ); + }, obj ); +} + + +/** + * A vector is protobuf object that is either packed in a single field or + * listed as N fields of the same type. + */ +template +void to_protobuf_object(const std::vector& vec, S& stream) { + uint32_t key = (1 << 3) | wire_type>::value; + if constexpr ( std::is_arithmetic_v< T > ) { /// [packed=true] + varuint32_to_bin( key, stream ); + auto size = uint32_t(vec.size() * sizeof( T )); + varuint32_to_bin( size, stream ); + stream.write( vec.data(), vec.size()*sizeof(T) ); + } + else { + for( const auto& item : vec ) { + varuint32_to_bin( key, stream ); + to_protobuf_member( item, stream ); + } + } +} + + +template +void to_protobuf_object(const T& obj, S& stream) { + reflect::for_each( [&]( const clio::meta& ref, auto m ) { + if constexpr( not std::is_member_function_pointer_v ) { + write_protobuf_field( ref.number, obj.*m, stream ); + } + }); +} + +template +void to_protobuf_object(const std::tuple& obj, S& stream) { + tuple_for_each( obj, [&]( int idx, const auto& m ) { + write_protobuf_field( idx+1, m, stream ); + }); +} + +template +void to_protobuf(const T& obj, S& stream) { + to_protobuf_object( obj, stream ); +} + +template +std::vector to_protobuf(const T& t) { + size_stream ss; + to_protobuf(t, ss); + + std::vector result(ss.size, 0); + fixed_buf_stream fbs(result.data(), result.size()); + + to_protobuf(t, fbs); + + if (fbs.pos != fbs.end) + throw_error( stream_error::underrun ); + return result; +} + + + +} // clio diff --git a/libraries/clio/include/clio/translator.hpp b/libraries/clio/include/clio/translator.hpp new file mode 100644 index 000000000..98388574b --- /dev/null +++ b/libraries/clio/include/clio/translator.hpp @@ -0,0 +1,402 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace clio { + + class meta_type_base { + public: + meta_type_base( uint32_t n ):_number(n){} + + virtual ~meta_type_base(){} + virtual const char* name()const = 0; + + virtual std::vector json_to_protobuf( std::string json )const = 0; + virtual std::vector json_to_bin( std::string json )const = 0; + + virtual std::string protobuf_to_json( const std::vector& b )const = 0; + virtual std::vector protobuf_to_bin( const std::vector& b )const = 0; + + virtual std::vector bin_to_protobuf( const std::vector& b )const = 0; + virtual std::string bin_to_json( const std::vector& b )const = 0; + + uint32_t number()const { return _number; } + private: + uint32_t _number; + }; + + template + class meta_type : public meta_type_base + { + public: + meta_type( uint32_t n ):meta_type_base(n){} + virtual ~meta_type() {}; + + virtual const char* name()const override { + return get_type_name();//reflect::name(); + } + + virtual std::vector json_to_protobuf( std::string json )const override { + auto t = from_json( json ); + return to_protobuf( t ); + } + + virtual std::vector json_to_bin( std::string json )const override { + auto t = from_json( json ); + return to_bin( t ); + } + + virtual std::string protobuf_to_json( const std::vector& b )const override { + auto t = from_protobuf( b ); + return to_json(t); + } + + virtual std::vector protobuf_to_bin( const std::vector& b )const override { + auto t = from_protobuf( b ); + return to_bin(t); + } + + virtual std::vector bin_to_protobuf( const std::vector& b )const override { + auto t = from_bin( b ); + return to_protobuf( t ); + } + + virtual std::string bin_to_json( const std::vector& b )const override { + auto t = from_bin( b ); + return to_json( t ); + } + }; + + + template + class translator { + public: + translator() { + _schema.generate( [&]( auto* p ) { + _types.push_back( new meta_type>( _types.size() ) ); + }); + } + string get_json_schema()const { + return format_json( _schema ); + } + string get_protobuf_schema() { + return to_protobuf_schema( _schema ); + } + // string get_gql_schema(); + + uint32_t get_type_num( const string& type_name )const { + for( const auto& t : _types ) + if( t->name() == type_name ) return t->number(); + return -1; + } + + string get_type_name( uint32_t type_num )const { + if( type_num < _types.size() ) + return _types[type_num]->name(); + return string(); + } + + std::vector json_to_bin( uint32_t type_num, std::string json )const { + if( type_num < _types.size() ) + return _types[type_num]->json_to_bin( json ); + return std::vector(); + } + + std::vector json_to_protobuf( int type_num, std::string json )const { + if( type_num < _types.size() ) + return _types[type_num]->json_to_protobuf( json ); + return std::vector(); + } + + string bin_to_json( int type_num, const std::vector& bin )const { + if( type_num < _types.size() ) + return _types[type_num]->bin_to_json( bin ); + return string(); + } + std::vector bin_to_protobuf( int type_num, const std::vector& bin )const { + if( type_num < _types.size() ) + return _types[type_num]->bin_to_protobuf( bin ); + return std::vector(); + } + + string protobuf_to_json( int type_num, const std::vector& pbuf )const { + if( type_num < _types.size() ) + return _types[type_num]->protobuf_to_json( pbuf ); + return string(); + } + std::vector protobuf_to_bin( int type_num, const std::vector& pbuf )const { + if( type_num < _types.size() ) + return _types[type_num]->protobuf_to_bin( pbuf); + return std::vector(); + } + + string query_protobuf_to_json( const std::vector& pbuf_query )const { + clio::input_stream in( pbuf_query.data(), pbuf_query.size() ); + auto pbquery = clio::from_protobuf(in); + auto jquery = protobuf::to_json_query( pbquery ); + return to_json( jquery ); + } + + std::vector query_json_to_protobuf( string json_query )const { + auto jq = from_json( std::move(json_query) ); + auto pbq = clio::protobuf::from_json_query( jq ); + return to_protobuf(pbq); + } + + // string query_protobuf_to_gql( const byte_view& pbuf_query ); + // string query_json_to_gql( const string_view& json_query ); + // string query_gql_to_protobuf( const string_view& gql_query ); + // string query_gql_to_josn( const string_view& gql_query ); + private: + schema _schema; + std::vector _types; + }; + + template + void translate_json_to_protobuf( uint32_t number, InStream& in, OutStream& out ) { + varuint32 key; + key.value = number << 3; + if constexpr( std::is_same_v ) { + // wiretype 0 + } else if constexpr ( sizeof(T) == 8 ) + key.value |= uint8_t(protobuf::fixed64); + else if constexpr ( sizeof(T) == 4 ) + key.value |= uint8_t(protobuf::fixed32); + T r; + from_json( r, in ); + to_bin( key, out ); + to_bin( r, out ); + } + + template + bool translate_json_to_protobuf( const schema& sch, const std::string& pbuf_type, + InStream& in, OutStream& out ) { + auto otype = sch.get_object( pbuf_type ); + if( not otype ) { + auto vtype = sch.get_vector( pbuf_type ); + if( vtype ) { + } + return false; + } + in.get_start_object(); + auto t = in.peek_token(); + while( t.type != json_token_type::type_end_object ) { + std::string_view key = in.get_key(); + auto member = otype->get_member_by_name( key ); + if( member ) { + if( member->type == "double" ) + translate_json_to_protobuf( member->number, in, out ); + else if( member->type == "int64_t" ) + translate_json_to_protobuf( member->number, in, out ); + else if( member->type == "uint64_t" ) + translate_json_to_protobuf( member->number, in, out ); + else if( member->type == "int32_t" ) + translate_json_to_protobuf( member->number, in, out ); + else if( member->type == "uint32_t" ) + translate_json_to_protobuf( member->number, in, out ); + else if( member->type == "int16_t" ) + translate_json_to_protobuf( member->number, in, out ); + else if( member->type == "uint16_t" ) + translate_json_to_protobuf( member->number, in, out ); + else if( member->type == "int8_t" ) + translate_json_to_protobuf( member->number, in, out ); + else if( member->type == "uint8_t" ) + translate_json_to_protobuf( member->number, in, out ); + else if( member->type == "bool" ) + translate_json_to_protobuf( member->number, in, out ); + else if( member->type == "char" ) + translate_json_to_protobuf( member->number, in, out ); + else if( member->type == "varuint32" ) + translate_json_to_protobuf( member->number, in, out ); + else { + auto st = sch.get_type( member->type ); + if( not st ) from_json_skip_value( in ); + else if( object_type* obj = std::get_if(&*st) ) { + /// wire_type == wire_type_enum::buffer + } + else if( vector_type* obj = std::get_if(&*st) ) { + /// wire_type == wire_type_enum::buffer + } + else if( variant_type* obj = std::get_if(&*st) ) { + /// wire_type == wire_type_enum::buffer + } + else if( tuple_type* obj = std::get_if(&*st) ) { + /// wire_type == wire_type_enum::buffer + } + } + /* + translate_json_to_protobuf( sch, member->type, in, out ); + auto t = in.peek_token(); + switch( t.type ) { + case json_token_type::type_string: + case json_token_type::type_null: + case json_token_type::type_bool: + case json_token_type::type_start_object: + case json_token_type::type_start_array: + } + */ + } else { + from_json_skip_value( in ); + } + } + } + + template + bool translate_protobuf_to_json( const schema& sch, const std::string& pbuf_type, + InStream& in, OutStream& out ) { + using namespace clio::protobuf; + auto otype = sch.get_object( pbuf_type ); + if( not otype ) { + auto vtype = sch.get_vector( pbuf_type ); + if( vtype ) { + out.write( '[' ); + bool first = true; + + while( in.remaining() ) { + uint32_t key = 0; + varuint32_from_bin( key, in ); + wire_type_enum type = wire_type_enum(uint8_t(key) & 0x07); + //uint32_t number = key >> 3; + + auto contained_type = vtype->type; + + if( first ) { + first = false; + } else { + out.write( ',' ); + } + + switch( type ) { + case wire_type_enum::varint: { + varuint32 val; + from_bin( val, in ); + if( contained_type == "bool" ) + to_json( bool(val.value), out ); + else + to_json( val.value, out ); + } break; + case wire_type_enum::fixed64: { + int64_t val; + from_bin( val, in ); + if( contained_type == "double" ) + to_json( *((double*)&val), out ); + else + to_json( val, out ); + } break; + case wire_type_enum::buffer: { + if( contained_type == "string" ) { + std::string val; + from_bin( val, in ); + to_json( val, out ); + } else if( contained_type == "bytes" ) { + bytes val; + from_bin( val, in ); + to_json( val, out ); + } else { + varuint32 size; + from_bin( size, in ); + if( size.value > in.remaining() ) + return false; + input_stream obj_in( in.pos, in.pos + size ); + translate_protobuf_to_json( sch, contained_type, obj_in, out ); + in.skip( size ); + } + } break; + case wire_type_enum::fixed32: { + int32_t val; + from_bin( val, in ); + if( contained_type == "float" ) + to_json( *((float*)&val), out ); + else + to_json( val, out ); + } break; + }; + } + + out.write( ']' ); + return true; + } + return false; + } + + out.write( '{' ); + bool first = true; + while( in.remaining() ) { + uint32_t key = 0; + varuint32_from_bin( key, in ); + wire_type_enum type = wire_type_enum(uint8_t(key) & 0x07); + uint32_t number = key >> 3; + + //std::cout <<"field number: " << number <<" type: " << int64_t(type)<<" "; + auto member_meta = otype->get_member_by_number( number ); + + if( first ) { + first = false; + } else { + out.write( ',' ); + } + + if( member_meta ) { to_json( member_meta->name, out ); } + else { to_json( std::to_string(number), out ); } + + out.write( ':' ); + + switch( type ) { + case wire_type_enum::varint: { + varuint32 val; + from_bin( val, in ); + if( member_meta->type == "bool" ) + to_json( bool(val.value), out ); + else + to_json( val.value, out ); + } break; + case wire_type_enum::fixed64: { + int64_t val; + from_bin( val, in ); + if( member_meta->type == "double" ) + to_json( *((double*)&val), out ); + else + to_json( val, out ); + } break; + case wire_type_enum::buffer: { + if( member_meta->type == "string" ) { + std::string val; + from_bin( val, in ); + to_json( val, out ); + } else if( member_meta->type == "bytes" ) { + bytes val; + from_bin( val, in ); + to_json( val, out ); + } else { + varuint32 size; + from_bin( size, in ); + if( size.value > in.remaining() ) + return false; + input_stream obj_in( in.pos, in.pos + size ); + translate_protobuf_to_json( sch, member_meta->type, obj_in, out ); + in.skip( size ); + } + // a.members.emplace_back( number, std::move(val) ); + } break; + case wire_type_enum::fixed32: { + int32_t val; + from_bin( val, in ); + if( member_meta->type == "float" ) + to_json( *((float*)&val), out ); + else + to_json( val, out ); + } break; + }; + } + out.write( '}' ); + return true; + } + +}; diff --git a/libraries/clio/include/clio/tuple.hpp b/libraries/clio/include/clio/tuple.hpp new file mode 100644 index 000000000..fac00854e --- /dev/null +++ b/libraries/clio/include/clio/tuple.hpp @@ -0,0 +1,85 @@ +#pragma once +#include + +namespace clio { +enum class tuple_error { + no_error, + invalid_tuple_index +}; // tuple_error +} // namespace clio + +namespace std { + template <> + struct is_error_code_enum : true_type {}; +} // namespace std + + +namespace clio { + +class tuple_error_category_type : public std::error_category { + public: + const char* name() const noexcept override final { return "ConversionError"; } + + std::string message(int c) const override final { + switch (static_cast(c)) { + // clang-format off + case tuple_error::no_error: return "No error"; + case tuple_error::invalid_tuple_index: return "invalid tuple index"; + default: return "unknown"; + }; + } +}; + +inline const tuple_error_category_type& tuple_error_category() { + static tuple_error_category_type c; + return c; +} + +inline std::error_code make_error_code(tuple_error e) { return { static_cast(e), tuple_error_category() }; } + + + template + void tuple_get(T& obj, int pos, L&& lambda) { + if constexpr (N < std::tuple_size_v) { + if( N == pos ) { + lambda( std::get(obj) ); + } + else tuple_get(obj, pos, std::forward(lambda) ); + } else { + throw_error(tuple_error::invalid_tuple_index); + } + } + + template + void tuple_get(std::tuple& obj, int pos, L&& lambda) { + tuple_get<0>(obj, pos, std::forward(lambda) ); + } + + template + void tuple_for_each(T& obj, L&& lambda) { + if constexpr (N < std::tuple_size_v) { + lambda( N, std::get(obj) ); + tuple_for_each(obj, std::forward(lambda) ); + } + + } + + template + void tuple_for_each(std::tuple& obj, L&& lambda) { + tuple_for_each<0>(obj, std::forward(lambda) ); + } + + template + void tuple_for_each(const T& obj, L&& lambda) { + if constexpr (N < std::tuple_size_v) { + lambda( N, std::get(obj) ); + tuple_for_each(obj, std::forward(lambda) ); + } + } + + template + void tuple_for_each(const std::tuple& obj, L&& lambda) { + tuple_for_each<0>(obj, std::forward(lambda) ); + } + +} diff --git a/libraries/clio/include/clio/varint.hpp b/libraries/clio/include/clio/varint.hpp new file mode 100644 index 000000000..ceed4f1ae --- /dev/null +++ b/libraries/clio/include/clio/varint.hpp @@ -0,0 +1,432 @@ +/** + * @file + * @copyright defined in clio/LICENSE + */ +#pragma once +#include + +namespace clio { +/** + * @defgroup varint Variable Length Integer Type + * @ingroup core + * @ingroup types + * @brief Defines variable length integer type which provides more efficient serialization + */ + +/** + * Variable Length Unsigned Integer. This provides more efficient serialization of 32-bit unsigned int. + * It serialuzes a 32-bit unsigned integer in as few bytes as possible + * `varuint32` is unsigned and uses [VLQ or Base-128 encoding](https://en.wikipedia.org/wiki/Variable-length_quantity) + * + * @ingroup varint + */ +struct unsigned_int { + /** + * Construct a new unsigned int object + * + * @param v - Source + */ + unsigned_int(uint32_t v = 0) : value(v) {} + + /** + * Construct a new unsigned int object from a type that is convertible to uint32_t + * + * @tparam T - Type of the source + * @param v - Source + * @pre T must be convertible to uint32_t + */ + template + unsigned_int(T v) : value(v) {} + + // operator uint32_t()const { return value; } + // operator uint64_t()const { return value; } + + /** + * Convert unsigned_int as T + * + * @tparam T - Target type of conversion + * @return T - Converted target + */ + template + operator T() const { + return static_cast(value); + } + + /// @cond OPERATORS + + /** + * Assign 32-bit unsigned integer + * + * @param v - Soruce + * @return unsigned_int& - Reference to this object + */ + unsigned_int& operator=(uint32_t v) { + value = v; + return *this; + } + + /// @endcond + + /** + * Contained value + */ + uint32_t value; + + /// @cond OPERATORS + + /** + * Check equality between a unsigned_int object and 32-bit unsigned integer + * + * @param i - unsigned_int object to compare + * @param v - 32-bit unsigned integer to compare + * @return true - if equal + * @return false - otherwise + */ + friend bool operator==(const unsigned_int& i, const uint32_t& v) { return i.value == v; } + + /** + * Check equality between 32-bit unsigned integer and a unsigned_int object + * + * @param i - 32-bit unsigned integer to compare + * @param v - unsigned_int object to compare + * @return true - if equal + * @return false - otherwise + */ + friend bool operator==(const uint32_t& i, const unsigned_int& v) { return i == v.value; } + + /** + * Check equality between two unsigned_int objects + * + * @param i - First unsigned_int object to compare + * @param v - Second unsigned_int object to compare + * @return true - if equal + * @return false - otherwise + */ + friend bool operator==(const unsigned_int& i, const unsigned_int& v) { return i.value == v.value; } + + /** + * Check inequality between a unsigned_int object and 32-bit unsigned integer + * + * @param i - unsigned_int object to compare + * @param v - 32-bit unsigned integer to compare + * @return true - if inequal + * @return false - otherwise + */ + friend bool operator!=(const unsigned_int& i, const uint32_t& v) { return i.value != v; } + + /** + * Check inequality between 32-bit unsigned integer and a unsigned_int object + * + * @param i - 32-bit unsigned integer to compare + * @param v - unsigned_int object to compare + * @return true - if unequal + * @return false - otherwise + */ + friend bool operator!=(const uint32_t& i, const unsigned_int& v) { return i != v.value; } + + /** + * Check inequality between two unsigned_int objects + * + * @param i - First unsigned_int object to compare + * @param v - Second unsigned_int object to compare + * @return true - if inequal + * @return false - otherwise + */ + friend bool operator!=(const unsigned_int& i, const unsigned_int& v) { return i.value != v.value; } + + /** + * Check if the given unsigned_int object is less than the given 32-bit unsigned integer + * + * @param i - unsigned_int object to compare + * @param v - 32-bit unsigned integer to compare + * @return true - if i less than v + * @return false - otherwise + */ + friend bool operator<(const unsigned_int& i, const uint32_t& v) { return i.value < v; } + + /** + * Check if the given 32-bit unsigned integer is less than the given unsigned_int object + * + * @param i - 32-bit unsigned integer to compare + * @param v - unsigned_int object to compare + * @return true - if i less than v + * @return false - otherwise + */ + friend bool operator<(const uint32_t& i, const unsigned_int& v) { return i < v.value; } + + /** + * Check if the first given unsigned_int is less than the second given unsigned_int object + * + * @param i - First unsigned_int object to compare + * @param v - Second unsigned_int object to compare + * @return true - if i less than v + * @return false - otherwise + */ + friend bool operator<(const unsigned_int& i, const unsigned_int& v) { return i.value < v.value; } + + /** + * Check if the given unsigned_int object is greater or equal to the given 32-bit unsigned integer + * + * @param i - unsigned_int object to compare + * @param v - 32-bit unsigned integer to compare + * @return true - if i is greater or equal to v + * @return false - otherwise + */ + friend bool operator>=(const unsigned_int& i, const uint32_t& v) { return i.value >= v; } + + /** + * Check if the given 32-bit unsigned integer is greater or equal to the given unsigned_int object + * + * @param i - 32-bit unsigned integer to compare + * @param v - unsigned_int object to compare + * @return true - if i is greater or equal to v + * @return false - otherwise + */ + friend bool operator>=(const uint32_t& i, const unsigned_int& v) { return i >= v.value; } + + /** + * Check if the first given unsigned_int is greater or equal to the second given unsigned_int object + * + * @param i - First unsigned_int object to compare + * @param v - Second unsigned_int object to compare + * @return true - if i is greater or equal to v + * @return false - otherwise + */ + friend bool operator>=(const unsigned_int& i, const unsigned_int& v) { return i.value >= v.value; } + + /// @endcond +}; + +using varuint32 = unsigned_int; +CLIO_REFLECT(varuint32, value); + +template +void convert(const varuint32& src, uint32_t& dst, F&& chooser) { + dst = src.value; +} + +template +void from_bin(varuint32& obj, S& stream); + +template +void to_bin(const varuint32& obj, S& stream); + +template +void from_json(varuint32& obj, S& stream); + +template +void to_json(const varuint32& obj, S& stream); + +template +void to_key(const varuint32& obj, S& stream) { + return to_key_varuint32(obj.value, stream); +} + +/** + * Variable Length Signed Integer. This provides more efficient serialization of 32-bit signed int. + * It serializes a 32-bit signed integer in as few bytes as possible. + * + * @ingroup varint + * @note `varint32' is signed and uses [Zig-Zag + * encoding](https://developers.google.com/protocol-buffers/docs/encoding#signed-integers) + */ +struct signed_int { + /** + * Construct a new signed int object + * + * @param v - Source + */ + signed_int(int32_t v = 0) : value(v) {} + + /// @cond OPERATORS + + /** + * Convert signed_int to primitive 32-bit signed integer + * + * @return int32_t - The converted result + */ + operator int32_t() const { return value; } + + /** + * Assign an object that is convertible to int32_t + * + * @tparam T - Type of the assignment object + * @param v - Source + * @return unsigned_int& - Reference to this object + */ + template + signed_int& operator=(const T& v) { + value = v; + return *this; + } + + /** + * Increment operator + * + * @return signed_int - New signed_int with value incremented from the current object's value + */ + signed_int operator++(int) { return value++; } + + /** + * Increment operator + * + * @return signed_int - Reference to current object + */ + signed_int& operator++() { + ++value; + return *this; + } + + /// @endcond + + /** + * Contained value + */ + int32_t value; + + /// @cond OPERATORS + + /** + * Check equality between a signed_int object and 32-bit integer + * + * @param i - signed_int object to compare + * @param v - 32-bit integer to compare + * @return true - if equal + * @return false - otherwise + */ + friend bool operator==(const signed_int& i, const int32_t& v) { return i.value == v; } + + /** + * Check equality between 32-bit integer and a signed_int object + * + * @param i - 32-bit integer to compare + * @param v - signed_int object to compare + * @return true - if equal + * @return false - otherwise + */ + friend bool operator==(const int32_t& i, const signed_int& v) { return i == v.value; } + + /** + * Check equality between two signed_int objects + * + * @param i - First signed_int object to compare + * @param v - Second signed_int object to compare + * @return true - if equal + * @return false - otherwise + */ + friend bool operator==(const signed_int& i, const signed_int& v) { return i.value == v.value; } + + /** + * Check inequality between a signed_int object and 32-bit integer + * + * @param i - signed_int object to compare + * @param v - 32-bit integer to compare + * @return true - if inequal + * @return false - otherwise + */ + friend bool operator!=(const signed_int& i, const int32_t& v) { return i.value != v; } + + /** + * Check inequality between 32-bit integer and a signed_int object + * + * @param i - 32-bit integer to compare + * @param v - signed_int object to compare + * @return true - if unequal + * @return false - otherwise + */ + friend bool operator!=(const int32_t& i, const signed_int& v) { return i != v.value; } + + /** + * Check inequality between two signed_int objects + * + * @param i - First signed_int object to compare + * @param v - Second signed_int object to compare + * @return true - if inequal + * @return false - otherwise + */ + friend bool operator!=(const signed_int& i, const signed_int& v) { return i.value != v.value; } + + /** + * Check if the given signed_int object is less than the given 32-bit integer + * + * @param i - signed_int object to compare + * @param v - 32-bit integer to compare + * @return true - if i less than v + * @return false - otherwise + */ + friend bool operator<(const signed_int& i, const int32_t& v) { return i.value < v; } + + /** + * Check if the given 32-bit integer is less than the given signed_int object + * + * @param i - 32-bit integer to compare + * @param v - signed_int object to compare + * @return true - if i less than v + * @return false - otherwise + */ + friend bool operator<(const int32_t& i, const signed_int& v) { return i < v.value; } + + /** + * Check if the first given signed_int is less than the second given signed_int object + * + * @param i - First signed_int object to compare + * @param v - Second signed_int object to compare + * @return true - if i less than v + * @return false - otherwise + */ + friend bool operator<(const signed_int& i, const signed_int& v) { return i.value < v.value; } + + /** + * Check if the given signed_int object is greater or equal to the given 32-bit integer + * + * @param i - signed_int object to compare + * @param v - 32-bit integer to compare + * @return true - if i is greater or equal to v + * @return false - otherwise + */ + friend bool operator>=(const signed_int& i, const int32_t& v) { return i.value >= v; } + + /** + * Check if the given 32-bit integer is greater or equal to the given signed_int object + * + * @param i - 32-bit integer to compare + * @param v - signed_int object to compare + * @return true - if i is greater or equal to v + * @return false - otherwise + */ + friend bool operator>=(const int32_t& i, const signed_int& v) { return i >= v.value; } + + /** + * Check if the first given signed_int is greater or equal to the second given signed_int object + * + * @param i - First signed_int object to compare + * @param v - Second signed_int object to compare + * @return true - if i is greater or equal to v + * @return false - otherwise + */ + friend bool operator>=(const signed_int& i, const signed_int& v) { return i.value >= v.value; } + + /// @endcond +}; + +using varint32 = signed_int; +CLIO_REFLECT(varint32, value); + +template +void from_bin(varint32& obj, S& stream); + +template +void to_bin(const varint32& obj, S& stream); + +template +void from_json(varint32& obj, S& stream); + +template +void to_json(const varint32& obj, S& stream); + +template +void to_key(const varint32& obj, S& stream) { + to_key_varint32(obj.value, stream); +} + +} // namespace clio diff --git a/libraries/clio/tests/CMakeLists.txt b/libraries/clio/tests/CMakeLists.txt new file mode 100644 index 000000000..54e0dde62 --- /dev/null +++ b/libraries/clio/tests/CMakeLists.txt @@ -0,0 +1,23 @@ +if(DEFINED IS_NATIVE) + find_package(Threads REQUIRED) + add_executable(clio-tests + clio_tests.cpp + flat_views.cpp + benchmark.cpp + crypto.cpp + ) + target_link_libraries(clio-tests clio clcrypto fc catch2-portable Threads::Threads) +endif() + +if(DEFINED IS_WASM) + add_executable(clio-tests + clio_tests.cpp + flat_views.cpp + benchmark.cpp + ) + target_link_libraries(clio-tests clio catch2-portable boost) +endif() + +set_target_properties(clio-tests PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${ROOT_BINARY_DIR}) + +native_and_wasm_test(clio-tests) diff --git a/libraries/clio/tests/benchmark.cpp b/libraries/clio/tests/benchmark.cpp new file mode 100644 index 000000000..e22fa533c --- /dev/null +++ b/libraries/clio/tests/benchmark.cpp @@ -0,0 +1,404 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +namespace benchmark +{ + struct no_sub + { + double myd = 3.14; + uint16_t my16 = 22; + ; + }; + CLIO_REFLECT(no_sub, myd, my16); + + struct sub_obj + { + int a; + int b; + std::string substr; + no_sub ns; + }; + CLIO_REFLECT(sub_obj, a, b, ns, substr); + + struct flat_object + { + uint32_t x; + double y; + std::string z; + std::vector veci; + std::vector vecstr = {"aaaaaaaaa", "bbbbbbbbbbb", "ccccccccccccc"}; + std::vector vecns; + std::vector nested; + sub_obj sub; + }; + CLIO_REFLECT(flat_object, x, y, z, veci, vecstr, vecns, nested, sub); +} // namespace benchmark + +TEST_CASE("benchmark") +{ + using namespace benchmark; + + flat_object tester; + tester.z = "my oh my"; + tester.x = 99; + tester.y = 99.99; + tester.veci = {1, 2, 3, 4, 6}; + tester.vecns = {{.myd = 33.33}, {}}; + tester.nested = {{.x = 11, .y = 3.21, .nested = {{.x = 88}}}, {.x = 33, .y = .123}}; + tester.sub.substr = "sub str"; + tester.sub.a = 3; + tester.sub.b = 4; + + SECTION("unpacking from flat") + { + clio::size_stream ss; + clio::flatpack(tester, ss); + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::flatpack(tester, ps); + auto start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + flat_object temp; + clio::input_stream in(buf.data(), buf.size()); + clio::flatunpack(temp, in); + } + auto end = std::chrono::steady_clock::now(); + auto delta = end - start; + + std::cout << "unpack flat: " << std::chrono::duration(delta).count() + << " ms size: " << ss.size << " "; + + auto compressed = clio::capn_compress(buf); + auto uncompressed = clio::capn_uncompress(compressed); + + clio::size_stream capsize; + clio::input_stream capin(buf.data(), buf.size()); + clio::capp_pack(capin, capsize); + + // std::cout <<"compressed.size: " << compressed.size() <<"\n"; + std::cout << "capsize: " << capsize.size << "\n"; + // std::cout <<"uncompressed.size: " << uncompressed.size() <<"\n"; + } + SECTION("validating flat without unpacking") + { + clio::size_stream ss; + clio::flatpack(tester, ss); + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::flatpack(tester, ps); + + auto start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + clio::input_stream in(buf.data(), buf.size()); + clio::flatcheck(in); + } + auto end = std::chrono::steady_clock::now(); + auto delta = end - start; + + std::cout << "check flat: " << std::chrono::duration(delta).count() + << " ms size: " << ss.size << " "; + + auto compressed = clio::capn_compress(buf); + auto uncompressed = clio::capn_uncompress(compressed); + + clio::size_stream capsize; + clio::input_stream capin(buf.data(), buf.size()); + clio::capp_pack(capin, capsize); + + // std::cout <<"compressed.size: " << compressed.size() <<"\n"; + std::cout << "capsize: " << capsize.size << "\n"; + // std::cout <<"uncompressed.size: " << uncompressed.size() <<"\n"; + } + + SECTION("flat unpack and uncompress") + { + clio::size_stream ss; + clio::flatpack(tester, ss); + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::flatpack(tester, ps); + + auto compressed = clio::capn_compress(buf); + + auto start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + flat_object temp; + auto uncompressed = clio::capn_uncompress(compressed); + clio::input_stream in(uncompressed.data(), uncompressed.size()); + clio::flatunpack(temp, in); + } + auto end = std::chrono::steady_clock::now(); + auto delta = end - start; + + std::cout << "unpack flat&cap: " << std::chrono::duration(delta).count() + << " ms size: " << ss.size << " "; + clio::size_stream capsize; + clio::input_stream capin(buf.data(), buf.size()); + clio::capp_pack(capin, capsize); + + std::cout << "capsize: " << capsize.size << "\n"; + } + + SECTION("protbuf unpack") + { + clio::size_stream ss; + clio::to_protobuf(tester, ss); + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::to_protobuf(tester, ps); + + auto start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + flat_object temp; + + clio::input_stream in(buf.data(), buf.size()); + clio::from_protobuf_object(temp, in); + } + auto end = std::chrono::steady_clock::now(); + auto delta = end - start; + std::cout << "unpack pb: " << std::chrono::duration(delta).count() + << " ms size: " << ss.size << " "; + clio::size_stream capsize; + clio::input_stream capin(buf.data(), buf.size()); + clio::capp_pack(capin, capsize); + + std::cout << "capsize: " << capsize.size << "\n"; + } + + SECTION("protbuf unpack and uncompress") + { + clio::size_stream ss; + clio::to_protobuf(tester, ss); + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::to_protobuf(tester, ps); + + auto compressed = clio::capn_compress(buf); + + auto start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + flat_object temp; + auto uncompressed = clio::capn_uncompress(compressed); + + clio::input_stream in(uncompressed.data(), uncompressed.size()); + clio::from_protobuf_object(temp, in); + } + auto end = std::chrono::steady_clock::now(); + auto delta = end - start; + std::cout << "unpack pb&cap: " << std::chrono::duration(delta).count() + << " ms size: " << ss.size << " "; + clio::size_stream capsize; + clio::input_stream capin(buf.data(), buf.size()); + clio::capp_pack(capin, capsize); + + std::cout << "capsize: " << capsize.size << "\n"; + } + + SECTION("bin unpack") + { + clio::size_stream ss; + clio::to_bin(tester, ss); + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::to_bin(tester, ps); + auto start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + flat_object temp; + clio::input_stream in(buf.data(), buf.size()); + clio::from_bin(temp, in); + } + auto end = std::chrono::steady_clock::now(); + auto delta = end - start; + std::cout << "unpack bin: " << std::chrono::duration(delta).count() + << " ms size: " << ss.size << " "; + clio::size_stream capsize; + clio::input_stream capin(buf.data(), buf.size()); + clio::capp_pack(capin, capsize); + + std::cout << "capsize: " << capsize.size << "\n"; + } + SECTION("bin unpack and uncompress") + { + clio::size_stream ss; + clio::to_bin(tester, ss); + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::to_bin(tester, ps); + + auto compressed = clio::capn_compress(buf); + + auto start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + flat_object temp; + auto uncompressed = clio::capn_uncompress(compressed); + + clio::input_stream in(uncompressed.data(), uncompressed.size()); + clio::from_bin(temp, in); + } + auto end = std::chrono::steady_clock::now(); + auto delta = end - start; + std::cout << "unpack bin&cap: " << std::chrono::duration(delta).count() + << " ms size: " << ss.size << " "; + clio::size_stream capsize; + clio::input_stream capin(buf.data(), buf.size()); + clio::capp_pack(capin, capsize); + + std::cout << "capsize: " << capsize.size << "\n"; + } + + SECTION("json unpack") + { + clio::size_stream ss; + clio::to_json(tester, ss); + + std::vector buf(ss.size + 1); + std::vector buf2(ss.size + 1); + buf.back() = 0; + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::to_json(tester, ps); + buf2 = buf; + auto start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + flat_object temp; + clio::json_token_stream in(buf.data()); + clio::from_json(temp, in); + buf = buf2; + } + auto end = std::chrono::steady_clock::now(); + auto delta = end - start; + std::cout << "unpack json: " << std::chrono::duration(delta).count() + << " ms size: " << ss.size << " capsize: "; + clio::size_stream capsize; + clio::input_stream capin(buf.data(), buf.size()); + clio::capp_pack(capin, capsize); + + std::cout << "capsize: " << capsize.size << "\n"; + } + + SECTION("packing") + { + size_t s = 0; + auto start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + clio::size_stream ss; + clio::flatpack(tester, ss); + s = ss.size; + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::flatpack(tester, ps); + // std::cout <<"capsize: " << capsize.size <<"\n"; + } + auto end = std::chrono::steady_clock::now(); + auto delta = end - start; + std::cout << "pack flat: " << std::chrono::duration(delta).count() + << " ms size: " << s << "\n"; + + s = 0; + start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + clio::size_stream ss; + clio::flatpack(tester, ss); + s = ss.size; + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::flatpack(tester, ps); + + // clio::size_stream capsize; + + std::vector buf2(ss.size); + clio::fixed_buf_stream capout(buf2.data(), buf2.size()); + clio::input_stream capin(buf.data(), buf.size()); + clio::capp_pack(capin, capout); + s = buf2.size() - capout.remaining(); // capsize.size; + + // std::cout <<"capsize: " << capsize.size <<"\n"; + } + end = std::chrono::steady_clock::now(); + delta = end - start; + std::cout << "pack flat&cap: " << std::chrono::duration(delta).count() + << " ms size: " << s << "\n"; + + start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + clio::size_stream ss; + clio::to_protobuf(tester, ss); + s = ss.size; + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::to_protobuf(tester, ps); + } + end = std::chrono::steady_clock::now(); + delta = end - start; + std::cout << "pack pb: " << std::chrono::duration(delta).count() + << " ms size: " << s << "\n"; + + start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + clio::size_stream ss; + clio::to_bin(tester, ss); + s = ss.size; + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::to_bin(tester, ps); + } + end = std::chrono::steady_clock::now(); + delta = end - start; + std::cout << "pack bin: " << std::chrono::duration(delta).count() + << " ms size: " << s << "\n"; + + start = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < 10000; ++i) + { + clio::size_stream ss; + clio::to_json(tester, ss); + s = ss.size; + + std::vector buf(ss.size); + clio::fixed_buf_stream ps(buf.data(), buf.size()); + clio::to_json(tester, ps); + } + end = std::chrono::steady_clock::now(); + delta = end - start; + std::cout << "pack json: " << std::chrono::duration(delta).count() + << " ms size: " << s << "\n"; + } +} diff --git a/libraries/clio/tests/clio_tests.cpp b/libraries/clio/tests/clio_tests.cpp new file mode 100644 index 000000000..4ed06df1f --- /dev/null +++ b/libraries/clio/tests/clio_tests.cpp @@ -0,0 +1,2 @@ +#define CATCH_CONFIG_MAIN +#include diff --git a/libraries/clio/tests/crypto.cpp b/libraries/clio/tests/crypto.cpp new file mode 100644 index 000000000..5740afd01 --- /dev/null +++ b/libraries/clio/tests/crypto.cpp @@ -0,0 +1,66 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +struct identity +{ + clcrypto::private_key priv; + clcrypto::public_key pub; + clcrypto::signature sig; + std::string hello; + clcrypto::sha256 digest; +}; + +CLIO_REFLECT(identity, hello, digest, priv, pub, sig) + +TEST_CASE("keygen") +{ + std::cout << "KEYGEN\n"; + auto Priv = fc::crypto::private_key::generate_r1(); + auto Pub = Priv.get_public_key(); + + std::cout << Priv.to_string() << "\n"; + std::cout << Pub.to_string() << "\n"; + + clcrypto::k1pri privk = clcrypto::k1pri::generate(); + clcrypto::k1pub pubk = privk.get_public_key(); // clcrypto::k1pub(); + + // auto hashhello = fc::sha256::hash( "hello world", 11 ); + auto hashhello = fc::sha256::hash(std::string("hello world")); + auto sig = privk.sign(hashhello); + + std::cout << std::string(pubk) << "\n"; + std::cout << std::string(privk) << "\n"; + std::cout << hashhello.str() << "\n"; + std::cout << std::string(sig) << "\n"; + + auto rek = sig.recover(hashhello); + std::cout << std::string(rek) << "\n"; + + identity id; + id.priv = clcrypto::r1pri::generate(); + id.pub = clcrypto::get_public_key(id.priv); + id.sig = clcrypto::sign(id.priv, id.digest); + std::cout << clio::to_json(id) << "\n"; +} diff --git a/libraries/clio/tests/database.cpp b/libraries/clio/tests/database.cpp new file mode 100644 index 000000000..4fdbf48a2 --- /dev/null +++ b/libraries/clio/tests/database.cpp @@ -0,0 +1,727 @@ +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +struct db_handle +{ + uint32_t id; +}; + +/** + * Defines an global API that represents an abstract database that is able to + * create, fetch, and free records. + */ +class intrinsic +{ + public: + static intrinsic& instance() + { + static intrinsic i; + return i; + } + + enum error_code + { + ok = 0, /// everything is ok + invalid_handle = 1, /// the handle is unknwon to us + freed_handle = 2, /// the handle requested has already been freed + invalid_buffer_pram = 3, + invalid_size_param = 4, + invalid_handle_param = 5 /// the user provided an invalid handle paramter ref + }; + + error_code db_create(size_t size, db_handle* new_handle) + { + if (size <= 0) + return invalid_size_param; + if (new_handle == nullptr) + return invalid_handle_param; + + _database.emplace_back(std::vector(size)); + + auto& rec = _database.back(); + rec.resize(size); + + new_handle->id = _database.size() - 1; + return ok; + } + + error_code db_set(db_handle h, const char* buffer, size_t size) + { + if (size <= 0) + return invalid_size_param; + if (h.id < _database.size()) + { + auto& rec = _database[h.id]; + if (rec.size() == 0) + return freed_handle; + rec.resize(size); + memcpy(rec.data(), buffer, size); + return ok; + } + else + return invalid_handle; + } + + size_t db_get_size(db_handle h) + { + if (h.id < _database.size()) + { + return _database[h.id].size(); + } + else + return invalid_handle; + } + + error_code db_get(db_handle h, char* buffer, size_t size_in) + { + if (h.id < _database.size()) + { + auto& rec = _database[h.id]; + if (buffer && size_in) + { + auto read_size = std::min(size_in, rec.size()); + memcpy(buffer, rec.data(), read_size); + } + else + { + return invalid_buffer_pram; + } + } + else + return invalid_handle; + return ok; + } + + error_code db_free(db_handle h) + { + if (h.id >= _database.size()) + return invalid_handle_param; + _free_list.emplace_back(h.id, std::move(_database[h.id])); + return ok; + } + + private: + std::vector>> _free_list; + std::vector> _database; +}; + +/** + * This serves as a wasm-side cache of records already feteched from the intrinsic interface, + * it has a singleton interface and all fetches from the database should check with the cache + * instead of going to the intrinsic API. The cache will fetch from the database if not found. + */ +class cache +{ + public: + /** + * This object holds the one-true copy of the buffer for a given db handle. + * It is movable but not copyable. + */ + class cache_record + { + public: + size_t size() const { return _buffer.size(); } + const char* data() const { return _buffer.data(); } + bool is_dirty() const { return _dirty; } + db_handle handle() const { return _handle; } + + void resize(uint32_t s) { _buffer.resize(s); } + char* data() { return _buffer.data(); } + + /** + * @return a non-const reference to this after setting the dirty bit and notifying + * the cache + */ + cache_record& modify() const + { + if (not _dirty) + { + _dirty = true; + _cache._dirty.push_back(this); + } + return *const_cast(this); + } + + cache_record(cache_record&& mv) + : _cache(mv._cache), + _handle(mv._handle), + _buffer(std::move(mv._buffer)), + _dirty(mv._dirty) + { + mv._handle.id = -1; + } + + cache_record& operator=(cache_record&& mv) + { + /// TODO assert &_cache == &mv._cache... this may be redundant + /// since the cache is a global singleton, there may be no good + /// reason to store _cache on the record at all and we can save + /// some memory. + + if (&mv != this) + { + _handle = mv._handle; + mv._handle.id = -1; + _buffer = std::move(mv._buffer); + _dirty = mv._dirty; + } + return *this; + } + + private: + cache_record(const cache_record&) = delete; + cache_record(cache& c, db_handle h) : _cache(c), _handle(h) {} + + cache& _cache; + db_handle _handle; + std::vector _buffer; + mutable bool _dirty = false; + + friend class cache; + }; + + cache_record& create(size_t new_size) + { + db_handle new_handle; + intrinsic::instance().db_create(new_size, &new_handle); + auto& r = _db_cache + .emplace(std::pair(new_handle.id, + cache_record(*this, new_handle))) + .first->second; + r.modify(); // pushes it to cache + return r; + } + + const cache_record& get(db_handle h) + { + auto itr = _db_cache.find(h.id); + if (itr != _db_cache.end()) + return itr->second; + + size_t s = intrinsic::instance().db_get_size(h); + + if (s <= 0) + { + /// some kind of error happened, abort! + } + + auto& r = _db_cache.emplace(h.id, cache_record(*this, h)).first->second; + + r._buffer.resize(s); + if (auto err = intrinsic::instance().db_get(h, r._buffer.data(), s)) + { + /// some kind of error happened, abort! + } + return r; + } + + static cache& instance() + { + static cache c; + return c; + } + + private: + std::vector _dirty; + std::map _db_cache; +}; + +template +struct db_ptr; + +template +struct db_ref +{ + db_ref(const cache::cache_record& c) : _cr(c) {} + + const T* operator->() const { return reinterpret_cast(_cr.data()); } + + T* operator->() { return reinterpret_cast(_cr.modify().data()); } + + operator db_ptr() const { return db_ptr(_cr.handle()); } + operator db_handle() const { return _cr.handle(); } + + private: + const cache::cache_record& _cr; +}; + +template +struct db_ptr +{ + db_ptr() {} + db_ptr(db_handle h) : id(h.id) {} + + uint32_t id = -1; + + bool operator!() const { return id == -1; } + + friend bool operator==(const db_ptr& a, const db_ptr& b) { return a.id == b.id; } + friend bool operator!=(const db_ptr& a, const db_ptr& b) { return a.id != b.id; } + + const db_ref operator->() const { return cache::instance().get({id}); } + const db_ref operator*() const { return cache::instance().get({id}); } +}; + +template +db_ref make_db_ptr(Args&&... args) +{ + auto& cr = cache::instance().create(sizeof(T)); + cr.resize(sizeof(T)); + new (cr.data()) T(std::forward(args)...); + return cr; +} + +/** + * This is the data that is actually stored in the database, it should + * be trivially copyable. + */ +template +struct node +{ + static_assert(std::is_trivially_copyable::value); + + template + node(Args&&... args) : data(std::forward(args)...) + { + } + + db_ptr> left; + db_ptr> right; + db_ptr> parent; + int height = 0; + T data; +}; + +/** + * The underlying node object only has db_ptr<> for left/right, which + * means that every time you dereference one of them you have to go to + * the cache and do a map/hash table lookup. This would dramatically hurt + * the performance of the log(n) lookup required. The node_cache object + * keeps a cache of the left/right db_ref that will only fectch if they + * have not yet been loaded from cache/db. + * + * When performing tree operations you must modify both self->left/right and + * this->left/right or the cache will break its mirror to the underlying. + */ +template +class node_cache : public std::enable_shared_from_this> +{ + public: + using node_cptr = std::shared_ptr>; + using node_db_ptr = db_ptr>; + + node_cache(db_ref> s) : self(s) {} + + const T& value() const { return self->data; } + int height() const { return self->height; } + + const node_cptr& left() const + { + if (not _left and !!self->left) + { + _left = std::make_shared>(*self->left); + } + return _left; + } + const node_cptr& right() const + { + if (not _right and !!self->right) + { + _right = std::make_shared>(*self->right); + } + return _right; + } + const node_cptr& parent() const + { + if (not _parent and !!self->parent) + { + _parent = std::make_shared>(*self->parent); + } + return _parent; + } + + void set_parent(node_cptr new_parent) const + { + _parent = new_parent; + + node_db_ptr new_parent_id = _parent ? _parent->ptr() : node_db_ptr(); + if (self->parent != new_parent_id) + { + const_cast>&>(self)->parent = new_parent_id; + } + } + + void set_right(node_cptr new_right) const + { + _right = new_right; + + node_db_ptr new_right_id = _right ? _right->ptr() : node_db_ptr(); + if (self->right != new_right_id) + { + const_cast>&>(self)->right = new_right_id; + } + if (_right) + _right->set_parent(this->shared_from_this()); + } + + void set_left(node_cptr new_left) const + { + _left = new_left; + node_db_ptr new_left_id = _left ? _left->ptr() : node_db_ptr(); + if (self->left != new_left_id) + { + const_cast>&>(self)->left = new_left_id; + } + if (_left) + _left->set_parent(this->shared_from_this()); + } + + void set_height(int h) const + { + if (self->height != h) + { + const_cast>&>(self)->height = h; + } + } + + /* + operator db_ref>& () { return self; } + operator const db_ref>& ()const { return self; } + operator db_ptr>()const { return self; } + operator db_handle()const { return self; } + db_handle handle()const { return self; } + */ + + db_ptr> ptr() const { return self; } + + private: + db_ref> self; + + mutable node_cptr _left; + mutable node_cptr _right; + mutable node_cptr _parent; +}; + +/** + * This is the datatype that stores the tree in the DB, it only + * stores the root. + */ +template +struct tree +{ + typedef db_ptr> node_type; + node_type root; +}; + +/** + * This is the interface that actually impliments the algorithms on + * the underlying memory. + */ +template +class tree_cache +{ + using node_cptr = std::shared_ptr>; + + public: + /** + * Constructed with a reference to the tree database record, + * the tree is a record that contains a single db handle to + * the current root of the tree. + */ + tree_cache(db_ref> s) : self(s) {} + + void insert(const T& v) const { set_root(insert(v, root())); } + + template + void visit(L&& v) const + { + visit(std::forward(v), root()); + } + + struct iterator + { + iterator() {} + const T& operator*() const { return pos->value(); } + iterator& operator++() + { + if (not pos) + return *this; + // std::cout << "start: " << **this << "\n"; + + auto& right = pos->right(); + if (right) + { + pos = right; + // std::cout << " go right to: " << **this << "\n"; + while (auto& left = pos->left()) + { + pos = left; + // std::cout << " go left to: " << **this << "\n"; + } + } + else + { + // std::cout << " go to parent while parent right != me\n"; + while (auto& parent = pos->parent()) + { + if (parent->right() == pos) + { + pos = parent; + // std::cout << " go to next parent: " << **this << "\n"; + } + else + { + // std::cout << " go to parent: " << **this << "\n"; + pos = parent; + return *this; + } + } + // std::cout << " no more parents! We are done\n"; + pos = node_cptr(); + } + return *this; + } + + friend bool operator==(const iterator& a, const iterator& b) { return a.pos == b.pos; } + friend bool operator!=(const iterator& a, const iterator& b) { return a.pos != b.pos; } + + private: + friend class tree_cache; + iterator(const node_cptr& p) : pos(p) {} + node_cptr pos; + }; + + iterator lower_bound(const T& v) const { return lower_bound(v, root()); } + iterator end() const { return iterator(node_cptr()); } + + private: + int height(const node_cptr& t) const { return t ? t->height() : -1; } + + template + void visit(L&& v, const node_cptr& t) const + { + if (not t) + return; + visit(std::forward(v), t->left()); + v(t->value()); + visit(std::forward(v), t->right()); + } + + iterator lower_bound(const T& v, const node_cptr& t) const + { + if (not t) + return iterator(); + std::cout << " t->value: " << t->value() << "\n"; + if (v < t->value()) + { + auto& left = t->left(); + if (left) + return lower_bound(v, left); + } + else if (v > t->value()) + { + auto& right = t->right(); + if (right) + return lower_bound(v, right); + } + return iterator(t); + } + + node_cptr insert(const T& v, node_cptr t) const + { + if (not t) + { + return make_node(v); + } + else if (v < t->value()) + { + t->set_left(insert(v, t->left())); + auto tl = t->left(); + auto tr = t->right(); + if (height(tl) - height(tr) == 2) + { + if (v < tl->value()) + { + t = single_right_rotate(t); + } + else + { + t = double_right_rotate(t); + } + } + } + else if (v > t->value()) + { + t->set_right(insert(v, t->right())); + auto tl = t->left(); + auto tr = t->right(); + if (height(tr) - height(tl) == 2) + { + if (v > tr->value()) + { + t = single_left_rotate(t); + } + else + { + t = double_left_rotate(t); + } + } + } + t->set_height(std::max(height(t->left()), height(t->right())) + 1); + return t; + } + + node_cptr single_right_rotate(const node_cptr& t) const + { + auto u = t->left(); + t->set_left(u->right()); + u->set_right(t); + t->set_height(std::max(height(t->left()), height(t->right())) + 1); + u->set_height(std::max(height(u->left()), height(u->right())) + 1); + return u; + } + node_cptr single_left_rotate(const node_cptr& t) const + { + auto u = t->right(); + t->set_right(u->left()); + u->set_left(t); + t->set_height(std::max(height(t->left()), height(t->right())) + 1); + u->set_height(std::max(height(u->left()), height(u->right())) + 1); + return u; + } + + node_cptr double_left_rotate(const node_cptr& t) const + { + t->set_right(single_right_rotate(t->right())); + return single_left_rotate(t); + } + + node_cptr double_right_rotate(const node_cptr& t) const + { + t->set_left(single_left_rotate(t->left())); + return single_right_rotate(t); + } + + const node_cptr& root() const + { + if (not _root and !!self->root) + { + set_root(std::make_shared>(*self->root)); + } + return _root; + } + + /** + * This function ensures the invariant that self.id == r->id, it + * is too easy change one without changing the other... therefore + * we use 'const' everywhere and this one function is given the + * right to cast it away and update the root. + */ + void set_root(const node_cptr& r) const { const_cast(this)->set_root(r); } + void set_root(const node_cptr& r) + { + _root = r; + if (not _root) + { + self->root = db_ptr>(); /// null id + } + else + { + std::cout << "_root->ptr.id(): " << _root->ptr().id << " self->root.id: " << self->root.id + << " value: " << _root->value() << "\n"; + /// TODO: make sure things really did change so we + /// don't trigger the dirty bit + // if( _root->handle().id != self->root ) { + + if (_root->ptr() != self->root) + { + std::cout << " current root: " << self->root.id; + std::cout << " setting root: " << _root->ptr().id << " \n"; + self->root = _root->ptr(); //.id = _root->handle().id; + } + _root->set_parent(node_cptr()); + } + } + + node_cptr make_node(const T& v) const + { + return std::make_shared>(make_db_ptr>(v)); + } + + db_ref> self; + node_cptr _root; +}; + +/* +template +struct base_index : { + T value; +} + +template +struct index : base_index { + index* left; + index* right; + index* parent; + int height; +} +*/ + +int main(int, char**) +{ + // auto root = make_db_ptr>( 5 ); + tree_cache tc(make_db_ptr>()); + /* + tc.insert( 6 ); + tc.insert( 8 ); + tc.insert( 9 ); + tc.insert( 2 ); + tc.insert( 41 ); + tc.insert( 42); + tc.insert( 43); + tc.insert( 44); + tc.insert( 45); + tc.insert( 46); + */ + for (uint32_t i = 0; i < 32; ++i) + tc.insert(i); + + auto itr = tc.lower_bound(10); + while (itr != tc.end()) + { + std::cout << "value: " << *itr << "\n"; + ++itr; + } + + /* + tc.visit( [&]( const auto& v ) { + std::cout << v << "\n"; + }); + */ + + // auto itr = tr->lower_bound(5); + + // tr->insert( 5 ); + + // std::cout << "root: " << root->data <<"\n"; + + // clio::flat_ptr> n = node(); + + return 0; +} diff --git a/libraries/clio/tests/db.cpp b/libraries/clio/tests/db.cpp new file mode 100644 index 000000000..8c8083ae8 --- /dev/null +++ b/libraries/clio/tests/db.cpp @@ -0,0 +1,222 @@ +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +class intrinsic +{ + public: + struct db_handle + { + uint32_t id; + }; + enum error_code + { + ok = 0, /// everything is ok + invalid_handle = 1, /// the handle is unknwon to us + freed_handle = 2, /// the handle requested has already been freed + invalid_buffer_pram = 3, + invalid_size_param = 4, + invalid_handle_param = 5 /// the user provided an invalid handle paramter ref + }; + + error_code db_create(size_t size, db_handle* new_handle) + { + if (size <= 0) + return invalid_size_param; + if (new_handle == nullptr) + return invalid_handle_param; + + _database.emplace_back(buffer, buffer + size); + + auto& rec = _database.back(); + rec.resize(size); + + new_handle->id = _database.size() - 1; + return ok; + } + + error_code db_set(db_handle h, const char* buffer, size_t size) + { + if (size <= 0) + return invalid_size_param; + if (h.id < _database.size()) + { + auto& rec = _database[h.id]; + if (rec.size() == 0) + return freed_handle; + rec.resize(size); + memcpy(rec.data(), buffer, size); + return ok; + } + else + return invalid_handle; + } + + size_t db_get_size(db_handle h) + { + if (h.id < _database.size()) + { + return _database[h.id].size(); + } + else + return invalid_handle; + } + + error_code db_get(db_handle h, char* buffer, size_t size_in) + { + if (h.id < _database.size()) + { + auto& rec = _database[h.id]; + if (buffer && size_in) + { + auto read_size = std::min(size_in, rec.size()); + memcpy(buffer, rec.data(), read_size); + } + else + { + return invalid_buffer_pram; + } + } + else + return invalid_handle; + } + + error_code db_free(db_handle h) + { + if (h.id >= _database.size()) + return invalid_handle_param; + _free_list.push_back(std::make_pair(h, std::move(_database[h.id]))); + return ok; + } + + private: + std::vector < std::pair > _free_list; + std::vector > _database; +}; + +class cache +{ + public: + class cache_record + { + public: + size_t size() const { return buffer.size(); } + const char* data() const { return buffer.data(); } + bool is_dirty() const { return _dirty; } + db_handle handle() const { return _handle; } + + template + cache_record& modify(M&& mod) + { + _dirty = true; + mod(_buffer); + return *this; + } + + cache_record(cache_record&& mv) + : _cache(mv._cache), _handle(mv._handle), _buffer(std::move(mv._buffer)) _dirty(mv._dirty) + { + _mv.handle.id = -1; + } + + private: + cache_record(const cache_record&) = delete; + cache_record(cache& c, db_handle h) : _cache(c), _handle(h) {} + + cache& _cache; + db_handle _handle; + std::vector _buffer; + bool _dirty = false; + }; + + template + cache_record& db_create(L&& constructor) + { + db_handle new_handle; + intrinsic::db_create(rec.size, &new_handle); + auto& r = _db_cache[new_handle.id] = + cache_record(*this, new_handle).modify(std::forward(constructor)); + _dirty.push_back(&r); + } + + cache_record& db_get(db_handle h) + { + auto itr = _db_cache.find(h.id); + if (itr != _db_cache.end()) + return itr->second; + + size_t s = db_get(h); + + if (s <= 0) + { + /// some kind of error happened, abort! + } + + auto& r = _db_cache[h] = cache_record(*this, h); + + r._buffer.resize(s); + if (auto err = intrinsic::db_get(h, r._buffer.data(), s)) + { + /// some kind of error happened, abort! + } + return r; + } + + private: + vector _dirty; + map _db_cache; +}; + +/** + * Can be serialized, and resets _record to nullptr when _handle changes + */ +class db_ptr +{ + public: + const char* data() const { return _get_record().data(); } + size_t size() const { return _get_record().size(); } + + template + void modify(L&& mod) + { + return _get_record().modify(std::forward(mod)); + } + + db_handle handle() const { return _handle; } + + private: + db_handle _handle; + cache::cache_record* _record = nullptr; +}; + +template +class view_ptr_impl : private db_ptr +{ + public: +}; + +template +struct tree_root +{ + T value; +}; + +int main(int argc, char** argv) +{ + return 0; +} diff --git a/libraries/clio/tests/flat_views.cpp b/libraries/clio/tests/flat_views.cpp new file mode 100644 index 000000000..09c1301fc --- /dev/null +++ b/libraries/clio/tests/flat_views.cpp @@ -0,0 +1,232 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +struct non_ref_nest +{ + float lite = 33; +}; +struct non_reflect_type +{ + int first = 0; + int second = 0; + double real = 3.14; + non_ref_nest nrn; +}; + +struct no_sub +{ + double myd = 3.14; + uint16_t my16 = 22; + ; +}; +CLIO_REFLECT(no_sub, myd, my16); + +struct sub_obj +{ + int a; + int b; + int c; + std::string substr; + no_sub ns; +}; +CLIO_REFLECT(sub_obj, a, b, c, substr, ns); + +struct flat_object +{ + non_reflect_type non_ref; + uint32_t x; + double y; + std::string z; + std::vector veci; + std::vector vecstr = {"aaaaaaaaa", "bbbbbbbbbbb", "ccccccccccccc"}; + std::vector vecns; + std::vector nested; + sub_obj sub; + clio::flat_ptr sub_view; +}; +CLIO_REFLECT(flat_object, non_ref, x, y, z, veci, vecstr, vecns, nested, sub, sub_view); + +struct fat +{ + uint16_t a; + uint16_t b; + std::string substr; +}; +CLIO_REFLECT(fat, a, b, substr); + +struct root +{ + std::string other; + clio::flat_ptr fatptr; +}; +CLIO_REFLECT(root, other, fatptr); + +TEST_CASE("flatptr") +{ + constexpr uint32_t offseta = + clio::get_tuple_offset<0, std::tuple>>::value; + constexpr uint32_t offsetb = + clio::get_tuple_offset<1, std::tuple>>::value; + std::cout << offseta << " " << offsetb << "\n"; + std::cout << "flatpacksize: " << clio::flatpack_size() << "\n"; + std::cout << "contains_offset_ptr: " << clio::contains_offset_ptr() << "\n"; + + /* + clio::flat_ptr r( root{ .other = "other", .fatptr = fat{ .a = 0x0A, .b = 0x0B, .substr = + "sub" } } ); std::cout << "r.size: " << r.size() << "\n"; std::vector d( r.data(), + r.data()+r.size() ); std::cout << clio::to_json(clio::bytes{d}) <<"\n"; + + std::cout << std::hex << r->fatptr->get()->a << "\n"; + std::cout << std::hex << r->fatptr->get()->b << "\n"; + std::cout << std::hex << r->fatptr->get()->substr << "\n"; + std::cout << std::dec << r->fatptr->get()->substr << "\n"; + */ + + flat_object tester; + tester.z = "my oh my"; + tester.x = 99; + tester.y = 99.99; + tester.veci = {1, 2, 3, 4, 6}; + tester.vecns = {{.myd = 33.33}, {}}; + tester.nested = {{.x = 11, .y = 3.21, .nested = {{.x = 88}}}, {.x = 33, .y = .123}}; + tester.sub.substr = "zzzzzzzzzz"; + tester.sub.ns.myd = .9876; + tester.sub.ns.my16 = 33; + tester.sub.a = 3; + tester.sub.b = 4; + tester.sub_view = sub_obj{.a = 0xdddd, .b = 0xbbbb, .substr = "yyyyyyyyyy"}; + + clio::flat_ptr me(tester); + std::cout << "fpsize: " << clio::flatpack_size() << "\n"; + // std::cerr<< "sub.a: " << std::hex << me->sub->a<<"\n"; + std::cerr << "sub_view->get()->a: \n"; + int x = me->sub_view->a; + std::cerr << "sub_view.a: " << std::hex << x << "\n"; + /* + + int y = me->sub->a.get(); + std::cerr<< "sub.a: " << std::hex << y <<"\n"; + + std::vector d( me.data(), me.data()+me.size() ); + std::cout << clio::to_json(clio::bytes{d}) <<"\n"; + std::cout << "start: " << std::dec << int64_t(me.data()) <<"\n"; + */ +} +TEST_CASE("flatview") +{ + flat_object tester; + tester.z = "my oh my"; + tester.x = 99; + tester.y = 99.99; + tester.veci = {1, 2, 3, 4, 6}; + tester.vecns = {{.myd = 33.33}, {}}; + tester.nested = {{.x = 11, .y = 3.21, .nested = {{.x = 88}}}, {.x = 33, .y = .123}}; + tester.sub.substr = "sub str"; + tester.sub.ns.myd = .9876; + tester.sub.ns.my16 = 33; + tester.sub.a = 3; + tester.sub.b = 4; + tester.sub_view = sub_obj{.a = 987, .b = 654, .substr = "subviewstr"}; + + clio::flat_ptr me(tester); + std::cerr << "me.size: " << me.size() << "\n"; + + auto start = std::chrono::steady_clock::now(); + uint64_t sum = 0; + for (uint32_t i = 0; i < 10000; ++i) + { + for (uint32_t x = 0; x < me->veci->size(); ++x) + sum += me->veci[x]; + } + auto end = std::chrono::steady_clock::now(); + auto delta = end - start; + + auto view_time = (delta).count(); + std::cout << "sum veci via view: " << std::chrono::duration(delta).count() + << " ms\n"; + + start = std::chrono::steady_clock::now(); + sum = 0; + for (uint32_t i = 0; i < 10000; ++i) + { + for (uint32_t x = 0; x < tester.veci.size(); ++x) + sum += tester.veci[x]; + } + end = std::chrono::steady_clock::now(); + delta = end - start; + std::cout << "sum: " << sum << "\n"; + std::cout << "sum veci via real: " << std::chrono::duration(delta).count() + << " ms\n"; + auto real_time = (delta).count(); + + std::cout << "view runtime: " << 100 * double(view_time) / real_time << "% of real\n"; + + std::cout << "sub.substr: " << me->sub->substr << "\n"; + + std::cout << "x: " << me->x << "\n"; + std::cout << "y: " << me->y << "\n"; + std::cout << "z: " << me->z << "\n"; + std::cout << "veci[0]: " << me->veci[0] << "\n"; + std::cout << "veci[1]: " << me->veci[1] << "\n"; + std::cout << "veci[2]: " << me->veci[2] << "\n"; + std::cout << "veci[3]: " << me->veci[3] << "\n"; + std::cout << "veci[4]: " << me->veci[4] << "\n"; + std::cout << "vecstr[0]: " << me->vecstr[0] << "\n"; + std::cout << "vecstr[1]: " << me->vecstr[1] << "\n"; + std::cout << "vecstr[2]: " << me->vecstr[2] << "\n"; + std::cout << "vecns[0].myd: " << me->vecns[0].myd << "\n"; + std::cout << "vecns[0].my16: " << me->vecns[0].my16 << "\n"; + std::cout << "vecns[1].myd: " << me->vecns[1].myd << "\n"; + std::cout << "vecns[1].my16: " << me->vecns[1].my16 << "\n"; + std::cout << "nested->size(): " << me->nested->size() << "\n"; + std::cout << "nested[0].x: " << me->nested[0].x << "\n"; + std::cout << "nested[0].y: " << me->nested[0].y << "\n"; + std::cout << "nested[0].z: " << me->nested[0].z << "\n"; + std::cout << "nested[0].nested[0].x: " << me->nested[0].nested[0].x << "\n"; + std::cout << "nested[0].nested[0].y: " << me->nested[0].nested[0].y << "\n"; + std::cout << "nested[1].z: " << me->nested[1].z << "\n"; + std::cout << "nested[1].x: " << me->nested[1].x << "\n"; + std::cout << "nested[1].y: " << me->nested[1].y << "\n"; + std::cout << "sub.a: " << me->sub->a << "\n"; + std::cout << "sub.b: " << me->sub->b << "\n"; + std::cout << "sub.ns.myd: " << me->sub->ns->myd << "\n"; + std::cout << "sub.ns.my16: " << me->sub->ns->my16 << "\n"; + + clio::input_stream in(me.data(), me.size()); + clio::flatcheck(in); + + std::cout << "sub_view.a: " << me->sub_view->a << "\n"; + // std::cout << "sub_view.b: " << me->sub_view->b<<"\n"; + + std::cout << "tester.sub.substr: " << tester.sub.substr << "\n"; + + flat_object copy = me; /// unpacks from buffer into full object + std::cout << "copy.sub.a: " << copy.sub.a << "\n"; + std::cout << "copy.sub.b: " << copy.sub.b << "\n"; + std::cout << "copy.sub.ns.myd: " << copy.sub.ns.myd << "\n"; + std::cout << "copy.sub.ns.my16: " << copy.sub.ns.my16 << "\n"; + std::cout << "copy.sub.substr: " << copy.sub.substr << "\n"; + std::cout << "copy.sub_view->a: " << copy.sub_view->a << "\n"; + std::cout << "copy.sub_view->b: " << copy.sub_view->b << "\n"; + std::cout << "copy.sub_view->b: " << copy.sub_view->substr << "\n"; + + std::cout << "non_ref->real: " << me->non_ref->real << "\n"; + std::cout << "non_ref->nrn.lite: " << me->non_ref->nrn.lite << "\n"; +} diff --git a/libraries/clio/tests/gql.cpp b/libraries/clio/tests/gql.cpp new file mode 100644 index 000000000..a887dfbbb --- /dev/null +++ b/libraries/clio/tests/gql.cpp @@ -0,0 +1,66 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +struct service +{ + int add(int a, int b) const + { + std::cout << a << " + " << b << "\n"; + return a + b; + } + int sub(int a, int b) + { + std::cout << a << " - " << b << "\n"; + return a - b; + } +}; + +CLIO_REFLECT_PB(service, (add, 0, a, b), (sub, 1, a, b)) + +TEST_CASE("try flatpack/unpack/view of std::varaint", "[variant]") +{ + using variant_type = std::variant; + + auto test_type = [](auto val) { + variant_type test_variant; + test_variant = val; + clio::flat_ptr packed = test_variant; + + variant_type unpacked = packed; + std::cout << "test_variant: " << clio::to_json(test_variant) << "\n"; + std::cout << "unpacked_variant: " << clio::to_json(unpacked) << "\n"; + + packed->visit([](const auto& v) { + std::cout << "visit value: " << v << "\n"; + /* + if constexpr ( std::is_same_v,std::decay_t> ) { + std::cout << "json value: " << clio::format_json( std::string_view(v) ) <<"\n"; + } else { + std::cout << "json value: " << clio::format_json(v) <<"\n"; + } + */ + }); + // clio::input_stream in( packed.data(), packed.size() ); + // clio::flatcheck( in ); + }; + test_type(33); + test_type(33.33); + test_type("hello world"); +} + +TEST_CASE("try flat pack/view of gql::query", "[gql]") +{ + clio::gql::query q; + + clio::schema service_schema; + service_schema.generate(); + // std::cout << "schema: " << clio::format_json( service_schema )<<"\n"; + + clio::flat_ptr fq(q); +} diff --git a/libraries/clio/tests/protobuf_query.cpp b/libraries/clio/tests/protobuf_query.cpp new file mode 100644 index 000000000..b927a2a33 --- /dev/null +++ b/libraries/clio/tests/protobuf_query.cpp @@ -0,0 +1,77 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +struct service +{ + int add(int a, int b) const + { + std::cout << a << " + " << b << "\n"; + return a + b; + } + int sub(int a, int b) + { + std::cout << a << " - " << b << "\n"; + return a - b; + } +}; + +CLIO_REFLECT(service, add, sub) + +TEST_CASE("try calling methods via protobuf", "[protobuf]") +{ + clio::schema service_schema; + service_schema.generate(); + service_schema.generate(); + std::cout << "schema: " << format_json(service_schema) << "\n"; + + clio::reflect::proxy query; + query.add(2, 3); + query.add(8, 5); + query.sub(2, 3); + + std::cout << clio::format_json(query->q) << "\n"; + auto protobuf_query = clio::to_protobuf(query->q); + + clio::input_stream protobuf_query_in(protobuf_query.data(), protobuf_query.size()); + std::string json_query; + clio::string_stream json_query_out(json_query); + + clio::translate_protobuf_to_json(service_schema, "query", protobuf_query_in, json_query_out); + + std::cout << "pbuf query: " << format_json(clio::bytes{protobuf_query}) << "\n"; + std::cout << "json pbuf query: " << json_query << "\n"; + std::cout << "json query: " + << clio::format_json(clio::protobuf::to_json_query(query->q)) << "\n"; + + service s; + auto result = clio::protobuf::dispatch(s, query->q); + + std::cout << clio::format_json(result) << "\n"; + + auto protobuf_result = clio::to_protobuf(result); + std::string json_result; + + clio::input_stream protobuf_in(protobuf_result.data(), protobuf_result.size()); + clio::string_stream json_out(json_result); + + clio::translate_protobuf_to_json(service_schema, "service", protobuf_in, json_out); + + std::cout << "json result: " << json_result << "\n"; +} From e86e98b77fdc48ad92a7a7fe0a3c426971575ec9 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Mon, 26 Apr 2021 20:33:46 -0400 Subject: [PATCH 2/3] format --- libraries/clio/include/clio/bytes.hpp | 15 +- .../clio/include/clio/bytes/from_json.hpp | 15 +- libraries/clio/include/clio/bytes/to_bin.hpp | 4 +- libraries/clio/include/clio/bytes/to_json.hpp | 15 +- libraries/clio/include/clio/compress.hpp | 245 +-- libraries/clio/include/clio/error.hpp | 32 +- libraries/clio/include/clio/flatbuf.hpp | 1627 ++++++++++------- libraries/clio/include/clio/fpconv.h | 546 +++--- libraries/clio/include/clio/from_bin.hpp | 520 +++--- .../clio/include/clio/from_bin/varint.hpp | 24 +- libraries/clio/include/clio/from_json.hpp | 1303 +++++++------ .../clio/include/clio/from_json/varint.hpp | 24 +- libraries/clio/include/clio/from_protobuf.hpp | 364 ++-- libraries/clio/include/clio/get_type_name.hpp | 334 ++-- libraries/clio/include/clio/gql.hpp | 189 +- libraries/clio/include/clio/json/any.hpp | 408 +++-- libraries/clio/include/clio/member_proxy.hpp | 159 +- libraries/clio/include/clio/murmur.hpp | 105 +- libraries/clio/include/clio/name.hpp | 364 ++-- libraries/clio/include/clio/powers.h | 78 +- libraries/clio/include/clio/protobuf/any.hpp | 258 +-- libraries/clio/include/clio/protobuf/json.hpp | 333 ++-- .../clio/include/clio/protobuf/query.hpp | 405 ++-- .../clio/include/clio/protobuf/schema.hpp | 377 ++-- libraries/clio/include/clio/reflect.hpp | 639 ++++--- libraries/clio/include/clio/schema.hpp | 569 +++--- libraries/clio/include/clio/stream.hpp | 533 +++--- libraries/clio/include/clio/to_bin.hpp | 375 ++-- libraries/clio/include/clio/to_bin/deque.hpp | 15 +- libraries/clio/include/clio/to_bin/list.hpp | 15 +- libraries/clio/include/clio/to_bin/map.hpp | 15 +- libraries/clio/include/clio/to_bin/set.hpp | 15 +- libraries/clio/include/clio/to_bin/varint.hpp | 24 +- libraries/clio/include/clio/to_json.hpp | 511 +++--- libraries/clio/include/clio/to_json/map.hpp | 53 +- .../clio/include/clio/to_json/varint.hpp | 24 +- libraries/clio/include/clio/to_key.hpp | 550 +++--- libraries/clio/include/clio/to_protobuf.hpp | 321 ++-- libraries/clio/include/clio/translator.hpp | 840 +++++---- libraries/clio/include/clio/tuple.hpp | 51 +- libraries/clio/include/clio/varint.hpp | 866 ++++----- 41 files changed, 7185 insertions(+), 5975 deletions(-) diff --git a/libraries/clio/include/clio/bytes.hpp b/libraries/clio/include/clio/bytes.hpp index e02cf8020..cae397467 100644 --- a/libraries/clio/include/clio/bytes.hpp +++ b/libraries/clio/include/clio/bytes.hpp @@ -2,12 +2,13 @@ #include #include -namespace clio { +namespace clio +{ + struct bytes + { + std::vector data; + }; -struct bytes { - std::vector data; -}; + CLIO_REFLECT(bytes, data); -CLIO_REFLECT(bytes, data); - -} // namespace clio +} // namespace clio diff --git a/libraries/clio/include/clio/bytes/from_json.hpp b/libraries/clio/include/clio/bytes/from_json.hpp index 216722542..8d03c6e85 100644 --- a/libraries/clio/include/clio/bytes/from_json.hpp +++ b/libraries/clio/include/clio/bytes/from_json.hpp @@ -2,11 +2,12 @@ #include #include -namespace clio { +namespace clio +{ + template + void from_json(bytes& obj, S& stream) + { + clio::from_json_hex(obj.data, stream); + } -template -void from_json(bytes& obj, S& stream) { - clio::from_json_hex(obj.data, stream); -} - -} // namespace clio +} // namespace clio diff --git a/libraries/clio/include/clio/bytes/to_bin.hpp b/libraries/clio/include/clio/bytes/to_bin.hpp index 16e44ea6e..18e8962fb 100644 --- a/libraries/clio/include/clio/bytes/to_bin.hpp +++ b/libraries/clio/include/clio/bytes/to_bin.hpp @@ -1,6 +1,6 @@ #pragma once #include -namespace clio { - +namespace clio +{ } diff --git a/libraries/clio/include/clio/bytes/to_json.hpp b/libraries/clio/include/clio/bytes/to_json.hpp index 6a3cd258e..7f23fa995 100644 --- a/libraries/clio/include/clio/bytes/to_json.hpp +++ b/libraries/clio/include/clio/bytes/to_json.hpp @@ -2,11 +2,12 @@ #include #include -namespace clio { +namespace clio +{ + template + void to_json(const bytes& obj, S& stream) + { + clio::to_json_hex(obj.data.data(), obj.data.size(), stream); + } -template -void to_json(const bytes& obj, S& stream) { - clio::to_json_hex(obj.data.data(), obj.data.size(), stream); -} - -} // namespace clio +} // namespace clio diff --git a/libraries/clio/include/clio/compress.hpp b/libraries/clio/include/clio/compress.hpp index 1bac24fe3..95e1a0bd4 100644 --- a/libraries/clio/include/clio/compress.hpp +++ b/libraries/clio/include/clio/compress.hpp @@ -1,157 +1,180 @@ #pragma once #include -namespace clio { - -template -void capp_unpack( InStream& in, OutStream& out ) { - while( in.remaining() ) { - uint8_t word_bits; - in.read(&word_bits,1); - - const uint64_t zero_word = 0; - - if( word_bits == 0x00 ) { - out.write(&zero_word,8); +namespace clio +{ + template + void capp_unpack(InStream& in, OutStream& out) + { + while (in.remaining()) + { + uint8_t word_bits; + in.read(&word_bits, 1); + + const uint64_t zero_word = 0; + + if (word_bits == 0x00) + { + out.write(&zero_word, 8); uint8_t zero_words; - in.read( &zero_words, 1 ); + in.read(&zero_words, 1); - while( zero_words ) { - out.write(&zero_word,8); - --zero_words; + while (zero_words) + { + out.write(&zero_word, 8); + --zero_words; } - if( in.pos >= in.end ) { - throw_error( clio::stream_error::overrun ); + if (in.pos >= in.end) + { + throw_error(clio::stream_error::overrun); } - } else if( word_bits == 0xff ) { - if( in.remaining() < 8 ) { - out.write( in.pos, in.remaining() ); - in.skip( in.remaining() ); - return; + } + else if (word_bits == 0xff) + { + if (in.remaining() < 8) + { + out.write(in.pos, in.remaining()); + in.skip(in.remaining()); + return; } uint64_t word; - in.read( &word, sizeof(word) ); - out.write( &word, 8 ); + in.read(&word, sizeof(word)); + out.write(&word, 8); uint8_t raw_words; - in.read( &raw_words,1); - if( raw_words ) { - if( in.remaining() < uint32_t(raw_words * 8) ) - throw_error( clio::stream_error::overrun ); - out.write( in.pos, uint32_t(raw_words) * 8 ); - in.pos += uint32_t(raw_words) * 8; + in.read(&raw_words, 1); + if (raw_words) + { + if (in.remaining() < uint32_t(raw_words * 8)) + throw_error(clio::stream_error::overrun); + out.write(in.pos, uint32_t(raw_words) * 8); + in.pos += uint32_t(raw_words) * 8; } - } else { - uint64_t out_word = 0; + } + else + { + uint64_t out_word = 0; uint8_t* out_p = (uint8_t*)&out_word; - for( auto i = 7; i >= 0; --i ) { - if( word_bits & (1<= 0; --i) + { + if (word_bits & (1 << i)) + { + in.read(out_p, 1); + out_p++; + } + else + { + out_p++; + } } - out.write( &out_word, sizeof(out_word) ); - } - } -} - -template -void capp_pack( InStream& in, OutStream& out ) { - while( in.remaining() >= 8 ) { - uint64_t next_word; - in.read_raw(next_word); - - if( not next_word ) { + out.write(&out_word, sizeof(out_word)); + } + } + } + + template + void capp_pack(InStream& in, OutStream& out) + { + while (in.remaining() >= 8) + { + uint64_t next_word; + in.read_raw(next_word); + + if (not next_word) + { out.write(char(0)); uint32_t num_zero = 0; in.read_raw(next_word); - while( not next_word && num_zero < 254) { - in.read_raw(next_word); - ++num_zero; + while (not next_word && num_zero < 254) + { + in.read_raw(next_word); + ++num_zero; } - out.write( char(num_zero) ); - } - + out.write(char(num_zero)); + } - uint8_t* inbyte = (uint8_t*)&next_word; - uint8_t* endin = inbyte + 8; + uint8_t* inbyte = (uint8_t*)&next_word; + uint8_t* endin = inbyte + 8; - uint8_t temp[9]; - uint8_t& zeros = temp[0]; - uint8_t* outbyte = &temp[1]; - zeros = 0; + uint8_t temp[9]; + uint8_t& zeros = temp[0]; + uint8_t* outbyte = &temp[1]; + zeros = 0; - do { + do + { zeros <<= 1; - if( bool(*inbyte) ) { - zeros += 1; - *outbyte = *inbyte; - ++outbyte; + if (bool(*inbyte)) + { + zeros += 1; + *outbyte = *inbyte; + ++outbyte; } ++inbyte; - } while( inbyte != endin ); - out.write( temp, outbyte-temp ); + } while (inbyte != endin); + out.write(temp, outbyte - temp); - if( zeros == 0xff ) { + if (zeros == 0xff) + { uint8_t num_raw = 0; auto* p = in.pos; auto* e = in.end; - while( *p and p != e ) ++p; + while (*p and p != e) + ++p; - num_raw = (p - in.pos)/8; - out.write( num_raw ); - out.write( in.pos, num_raw * 8 ); - in.skip( num_raw * 8 ); + num_raw = (p - in.pos) / 8; + out.write(num_raw); + out.write(in.pos, num_raw * 8); + in.skip(num_raw * 8); - // in.read_raw(next_word); - /// TODO: once we enter "raw" mode we shouldn't exit until + // in.read_raw(next_word); + /// TODO: once we enter "raw" mode we shouldn't exit until /// we have seen a certain percentage of zero bytes... for example, - /// one zero per word isn't worth encoding... - } - - - } - if( in.remaining() ) { - out.write( char(0xff) ); /// indicate all bytes are present, receiver will truncate - out.write( in.pos, in.remaining() ); - in.skip( in.remaining() ); - } - return; -}; - -inline std::vector capn_compress( const std::vector& c ) { - clio::input_stream in( c.data(), c.size() ); - /* + /// one zero per word isn't worth encoding... + } + } + if (in.remaining()) + { + out.write(char(0xff)); /// indicate all bytes are present, receiver will truncate + out.write(in.pos, in.remaining()); + in.skip(in.remaining()); + } + return; + }; + + inline std::vector capn_compress(const std::vector& c) + { + clio::input_stream in(c.data(), c.size()); + /* clio::size_stream sizess; capp_unpack( in, sizess); in.pos = c.data(); */ - std::vector out; - out.reserve( c.size() ); - clio::vector_stream vout(out); - capp_pack( in, vout ); - return out; -} -inline std::vector capn_uncompress( const std::vector& c ) { - clio::input_stream in( c.data(), c.size() ); - /* + std::vector out; + out.reserve(c.size()); + clio::vector_stream vout(out); + capp_pack(in, vout); + return out; + } + inline std::vector capn_uncompress(const std::vector& c) + { + clio::input_stream in(c.data(), c.size()); + /* clio::size_stream sizess; capp_unpack( in, sizess); in.pos = c.data(); */ - std::vector out; - out.reserve( c.size()*1.5 ); - clio::vector_stream vout(out); - capp_unpack( in, vout ); - return out; -} + std::vector out; + out.reserve(c.size() * 1.5); + clio::vector_stream vout(out); + capp_unpack(in, vout); + return out; + } -} /// clio +} // namespace clio diff --git a/libraries/clio/include/clio/error.hpp b/libraries/clio/include/clio/error.hpp index 010f1a489..b6bcea931 100644 --- a/libraries/clio/include/clio/error.hpp +++ b/libraries/clio/include/clio/error.hpp @@ -1,17 +1,19 @@ #pragma once -namespace clio { - /** - * On some platforms we need to disable - * exceptions and do something else, this - * wraps that. - */ - template - [[noreturn]] void throw_error( T&& e ){ - #ifdef __cpp_exceptions - throw e; - #else - abort(); - #endif - } -}; +namespace clio +{ + /** + * On some platforms we need to disable + * exceptions and do something else, this + * wraps that. + */ + template + [[noreturn]] void throw_error(T&& e) + { +#ifdef __cpp_exceptions + throw e; +#else + abort(); +#endif + } +}; // namespace clio diff --git a/libraries/clio/include/clio/flatbuf.hpp b/libraries/clio/include/clio/flatbuf.hpp index 077c9f237..7851be0e3 100644 --- a/libraries/clio/include/clio/flatbuf.hpp +++ b/libraries/clio/include/clio/flatbuf.hpp @@ -1,748 +1,953 @@ #pragma once #include -#include #include +#include -namespace clio { - - struct flat_view_proxy_impl; - - template - class flat_ptr; - - template - struct is_flat_ptr : std::false_type {}; - - template - struct is_flat_ptr> : std::true_type { using value_type = T; }; - - template - class flat { - using T::flat_type_not_defined; - }; - - template<> - class flat; - - template - class flat >; - - - template - class flat>; - - template - auto get_view_type() { -// if constexpr( is_flat_ptr::value ) { -// return get_view_type(); - if constexpr( reflect::is_struct ) { - using view_type = typename reflect::template proxy; - return (view_type*)(nullptr); - } - else if constexpr( std::is_trivially_copyable::value ) - return (T*)(nullptr); - else - return (flat*)nullptr; - } - - template - using flat_view = std::remove_pointer_t())>; - - struct offset_ptr { - uint32_t offset; - - template - auto get()const; - }; - - template - constexpr bool contains_offset_ptr(); - - template - struct tuple_contains_offset; - - - template - constexpr uint32_t get_contains_offset_ptr() { - if constexpr( sizeof...(Ts) == 0 ) - return contains_offset_ptr(); - else { - return contains_offset_ptr() | - get_contains_offset_ptr(); - } - } - - template - struct tuple_contains_offset< std::tuple > { - static constexpr const auto value = get_contains_offset_ptr(); - }; - - - /** - * Recursively checks the types for any field which requires dynamic allocation, - */ - template - constexpr bool contains_offset_ptr() { - if constexpr ( is_flat_ptr::value ) { - return true; - } else if constexpr( is_std_tuple::value ) { - return tuple_contains_offset::value; - } else if constexpr( is_std_variant::value ) { - return contains_offset_ptr::alts_as_tuple>(); - } else if constexpr( std::is_same_v ) { - return true; - } else if constexpr( is_std_vector::value ) { - return true; - } else if constexpr( clio::reflect::is_struct ) { - bool is_flat = true; - clio::reflect::for_each( [&]( const clio::meta& ref, auto mptr ){ - using member_type = std::decay_t; - is_flat &= not contains_offset_ptr(); - }); - return not is_flat; - } else if constexpr( std::is_arithmetic_v ) { - return false; - } else if constexpr( std::is_trivially_copyable::value ) { - return false; - } else { - T::contains_offset_ptr_not_defined; - } - } - - - template - constexpr uint32_t flatpack_size() { - if constexpr( is_flat_ptr::value ) { - return sizeof(offset_ptr); - } else if constexpr( is_std_variant::value ) { - return 16; - } else if constexpr( reflect::is_struct ) { - uint32_t size = 0; - reflect::for_each( [&]( const meta& ref, const auto& mptr ){ - using member_type = decltype(result_of_member(mptr)); - if constexpr ( contains_offset_ptr() ) { - size += sizeof(offset_ptr); - } else { - size += flatpack_size(); - } - }); - return size; - } else if constexpr( std::is_same_v || is_std_vector::value) { - return sizeof(offset_ptr); - } else if constexpr( std::is_arithmetic_v ) { - return sizeof(T); - } else if constexpr( std::is_trivially_copyable::value ) { - return sizeof(T); - } else { - T::flatpack_size_not_defined; - } - } - - template - struct get_tuple_offset; - - template - constexpr uint32_t get_offset() { - static_assert( Idx < sizeof...(Ts) + 1, "index out of range" ); - if constexpr ( Idx == 0 ) return 0; - else if constexpr( sizeof...(Ts) == 0 ) - if constexpr( contains_offset_ptr() ) return 4; - /// if contains ptr then 8 else flatpack_size... - else return flatpack_size(); - // return get_flat_size( ((const First*)(nullptr)) ); - else { - if constexpr( contains_offset_ptr() ) - return get_offset() + 4; //flatpack_size(); - else return get_offset() + flatpack_size(); - // return get_offset< Idx-1, Ts...>() + get_flat_size( ((const First*)(nullptr)) ); - } - } - - template - struct get_tuple_offset< I, std::tuple > { - static constexpr const uint32_t value = get_offset(); - }; - - - struct flat_view_proxy_impl { - - /** This method is called by the reflection library to get the field */ - template - constexpr auto get() { - using class_type = decltype( clio::class_of_member(MemberPtr) ); - using tuple_type = typename clio::reflect::struct_tuple_type; - using member_type = decltype( clio::result_of_member(MemberPtr) ); - - constexpr uint32_t offset = clio::get_tuple_offset::value; - - char* out_ptr = reinterpret_cast(this)+offset; - - if constexpr ( contains_offset_ptr() ) { - clio::offset_ptr* ptr = reinterpret_cast(out_ptr); - return ptr->get(); - } else { - return reinterpret_cast(out_ptr); - } - } - - template - constexpr const auto get()const { - using class_type = decltype( clio::class_of_member(MemberPtr) ); - using tuple_type = typename clio::reflect::struct_tuple_type; - using member_type = decltype( clio::result_of_member(MemberPtr) ); - - constexpr uint32_t offset = clio::get_tuple_offset::value; - - auto out_ptr = reinterpret_cast(this)+offset; - - if constexpr ( contains_offset_ptr() ) { - const clio::offset_ptr* ptr = reinterpret_cast(out_ptr); - return ptr->get(); - } else { - return reinterpret_cast(out_ptr); - } - } - - }; - - /** - * A flat view of a flat pointer. - */ - template - class flat > { - public: - auto get() { - return reinterpret_cast< decltype( get_view_type() ) >(_data); - } - private: - uint32_t _size = 0; - char _data[]; - }; - - - template<> - class flat { - public: - uint32_t size()const { return _size; } - const char* c_str()const { return _size ? _data : (const char*)this; } - const char* data()const { return _size ? _data : nullptr; } - char* data() { return _size ? _data : nullptr; } - - operator std::string_view()const { return std::string_view( _data, _size ); } - - template - friend S& operator << ( S& stream, const flat& str ) { - return stream << str.c_str(); - } - private: - uint32_t _size = 0; - char _data[]; - }; - - template - class flat> { - public: - uint64_t type = 0; - uint32_t flat_data = 0; - offset_ptr offset_data; - - flat() { - static_assert( sizeof(flat) == 16 ); - static_assert( std::is_trivially_copyable< flat >::value ); - } - - int64_t index_from_type()const { - return get_index_from_type(); +namespace clio +{ + struct flat_view_proxy_impl; + + template + class flat_ptr; + + template + struct is_flat_ptr : std::false_type + { + }; + + template + struct is_flat_ptr> : std::true_type + { + using value_type = T; + }; + + template + class flat + { + using T::flat_type_not_defined; + }; + + template <> + class flat; + + template + class flat>; + + template + class flat>; + + template + auto get_view_type() + { + // if constexpr( is_flat_ptr::value ) { + // return get_view_type(); + if constexpr (reflect::is_struct) + { + using view_type = typename reflect::template proxy; + return (view_type*)(nullptr); + } + else if constexpr (std::is_trivially_copyable::value) + return (T*)(nullptr); + else + return (flat*)nullptr; + } + + template + using flat_view = std::remove_pointer_t())>; + + struct offset_ptr + { + uint32_t offset; + + template + auto get() const; + }; + + template + constexpr bool contains_offset_ptr(); + + template + struct tuple_contains_offset; + + template + constexpr uint32_t get_contains_offset_ptr() + { + if constexpr (sizeof...(Ts) == 0) + return contains_offset_ptr(); + else + { + return contains_offset_ptr() | get_contains_offset_ptr(); + } + } + + template + struct tuple_contains_offset> + { + static constexpr const auto value = get_contains_offset_ptr(); + }; + + /** + * Recursively checks the types for any field which requires dynamic allocation, + */ + template + constexpr bool contains_offset_ptr() + { + if constexpr (is_flat_ptr::value) + { + return true; + } + else if constexpr (is_std_tuple::value) + { + return tuple_contains_offset::value; + } + else if constexpr (is_std_variant::value) + { + return contains_offset_ptr::alts_as_tuple>(); + } + else if constexpr (std::is_same_v) + { + return true; + } + else if constexpr (is_std_vector::value) + { + return true; + } + else if constexpr (clio::reflect::is_struct) + { + bool is_flat = true; + clio::reflect::for_each([&](const clio::meta& ref, auto mptr) { + using member_type = std::decay_t; + is_flat &= not contains_offset_ptr(); + }); + return not is_flat; + } + else if constexpr (std::is_arithmetic_v) + { + return false; + } + else if constexpr (std::is_trivially_copyable::value) + { + return false; + } + else + { + T::contains_offset_ptr_not_defined; + } + } + + template + constexpr uint32_t flatpack_size() + { + if constexpr (is_flat_ptr::value) + { + return sizeof(offset_ptr); + } + else if constexpr (is_std_variant::value) + { + return 16; + } + else if constexpr (reflect::is_struct) + { + uint32_t size = 0; + reflect::for_each([&](const meta& ref, const auto& mptr) { + using member_type = decltype(result_of_member(mptr)); + if constexpr (contains_offset_ptr()) + { + size += sizeof(offset_ptr); } - - void init_variant( std::variant& v ) { - _init_variant( v ); + else + { + size += flatpack_size(); } - - template - void visit( Visitor&& v ) { - _visit_variant( std::forward(v) ); + }); + return size; + } + else if constexpr (std::is_same_v || is_std_vector::value) + { + return sizeof(offset_ptr); + } + else if constexpr (std::is_arithmetic_v) + { + return sizeof(T); + } + else if constexpr (std::is_trivially_copyable::value) + { + return sizeof(T); + } + else + { + T::flatpack_size_not_defined; + } + } + + template + struct get_tuple_offset; + + template + constexpr uint32_t get_offset() + { + static_assert(Idx < sizeof...(Ts) + 1, "index out of range"); + if constexpr (Idx == 0) + return 0; + else if constexpr (sizeof...(Ts) == 0) + if constexpr (contains_offset_ptr()) + return 4; + /// if contains ptr then 8 else flatpack_size... + else + return flatpack_size(); + // return get_flat_size( ((const First*)(nullptr)) ); + else + { + if constexpr (contains_offset_ptr()) + return get_offset() + 4; // flatpack_size(); + else + return get_offset() + flatpack_size(); + // return get_offset< Idx-1, Ts...>() + get_flat_size( ((const First*)(nullptr)) ); + } + } + + template + struct get_tuple_offset> + { + static constexpr const uint32_t value = get_offset(); + }; + + struct flat_view_proxy_impl + { + /** This method is called by the reflection library to get the field */ + template + constexpr auto get() + { + using class_type = decltype(clio::class_of_member(MemberPtr)); + using tuple_type = typename clio::reflect::struct_tuple_type; + using member_type = decltype(clio::result_of_member(MemberPtr)); + + constexpr uint32_t offset = clio::get_tuple_offset::value; + + char* out_ptr = reinterpret_cast(this) + offset; + + if constexpr (contains_offset_ptr()) + { + clio::offset_ptr* ptr = reinterpret_cast(out_ptr); + return ptr->get(); + } + else + { + return reinterpret_cast(out_ptr); + } + } + + template + constexpr const auto get() const + { + using class_type = decltype(clio::class_of_member(MemberPtr)); + using tuple_type = typename clio::reflect::struct_tuple_type; + using member_type = decltype(clio::result_of_member(MemberPtr)); + + constexpr uint32_t offset = clio::get_tuple_offset::value; + + auto out_ptr = reinterpret_cast(this) + offset; + + if constexpr (contains_offset_ptr()) + { + const clio::offset_ptr* ptr = reinterpret_cast(out_ptr); + return ptr->get(); + } + else + { + return reinterpret_cast(out_ptr); + } + } + }; + + /** + * A flat view of a flat pointer. + */ + template + class flat> + { + public: + auto get() { return reinterpret_cast())>(_data); } + + private: + uint32_t _size = 0; + char _data[]; + }; + + template <> + class flat + { + public: + uint32_t size() const { return _size; } + const char* c_str() const { return _size ? _data : (const char*)this; } + const char* data() const { return _size ? _data : nullptr; } + char* data() { return _size ? _data : nullptr; } + + operator std::string_view() const { return std::string_view(_data, _size); } + + template + friend S& operator<<(S& stream, const flat& str) + { + return stream << str.c_str(); + } + + private: + uint32_t _size = 0; + char _data[]; + }; + + template + class flat> + { + public: + uint64_t type = 0; + uint32_t flat_data = 0; + offset_ptr offset_data; + + flat() + { + static_assert(sizeof(flat) == 16); + static_assert(std::is_trivially_copyable::value); + } + + int64_t index_from_type() const { return get_index_from_type(); } + + void init_variant(std::variant& v) { _init_variant(v); } + + template + void visit(Visitor&& v) + { + _visit_variant(std::forward(v)); + } + + private: + template + int64_t get_index_from_type() const + { + if constexpr (sizeof...(Rest) == 0) + { + return get_type_hashname() != type; + } + else + { + if (get_type_hashname() == type) + return 0; + else + return 1 + get_index_from_type(); + } + } + + template + void _init_variant(std::variant& v) const + { + if (get_type_hashname() == type) + v = First(); + else if constexpr (sizeof...(Rest) > 0) + { + _init_variant(v); + } + } + template + void _visit_variant(Visitor&& v) const + { + if (get_type_hashname() == type) + { + if constexpr (not get_contains_offset_ptr() and flatpack_size() <= 8) + { + v(*((const flat_view*)&flat_data)); } - private: - template - int64_t get_index_from_type()const { - if constexpr( sizeof...(Rest) == 0 ) { - return get_type_hashname() != type; - } else { - if( get_type_hashname() == type ) - return 0; - else return 1 + get_index_from_type(); - } + else + { + v(*((const flat_view*)(((char*)(&offset_data)) + offset_data.offset))); } - - template - void _init_variant( std::variant& v )const { - if( get_type_hashname() == type ) - v = First(); - else if constexpr( sizeof...(Rest) > 0 ) { - _init_variant( v ); + } + else if constexpr (sizeof...(Rest) > 0) + { + _visit_variant(std::forward(v)); + } + } + }; + + /// T == value of the array elements + template + class flat> + { + public: + auto& operator[](uint32_t index) + { + if (index >= _size) + throw_error(stream_error::overrun); + /** in this case the data is a series of offset_ptr<> */ + if constexpr (std::is_same::value) + { + auto ptr_array = reinterpret_cast(_data); + return *reinterpret_cast*>(ptr_array[index].get>()); + } + else if constexpr (contains_offset_ptr()) + { + auto ptr_array = reinterpret_cast(_data); + return *reinterpret_cast*>(ptr_array[index].get>()); + } + else if constexpr (reflect::is_struct) + { /// the data is a series of packed T + const auto offset = index * flatpack_size(); + return *reinterpret_cast*>(&_data[offset]); + } + else if constexpr (std::is_trivially_copyable::value) + { + auto T_array = reinterpret_cast(_data); + return T_array[index]; + } + else + { + T::is_not_a_known_flat_type; + } + } + const auto& operator[](uint32_t index) const + { + if (index >= _size) + throw_error(stream_error::overrun); + + /** in this case the data is a series of offset_ptr<> */ + if constexpr (std::is_same::value) + { + auto ptr_array = reinterpret_cast(_data); + return *reinterpret_cast*>( + ptr_array[index].get>()); + } + else if constexpr (contains_offset_ptr()) + { + auto ptr_array = + reinterpret_cast(_data + sizeof(offset_ptr) * index); + const auto& r = + *reinterpret_cast*>(ptr_array->get>()); + return r; + } + else if constexpr (reflect::is_struct) + { /// the data is a series of packed T + const auto offset = index * flatpack_size(); + return *reinterpret_cast*>(&_data[offset]); + } + else if constexpr (std::is_trivially_copyable::value) + { + auto T_array = reinterpret_cast(_data); + return T_array[index]; + } + else + { + T::is_not_a_known_flat_type; + } + } + + uint32_t size() const { return _size; } + + private: + uint32_t _size = 0; + char _data[]; + }; + + template + auto offset_ptr::get() const + { + const auto ptr = ((char*)this) + offset; + if constexpr (is_flat_ptr::value) + { + return reinterpret_cast())>(ptr + 4); + } + else if constexpr (reflect::is_struct) + { + return reinterpret_cast*>(ptr); + } + else if constexpr (std::is_same_v) + { + return reinterpret_cast*>(ptr); + } + else if constexpr (is_std_vector::value) + { + return reinterpret_cast*>(ptr); + } + else + { + T::is_not_reflected_for_offset_ptr; + } + } + + /** + * Verifies that the type pointed at could be unpacked without a buffer overflow, + * which means that all offset pointers are in bounds. It does this by unpacking + * the stream without allocating any memory. + */ + struct validate_input_stream : public input_stream + { + using input_stream::input_stream; + }; + + template + void flatcheck(InputStream& stream) + { + if constexpr (is_flat_ptr::value) + { + uint32_t size; + stream.read(&size, sizeof(size)); + stream.skip(size); + } + else if constexpr (is_std_variant::value) + { + flat fv; + stream.read(&fv, sizeof(fv)); + + T temp; /// this could do memory allocation for certain types... but hopefully won't, + /// this is used to do std::visit in the next line... in theory we could do + /// a dispatcher that only deals in types and not values. + fv.init_variant(temp); + std::visit( + [&](auto& iv) { + using item_type = std::decay_t; + if constexpr (get_contains_offset_ptr()) + { + /// the stream.pos is at the END of reading the variant, which + /// should be the same as the end of flat::offset_data + InputStream in(stream.pos + fv.offset_data.offset - sizeof(offset_ptr), + stream.end); + flatunpack(iv, in); } - } - template - void _visit_variant( Visitor&& v )const { - if( get_type_hashname() == type ) { - if constexpr ( not get_contains_offset_ptr() and flatpack_size() <= 8 ) { - v( *((const flat_view*)&flat_data) ); - } else { - v( *((const flat_view*) ( ((char*)(&offset_data)) + offset_data.offset ) ) ); - } + else if constexpr (flatpack_size() <= 8) + { } - else if constexpr( sizeof...(Rest) > 0 ) { - _visit_variant( std::forward(v) ); + else + { + InputStream in(stream.pos + fv.offset_data.offset - sizeof(offset_ptr), + stream.end); + flatunpack(iv, in); } + }, + temp); + /// maybe deref pointer.. + } + else if constexpr (std::is_same_v) + { + uint32_t size; + stream.read(&size, sizeof(size)); + stream.skip(size); + } + else if constexpr (is_std_vector::value) + { + uint32_t size; + stream.read(&size, sizeof(size)); + if constexpr (contains_offset_ptr::value_type>()) + { + auto start = stream.pos; + stream.skip(size * sizeof(offset_ptr)); + + offset_ptr* ptr = (offset_ptr*)(start); + for (uint32_t i = 0; i < size; ++i) + { + InputStream in(((char*)ptr) + ptr->offset, stream.end); + flatcheck(in); } - }; - - - /// T == value of the array elements - template - class flat> { - public: - auto& operator[]( uint32_t index ) { - if( index >= _size ) - throw_error( stream_error::overrun ); - /** in this case the data is a series of offset_ptr<> */ - if constexpr( std::is_same::value ) { - auto ptr_array = reinterpret_cast(_data); - return *reinterpret_cast*>( ptr_array[index].get< std::vector >() ); - } else if constexpr( contains_offset_ptr() ) { - auto ptr_array = reinterpret_cast(_data); - return *reinterpret_cast*>( ptr_array[index].get< std::vector >() ); - } else if constexpr ( reflect::is_struct ) { /// the data is a series of packed T - const auto offset = index * flatpack_size(); - return *reinterpret_cast*>( &_data[offset] ); - } else if constexpr ( std::is_trivially_copyable::value ) { - auto T_array = reinterpret_cast(_data); - return T_array[index]; - } else { - T::is_not_a_known_flat_type; + } + else + { + stream.skip(size * flatpack_size()); + } + } + else if constexpr (reflect::is_struct) + { + if constexpr (contains_offset_ptr()) + { + reflect::for_each([&](const meta& ref, const auto& mptr) { + using member_type = decltype(result_of_member(mptr)); + + if constexpr (contains_offset_ptr()) + { + offset_ptr ptr; + stream.read(&ptr, sizeof(ptr)); + + InputStream substream(stream.pos + ptr.offset - sizeof(ptr), stream.end); + flatcheck(substream); + } + else + { + stream.skip(flatpack_size()); + } + }); + } + else + { + stream.skip(flatpack_size()); + } + } + else if constexpr (std::is_trivially_copyable::value) + { + stream.skip(sizeof(T)); + } + else + { + T::is_not_defined; + } + } + + template + void flatunpack(T& v, S& stream) + { + if constexpr (is_flat_ptr::value) + { + uint32_t size; + stream.read(&size, sizeof(size)); + v.reset(size); + stream.read(v.data(), size); + } + else if constexpr (is_std_variant::value) + { + flat fv; + stream.read(&fv, sizeof(fv)); + fv.init_variant(v); + std::visit( + [&](auto& iv) { + using item_type = std::decay_t; + if constexpr (get_contains_offset_ptr()) + { + /// the stream.pos is at the END of reading the variant, which + /// should be the same as the end of flat::offset_data + input_stream in(stream.pos + fv.offset_data.offset - sizeof(offset_ptr), + stream.end); + flatunpack(iv, in); } - } - const auto& operator[]( uint32_t index )const { - if( index >= _size ) - throw_error( stream_error::overrun ); - - /** in this case the data is a series of offset_ptr<> */ - if constexpr( std::is_same::value ) { - auto ptr_array = reinterpret_cast(_data); - return *reinterpret_cast*>( ptr_array[index].get< std::vector >() ); - }else if constexpr( contains_offset_ptr() ) { - auto ptr_array = reinterpret_cast(_data+sizeof(offset_ptr)*index); - const auto& r = *reinterpret_cast*>( ptr_array->get< std::vector >() ); - return r; - } else if constexpr ( reflect::is_struct ) { /// the data is a series of packed T - const auto offset = index * flatpack_size(); - return *reinterpret_cast*>( &_data[offset] ); - } else if constexpr ( std::is_trivially_copyable::value ) { - auto T_array = reinterpret_cast(_data); - return T_array[index]; - } else { - T::is_not_a_known_flat_type; + else if constexpr (flatpack_size() <= 8) + { + input_stream st((const char*)&fv.flat_data, 8); + flatunpack(iv, st); } - } - - - uint32_t size()const { return _size; } - - private: - uint32_t _size = 0; - char _data[]; - }; - - template - auto offset_ptr::get()const { - const auto ptr = ((char*)this)+offset; - if constexpr( is_flat_ptr::value ) { - return reinterpret_cast< decltype(get_view_type()) >(ptr+4); - } else if constexpr( reflect::is_struct ) { - return reinterpret_cast*>(ptr); - } else if constexpr( std::is_same_v ) { - return reinterpret_cast< flat* >(ptr); - } else if constexpr( is_std_vector::value ) { - return reinterpret_cast< flat* >(ptr); - } else { - T::is_not_reflected_for_offset_ptr; - } - } - - - /** - * Verifies that the type pointed at could be unpacked without a buffer overflow, - * which means that all offset pointers are in bounds. It does this by unpacking - * the stream without allocating any memory. - */ - struct validate_input_stream : public input_stream { - using input_stream::input_stream; - }; - - template - void flatcheck( InputStream& stream ) { - if constexpr ( is_flat_ptr::value ) { - uint32_t size; - stream.read( &size, sizeof(size) ); - stream.skip( size ); - } else if constexpr ( is_std_variant::value ) { - flat fv; - stream.read( &fv, sizeof(fv) ); - - T temp; /// this could do memory allocation for certain types... but hopefully won't, - /// this is used to do std::visit in the next line... in theory we could do - /// a dispatcher that only deals in types and not values. - fv.init_variant( temp ); - std::visit( [&]( auto& iv ){ - using item_type = std::decay_t< decltype(iv) >; - if constexpr ( get_contains_offset_ptr() ) { - /// the stream.pos is at the END of reading the variant, which - /// should be the same as the end of flat::offset_data - InputStream in( stream.pos + fv.offset_data.offset-sizeof(offset_ptr), stream.end ); - flatunpack( iv, in ); - } else if constexpr ( flatpack_size() <= 8 ){ - } else { - InputStream in( stream.pos + fv.offset_data.offset-sizeof(offset_ptr), stream.end ); - flatunpack( iv, in ); - } - }, temp); - /// maybe deref pointer.. - } else if constexpr ( std::is_same_v ) { - uint32_t size; - stream.read( &size, sizeof(size) ); - stream.skip( size ); - } else if constexpr ( is_std_vector::value ) { - uint32_t size; - stream.read( &size, sizeof(size) ); - if constexpr( contains_offset_ptr< typename is_std_vector::value_type >() ) { - auto start = stream.pos; - stream.skip( size * sizeof( offset_ptr ) ); - - offset_ptr* ptr = (offset_ptr*)(start); - for( uint32_t i = 0; i < size; ++i ) { - InputStream in( ((char*)ptr) + ptr->offset, stream.end ); - flatcheck(in); + else + { + input_stream in(stream.pos + fv.offset_data.offset - sizeof(offset_ptr), + stream.end); + flatunpack(iv, in); } - } else { - stream.skip( size * flatpack_size() ); + }, + v); + } + else if constexpr (std::is_same_v) + { + uint32_t size; + stream.read(&size, sizeof(size)); + v.resize(size); + stream.read(v.data(), size); + stream.skip(1); // null + } + else if constexpr (is_std_vector::value) + { + uint32_t size; + stream.read(&size, sizeof(size)); + v.resize(size); + + if constexpr (contains_offset_ptr::value_type>()) + { + for (auto& item : v) + { + offset_ptr ptr; + stream.read(&ptr, sizeof(ptr)); + + /// TODO: we don't know the size of the buffer here...is it safe? + input_stream in(stream.pos + ptr.offset - sizeof(ptr), stream.end); + flatunpack(item, in); } - } else if constexpr ( reflect::is_struct ) { - if constexpr( contains_offset_ptr< T >() ) { - reflect::for_each( [&]( const meta& ref, const auto& mptr ){ - using member_type = decltype(result_of_member(mptr)); - - if constexpr ( contains_offset_ptr() ) { - offset_ptr ptr; - stream.read(&ptr,sizeof(ptr)); - - InputStream substream( stream.pos+ptr.offset-sizeof(ptr), stream.end ); - flatcheck( substream ); - } else { - stream.skip( flatpack_size() ); - } - }); - } else { - stream.skip( flatpack_size() ); + } + else + { + for (auto& item : v) + { + flatunpack(item, stream); } - } else if constexpr( std::is_trivially_copyable::value ) { - stream.skip( sizeof(T) ); - } else { - T::is_not_defined; - } - } - - template - void flatunpack( T& v, S& stream ) { - if constexpr ( is_flat_ptr::value ) { - uint32_t size; - stream.read( &size, sizeof(size) ); - v.reset(size); - stream.read( v.data(), size ) ; - } else if constexpr ( is_std_variant::value ) { - flat fv; - stream.read( &fv, sizeof(fv) ); - fv.init_variant( v ); - std::visit( [&]( auto& iv ){ - using item_type = std::decay_t< decltype(iv) >; - if constexpr ( get_contains_offset_ptr() ) { - /// the stream.pos is at the END of reading the variant, which - /// should be the same as the end of flat::offset_data - input_stream in( stream.pos + fv.offset_data.offset-sizeof(offset_ptr), stream.end ); - flatunpack( iv, in ); - } else if constexpr ( flatpack_size() <= 8 ){ - input_stream st( (const char*)&fv.flat_data, 8 ); - flatunpack( iv, st ); - } else { - input_stream in( stream.pos + fv.offset_data.offset-sizeof(offset_ptr), stream.end ); - flatunpack( iv, in ); - } - }, v); - } else if constexpr ( std::is_same_v ) { - uint32_t size; - stream.read( &size, sizeof(size) ); - v.resize(size); - stream.read( v.data(), size ) ; - stream.skip(1); // null - } else if constexpr ( is_std_vector::value ) { - uint32_t size; - stream.read( &size, sizeof(size) ); - v.resize( size ); - - if constexpr( contains_offset_ptr< typename is_std_vector::value_type >() ) { - for( auto& item : v ) { - offset_ptr ptr; - stream.read(&ptr,sizeof(ptr)); - - /// TODO: we don't know the size of the buffer here...is it safe? - input_stream in( stream.pos + ptr.offset-sizeof(ptr), stream.end ); - flatunpack( item, in ); - } - } else { - for( auto& item : v ) { - flatunpack( item, stream ); - } + } + } + else if constexpr (reflect::is_struct) + { + reflect::for_each([&](const meta& ref, const auto& mptr) { + auto& member = v.*mptr; + using member_type = decltype(result_of_member(mptr)); + + if constexpr (contains_offset_ptr()) + { + offset_ptr ptr; + stream.read(&ptr, sizeof(ptr)); + + input_stream substream(stream.pos + ptr.offset - sizeof(ptr), stream.end); + flatunpack(member, substream); } - } else if constexpr ( reflect::is_struct ) { - reflect::for_each( [&]( const meta& ref, const auto& mptr ){ - auto& member = v.*mptr; - using member_type = decltype(result_of_member(mptr)); - - if constexpr ( contains_offset_ptr() ) { - offset_ptr ptr; - stream.read(&ptr,sizeof(ptr)); - - input_stream substream( stream.pos+ptr.offset-sizeof(ptr), stream.end ); - flatunpack( member, substream ); - } else { - flatunpack( member, stream ); - } - }); - } else if constexpr( std::is_trivially_copyable::value ) { - stream.read( (char*)&v, sizeof(v) ); - } else { - T::is_not_defined; - } - } - - template - uint32_t flatpack( const T& v, S& stream ) { - uint32_t alloc_pos = 0; - uint32_t cur_pos = 0; - - if constexpr ( is_flat_ptr::value ) { - uint32_t size = v.size(); - stream.write( &size, sizeof(size) ); - cur_pos += sizeof(size); - - if( size ) { - stream.write( v.data(), v.size() ); + else + { + flatunpack(member, stream); } - } else if constexpr ( is_std_variant::value ) { - alloc_pos = flatpack_size(); - std::visit( [&]( const auto& iv ){ - using item_type = std::decay_t< decltype(iv) >; + }); + } + else if constexpr (std::is_trivially_copyable::value) + { + stream.read((char*)&v, sizeof(v)); + } + else + { + T::is_not_defined; + } + } + + template + uint32_t flatpack(const T& v, S& stream) + { + uint32_t alloc_pos = 0; + uint32_t cur_pos = 0; + + if constexpr (is_flat_ptr::value) + { + uint32_t size = v.size(); + stream.write(&size, sizeof(size)); + cur_pos += sizeof(size); + + if (size) + { + stream.write(v.data(), v.size()); + } + } + else if constexpr (is_std_variant::value) + { + alloc_pos = flatpack_size(); + std::visit( + [&](const auto& iv) { + using item_type = std::decay_t; flat fv; fv.type = get_type_hashname(); - if constexpr ( not get_contains_offset_ptr() and flatpack_size() <= 8 ) { - fixed_buf_stream st( (char*)&fv.flat_data, 8 ); - flatpack( iv, st ); - - static_assert( sizeof(fv) == 16 ); - stream.write( &fv, sizeof(fv) ); - } else { - fv.offset_data.offset = (alloc_pos - (cur_pos+12)); //+8 because we haven't written the type yet / pad - size_stream size_str; - flatpack( iv, size_str ); - alloc_pos += size_str.size; - - static_assert( sizeof(fv) == 16 ); - stream.write( &fv, sizeof(fv) ); - - if constexpr( std::is_same_v ) { - stream.skip( size_str.size ); /// we already calculated this above - } - else { - /// now pack the member into the allocated spot - fixed_buf_stream substream( stream.pos+fv.offset_data.offset-sizeof(fv.offset_data), - size_str.size ); - if( substream.end > stream.end ) - throw_error( stream_error::overrun ); - flatpack( iv, substream ); - } - } - }, v ); - } else if constexpr ( std::is_same_v ) { - uint32_t size = v.size(); - stream.write( &size, sizeof(size) ); - cur_pos += sizeof(size); - - if( size ) { - stream.write( v.data(), v.size() ); - stream.write( "\0", 1 ); /// null term - cur_pos += v.size() + 1; + if constexpr (not get_contains_offset_ptr() and + flatpack_size() <= 8) + { + fixed_buf_stream st((char*)&fv.flat_data, 8); + flatpack(iv, st); + + static_assert(sizeof(fv) == 16); + stream.write(&fv, sizeof(fv)); } - } else if constexpr ( is_std_vector::value ) { - - if constexpr( contains_offset_ptr< typename T::value_type >() ) { - - uint32_t size = v.size(); - stream.write( &size, sizeof(size) ); - cur_pos += sizeof(size); - alloc_pos += sizeof(size) + size * sizeof(offset_ptr); - - - for( const auto& member: v ) { - - if constexpr ( std::is_same_v || is_std_vector::value ) { - if( member.size() == 0 ) { - offset_ptr ptr = { .offset = 0 }; - stream.write( &ptr, sizeof(ptr) ); - cur_pos += sizeof(ptr); - continue; - } + else + { + fv.offset_data.offset = + (alloc_pos - + (cur_pos + 12)); //+8 because we haven't written the type yet / pad + size_stream size_str; + flatpack(iv, size_str); + alloc_pos += size_str.size; + + static_assert(sizeof(fv) == 16); + stream.write(&fv, sizeof(fv)); + + if constexpr (std::is_same_v) + { + stream.skip(size_str.size); /// we already calculated this above + } + else + { + /// now pack the member into the allocated spot + fixed_buf_stream substream( + stream.pos + fv.offset_data.offset - sizeof(fv.offset_data), + size_str.size); + if (substream.end > stream.end) + throw_error(stream_error::overrun); + flatpack(iv, substream); } - - size_stream size_str; - flatpack( member, size_str ); - - offset_ptr ptr = { .offset = (alloc_pos - cur_pos) }; - - alloc_pos += size_str.size; - - stream.write( &ptr, sizeof(ptr) ); - - cur_pos += sizeof(ptr); - - //? cur_pos += size_str.size; - if constexpr( std::is_same_v ) { - stream.skip( size_str.size ); /// we already calculated this above - } - else { - /// now pack the member into the allocated spot - fixed_buf_stream substream( stream.pos+ptr.offset-sizeof(ptr), - size_str.size ); //ptr.size ); - if( substream.end > stream.end ) - throw_error( stream_error::overrun ); - flatpack( member, substream ); - } - } - } - else { - // std::cout << "vector type, T, is flat types: " << boost::core::demangle(typeid(typename T::value_type).name()) <<"\n"; - uint32_t size = v.size(); - stream.write( &size, sizeof(size) ); - cur_pos += sizeof(size); - for( const auto& item : v ) { - cur_pos += flatpack( item, stream ); } - } - } else if constexpr ( reflect::is_struct ) { - alloc_pos = flatpack_size();//::struct_tuple_type>::value; - reflect::for_each( [&]( const meta& ref, const auto& mptr ){ - - const auto& member = v.*mptr; - - using member_type = decltype(result_of_member(mptr)); - if constexpr ( contains_offset_ptr() ) { + }, + v); + } + else if constexpr (std::is_same_v) + { + uint32_t size = v.size(); + stream.write(&size, sizeof(size)); + cur_pos += sizeof(size); + + if (size) + { + stream.write(v.data(), v.size()); + stream.write("\0", 1); /// null term + cur_pos += v.size() + 1; + } + } + else if constexpr (is_std_vector::value) + { + if constexpr (contains_offset_ptr()) + { + uint32_t size = v.size(); + stream.write(&size, sizeof(size)); + cur_pos += sizeof(size); + alloc_pos += sizeof(size) + size * sizeof(offset_ptr); + + for (const auto& member : v) + { + if constexpr (std::is_same_v || + is_std_vector::value) + { + if (member.size() == 0) + { + offset_ptr ptr = {.offset = 0}; + stream.write(&ptr, sizeof(ptr)); + cur_pos += sizeof(ptr); + continue; + } + } - if constexpr ( std::is_same_v || is_std_vector::value ) { - if( member.size() == 0 ) { - offset_ptr ptr = { .offset = 0 }; - stream.write( &ptr, sizeof(ptr) ); - cur_pos += sizeof(ptr); + size_stream size_str; + flatpack(member, size_str); - return; - } - } + offset_ptr ptr = {.offset = (alloc_pos - cur_pos)}; - size_stream size_str; - flatpack( member, size_str ); + alloc_pos += size_str.size; - offset_ptr ptr = { .offset = alloc_pos - cur_pos }; + stream.write(&ptr, sizeof(ptr)); - if( ptr.offset == 0 ) exit(-1); + cur_pos += sizeof(ptr); + //? cur_pos += size_str.size; + if constexpr (std::is_same_v) + { + stream.skip(size_str.size); /// we already calculated this above + } + else + { + /// now pack the member into the allocated spot + fixed_buf_stream substream(stream.pos + ptr.offset - sizeof(ptr), + size_str.size); // ptr.size ); + if (substream.end > stream.end) + throw_error(stream_error::overrun); + flatpack(member, substream); + } + } + } + else + { + // std::cout << "vector type, T, is flat types: " << boost::core::demangle(typeid(typename T::value_type).name()) <<"\n"; + uint32_t size = v.size(); + stream.write(&size, sizeof(size)); + cur_pos += sizeof(size); + for (const auto& item : v) + { + cur_pos += flatpack(item, stream); + } + } + } + else if constexpr (reflect::is_struct) + { + alloc_pos = flatpack_size(); //::struct_tuple_type>::value; + reflect::for_each([&](const meta& ref, const auto& mptr) { + const auto& member = v.*mptr; + + using member_type = decltype(result_of_member(mptr)); + if constexpr (contains_offset_ptr()) + { + if constexpr (std::is_same_v || + is_std_vector::value) + { + if (member.size() == 0) + { + offset_ptr ptr = {.offset = 0}; + stream.write(&ptr, sizeof(ptr)); + cur_pos += sizeof(ptr); + + return; + } + } - alloc_pos += size_str.size; - stream.write( &ptr, sizeof(ptr) ); - cur_pos += sizeof(ptr); + size_stream size_str; + flatpack(member, size_str); + offset_ptr ptr = {.offset = alloc_pos - cur_pos}; - if constexpr( std::is_same_v ) { - stream.skip( size_str.size ); /// we already calculated this above - } - else { - /// now pack the member into the allocated spot - fixed_buf_stream substream( stream.pos+ptr.offset-sizeof(ptr), size_str.size ); + if (ptr.offset == 0) + exit(-1); - if( substream.end > stream.end ) { - throw_error( stream_error::overrun ); - } - flatpack( member, substream ); - } - } else { - cur_pos += flatpack( member, stream ); - } - }); - } else if constexpr( std::is_trivially_copyable::value ) { - stream.write( &v, sizeof(v) ); - cur_pos += sizeof(v); - } else { - T::flatpack_is_not_defined; - } - return cur_pos; - } - - - /** - * Behaves like a shared_ptr, copies will all point to the same flat data array. - */ - template - class flat_ptr { - public: - typedef T value_type; - - flat_ptr( const T& from ) { - clio::size_stream ss; - clio::flatpack( from, ss ); - _size = ss.size; - if( _size ) { - _data = std::shared_ptr( new char[_size], [](char* c){ delete[] c; } ); - fixed_buf_stream out( _data.get(), _size ); - clio::flatpack( from, out ); - } - } - flat_ptr() {}; - operator bool()const { return _size; }; + alloc_pos += size_str.size; + stream.write(&ptr, sizeof(ptr)); + cur_pos += sizeof(ptr); - auto operator->() { - return reinterpret_cast*>(_data.get()); - } - auto operator->()const { - return reinterpret_cast*>(_data.get()); - } - const char* data()const { return _data.get(); } - char* data() { return _data.get(); } - size_t size()const { return _size; } - - void reset( size_t s = 0 ) { - _size = s; - if( _size ) - _data = std::shared_ptr( new char[_size], [](char* c){ delete[] c; } ); - else _data.reset(); + if constexpr (std::is_same_v) + { + stream.skip(size_str.size); /// we already calculated this above + } + else + { + /// now pack the member into the allocated spot + fixed_buf_stream substream(stream.pos + ptr.offset - sizeof(ptr), size_str.size); + + if (substream.end > stream.end) + { + throw_error(stream_error::overrun); + } + flatpack(member, substream); + } } - - operator T()const { - T tmp; - input_stream in(_data.get(), _size ); - clio::flatunpack( tmp, in ); - return tmp; + else + { + cur_pos += flatpack(member, stream); } - - private: - std::shared_ptr _data; - size_t _size = 0; - }; - -} /// namespace clio + }); + } + else if constexpr (std::is_trivially_copyable::value) + { + stream.write(&v, sizeof(v)); + cur_pos += sizeof(v); + } + else + { + T::flatpack_is_not_defined; + } + return cur_pos; + } + + /** + * Behaves like a shared_ptr, copies will all point to the same flat data array. + */ + template + class flat_ptr + { + public: + typedef T value_type; + + flat_ptr(const T& from) + { + clio::size_stream ss; + clio::flatpack(from, ss); + _size = ss.size; + if (_size) + { + _data = std::shared_ptr(new char[_size], [](char* c) { delete[] c; }); + fixed_buf_stream out(_data.get(), _size); + clio::flatpack(from, out); + } + } + flat_ptr(){}; + operator bool() const { return _size; }; + + auto operator->() { return reinterpret_cast*>(_data.get()); } + auto operator->() const { return reinterpret_cast*>(_data.get()); } + const char* data() const { return _data.get(); } + char* data() { return _data.get(); } + size_t size() const { return _size; } + + void reset(size_t s = 0) + { + _size = s; + if (_size) + _data = std::shared_ptr(new char[_size], [](char* c) { delete[] c; }); + else + _data.reset(); + } + + operator T() const + { + T tmp; + input_stream in(_data.get(), _size); + clio::flatunpack(tmp, in); + return tmp; + } + + private: + std::shared_ptr _data; + size_t _size = 0; + }; + +} // namespace clio diff --git a/libraries/clio/include/clio/fpconv.h b/libraries/clio/include/clio/fpconv.h index 4c9962823..d6620db80 100644 --- a/libraries/clio/include/clio/fpconv.h +++ b/libraries/clio/include/clio/fpconv.h @@ -8,329 +8,369 @@ #include -#define fracmask 0x000FFFFFFFFFFFFFU -#define expmask 0x7FF0000000000000U +#define fracmask 0x000FFFFFFFFFFFFFU +#define expmask 0x7FF0000000000000U #define hiddenbit 0x0010000000000000U -#define signmask 0x8000000000000000U -#define expbias (1023 + 52) +#define signmask 0x8000000000000000U +#define expbias (1023 + 52) #define absv(n) ((n) < 0 ? -(n) : (n)) #define minv(a, b) ((a) < (b) ? (a) : (b)) -static constexpr uint64_t tens[] = { - 10000000000000000000U, 1000000000000000000U, 100000000000000000U, - 10000000000000000U, 1000000000000000U, 100000000000000U, - 10000000000000U, 1000000000000U, 100000000000U, - 10000000000U, 1000000000U, 100000000U, - 10000000U, 1000000U, 100000U, - 10000U, 1000U, 100U, - 10U, 1U -}; +static constexpr uint64_t tens[] = {10000000000000000000U, + 1000000000000000000U, + 100000000000000000U, + 10000000000000000U, + 1000000000000000U, + 100000000000000U, + 10000000000000U, + 1000000000000U, + 100000000000U, + 10000000000U, + 1000000000U, + 100000000U, + 10000000U, + 1000000U, + 100000U, + 10000U, + 1000U, + 100U, + 10U, + 1U}; static inline uint64_t get_dbits(double d) { - union { - double dbl; - uint64_t i; - } dbl_bits = { d }; + union + { + double dbl; + uint64_t i; + } dbl_bits = {d}; - return dbl_bits.i; + return dbl_bits.i; } static inline Fp build_fp(double d) { - uint64_t bits = get_dbits(d); - - Fp fp; - fp.frac = bits & fracmask; - fp.exp = (bits & expmask) >> 52; - - if(fp.exp) { - fp.frac += hiddenbit; - fp.exp -= expbias; - - } else { - fp.exp = -expbias + 1; - } - - return fp; + uint64_t bits = get_dbits(d); + + Fp fp; + fp.frac = bits & fracmask; + fp.exp = (bits & expmask) >> 52; + + if (fp.exp) + { + fp.frac += hiddenbit; + fp.exp -= expbias; + } + else + { + fp.exp = -expbias + 1; + } + + return fp; } static inline void normalize(Fp* fp) { - while ((fp->frac & hiddenbit) == 0) { - fp->frac <<= 1; - fp->exp--; - } - - int shift = 64 - 52 - 1; - fp->frac <<= shift; - fp->exp -= shift; + while ((fp->frac & hiddenbit) == 0) + { + fp->frac <<= 1; + fp->exp--; + } + + int shift = 64 - 52 - 1; + fp->frac <<= shift; + fp->exp -= shift; } static inline void get_normalized_boundaries(Fp* fp, Fp* lower, Fp* upper) { - upper->frac = (fp->frac << 1) + 1; - upper->exp = fp->exp - 1; + upper->frac = (fp->frac << 1) + 1; + upper->exp = fp->exp - 1; - while ((upper->frac & (hiddenbit << 1)) == 0) { - upper->frac <<= 1; - upper->exp--; - } + while ((upper->frac & (hiddenbit << 1)) == 0) + { + upper->frac <<= 1; + upper->exp--; + } - int u_shift = 64 - 52 - 2; + int u_shift = 64 - 52 - 2; - upper->frac <<= u_shift; - upper->exp = upper->exp - u_shift; + upper->frac <<= u_shift; + upper->exp = upper->exp - u_shift; + int l_shift = fp->frac == hiddenbit ? 2 : 1; - int l_shift = fp->frac == hiddenbit ? 2 : 1; + lower->frac = (fp->frac << l_shift) - 1; + lower->exp = fp->exp - l_shift; - lower->frac = (fp->frac << l_shift) - 1; - lower->exp = fp->exp - l_shift; - - - lower->frac <<= lower->exp - upper->exp; - lower->exp = upper->exp; + lower->frac <<= lower->exp - upper->exp; + lower->exp = upper->exp; } static inline Fp multiply(Fp* a, Fp* b) { - const uint64_t lomask = 0x00000000FFFFFFFF; + const uint64_t lomask = 0x00000000FFFFFFFF; - uint64_t ah_bl = (a->frac >> 32) * (b->frac & lomask); - uint64_t al_bh = (a->frac & lomask) * (b->frac >> 32); - uint64_t al_bl = (a->frac & lomask) * (b->frac & lomask); - uint64_t ah_bh = (a->frac >> 32) * (b->frac >> 32); + uint64_t ah_bl = (a->frac >> 32) * (b->frac & lomask); + uint64_t al_bh = (a->frac & lomask) * (b->frac >> 32); + uint64_t al_bl = (a->frac & lomask) * (b->frac & lomask); + uint64_t ah_bh = (a->frac >> 32) * (b->frac >> 32); - uint64_t tmp = (ah_bl & lomask) + (al_bh & lomask) + (al_bl >> 32); - /* round up */ - tmp += 1U << 31; + uint64_t tmp = (ah_bl & lomask) + (al_bh & lomask) + (al_bl >> 32); + /* round up */ + tmp += 1U << 31; - Fp fp = { - ah_bh + (ah_bl >> 32) + (al_bh >> 32) + (tmp >> 32), - a->exp + b->exp + 64 - }; + Fp fp = {ah_bh + (ah_bl >> 32) + (al_bh >> 32) + (tmp >> 32), a->exp + b->exp + 64}; - return fp; + return fp; } -static inline void round_digit(char* digits, int ndigits, uint64_t delta, uint64_t rem, uint64_t kappa, uint64_t frac) +static inline void round_digit(char* digits, + int ndigits, + uint64_t delta, + uint64_t rem, + uint64_t kappa, + uint64_t frac) { - while (rem < frac && delta - rem >= kappa && - (rem + kappa < frac || frac - rem > rem + kappa - frac)) { - - digits[ndigits - 1]--; - rem += kappa; - } + while (rem < frac && delta - rem >= kappa && + (rem + kappa < frac || frac - rem > rem + kappa - frac)) + { + digits[ndigits - 1]--; + rem += kappa; + } } static inline int generate_digits(Fp* fp, Fp* upper, Fp* lower, char* digits, int* K) { - uint64_t wfrac = upper->frac - fp->frac; - uint64_t delta = upper->frac - lower->frac; - - Fp one; - one.frac = 1ULL << -upper->exp; - one.exp = upper->exp; - - uint64_t part1 = upper->frac >> -one.exp; - uint64_t part2 = upper->frac & (one.frac - 1); - - int idx = 0, kappa = 10; - const uint64_t* divp; - /* 1000000000 */ - for(divp = tens + 10; kappa > 0; divp++) { - - uint64_t div = *divp; - unsigned digit = part1 / div; - - if (digit || idx) { - digits[idx++] = digit + '0'; - } - - part1 -= digit * div; - kappa--; - - uint64_t tmp = (part1 <<-one.exp) + part2; - if (tmp <= delta) { - *K += kappa; - round_digit(digits, idx, delta, tmp, div << -one.exp, wfrac); - - return idx; - } - } - - /* 10 */ - const uint64_t* unit = tens + 18; - - while(true) { - part2 *= 10; - delta *= 10; - kappa--; - - unsigned digit = part2 >> -one.exp; - if (digit || idx) { - digits[idx++] = digit + '0'; - } - - part2 &= one.frac - 1; - if (part2 < delta) { - *K += kappa; - round_digit(digits, idx, delta, part2, one.frac, wfrac * *unit); - - return idx; - } - - unit--; - } + uint64_t wfrac = upper->frac - fp->frac; + uint64_t delta = upper->frac - lower->frac; + + Fp one; + one.frac = 1ULL << -upper->exp; + one.exp = upper->exp; + + uint64_t part1 = upper->frac >> -one.exp; + uint64_t part2 = upper->frac & (one.frac - 1); + + int idx = 0, kappa = 10; + const uint64_t* divp; + /* 1000000000 */ + for (divp = tens + 10; kappa > 0; divp++) + { + uint64_t div = *divp; + unsigned digit = part1 / div; + + if (digit || idx) + { + digits[idx++] = digit + '0'; + } + + part1 -= digit * div; + kappa--; + + uint64_t tmp = (part1 << -one.exp) + part2; + if (tmp <= delta) + { + *K += kappa; + round_digit(digits, idx, delta, tmp, div << -one.exp, wfrac); + + return idx; + } + } + + /* 10 */ + const uint64_t* unit = tens + 18; + + while (true) + { + part2 *= 10; + delta *= 10; + kappa--; + + unsigned digit = part2 >> -one.exp; + if (digit || idx) + { + digits[idx++] = digit + '0'; + } + + part2 &= one.frac - 1; + if (part2 < delta) + { + *K += kappa; + round_digit(digits, idx, delta, part2, one.frac, wfrac * *unit); + + return idx; + } + + unit--; + } } static inline int grisu2(double d, char* digits, int* K) { - Fp w = build_fp(d); + Fp w = build_fp(d); - Fp lower, upper; - get_normalized_boundaries(&w, &lower, &upper); + Fp lower, upper; + get_normalized_boundaries(&w, &lower, &upper); - normalize(&w); + normalize(&w); - int k; - Fp cp = find_cachedpow10(upper.exp, &k); + int k; + Fp cp = find_cachedpow10(upper.exp, &k); - w = multiply(&w, &cp); - upper = multiply(&upper, &cp); - lower = multiply(&lower, &cp); + w = multiply(&w, &cp); + upper = multiply(&upper, &cp); + lower = multiply(&lower, &cp); - lower.frac++; - upper.frac--; + lower.frac++; + upper.frac--; - *K = -k; + *K = -k; - return generate_digits(&w, &upper, &lower, digits, K); + return generate_digits(&w, &upper, &lower, digits, K); } static inline int emit_digits(char* digits, int ndigits, char* dest, int K, bool neg) { - int exp = absv(K + ndigits - 1); - - /* write plain integer */ - if(K >= 0 && (exp < (ndigits + 7))) { - memcpy(dest, digits, ndigits); - memset(dest + ndigits, '0', K); - - return ndigits + K; - } - - /* write decimal w/o scientific notation */ - if(K < 0 && (K > -7 || exp < 4)) { - int offset = ndigits - absv(K); - /* fp < 1.0 -> write leading zero */ - if(offset <= 0) { - offset = -offset; - dest[0] = '0'; - dest[1] = '.'; - memset(dest + 2, '0', offset); - memcpy(dest + offset + 2, digits, ndigits); - - return ndigits + 2 + offset; - - /* fp > 1.0 */ - } else { - memcpy(dest, digits, offset); - dest[offset] = '.'; - memcpy(dest + offset + 1, digits + offset, ndigits - offset); - - return ndigits + 1; - } - } - - /* write decimal w/ scientific notation */ - ndigits = minv(ndigits, 18 - neg); - - int idx = 0; - dest[idx++] = digits[0]; - - if(ndigits > 1) { - dest[idx++] = '.'; - memcpy(dest + idx, digits + 1, ndigits - 1); - idx += ndigits - 1; - } - - dest[idx++] = 'e'; - - char sign = K + ndigits - 1 < 0 ? '-' : '+'; - dest[idx++] = sign; - - int cent = 0; - - if(exp > 99) { - cent = exp / 100; - dest[idx++] = cent + '0'; - exp -= cent * 100; - } - if(exp > 9) { - int dec = exp / 10; - dest[idx++] = dec + '0'; - exp -= dec * 10; - - } else if(cent) { - dest[idx++] = '0'; - } - - dest[idx++] = exp % 10 + '0'; - - return idx; + int exp = absv(K + ndigits - 1); + + /* write plain integer */ + if (K >= 0 && (exp < (ndigits + 7))) + { + memcpy(dest, digits, ndigits); + memset(dest + ndigits, '0', K); + + return ndigits + K; + } + + /* write decimal w/o scientific notation */ + if (K < 0 && (K > -7 || exp < 4)) + { + int offset = ndigits - absv(K); + /* fp < 1.0 -> write leading zero */ + if (offset <= 0) + { + offset = -offset; + dest[0] = '0'; + dest[1] = '.'; + memset(dest + 2, '0', offset); + memcpy(dest + offset + 2, digits, ndigits); + + return ndigits + 2 + offset; + + /* fp > 1.0 */ + } + else + { + memcpy(dest, digits, offset); + dest[offset] = '.'; + memcpy(dest + offset + 1, digits + offset, ndigits - offset); + + return ndigits + 1; + } + } + + /* write decimal w/ scientific notation */ + ndigits = minv(ndigits, 18 - neg); + + int idx = 0; + dest[idx++] = digits[0]; + + if (ndigits > 1) + { + dest[idx++] = '.'; + memcpy(dest + idx, digits + 1, ndigits - 1); + idx += ndigits - 1; + } + + dest[idx++] = 'e'; + + char sign = K + ndigits - 1 < 0 ? '-' : '+'; + dest[idx++] = sign; + + int cent = 0; + + if (exp > 99) + { + cent = exp / 100; + dest[idx++] = cent + '0'; + exp -= cent * 100; + } + if (exp > 9) + { + int dec = exp / 10; + dest[idx++] = dec + '0'; + exp -= dec * 10; + } + else if (cent) + { + dest[idx++] = '0'; + } + + dest[idx++] = exp % 10 + '0'; + + return idx; } static inline int filter_special(double fp, char* dest) { - if(fp == 0.0) { - dest[0] = '0'; - return 1; - } - - uint64_t bits = get_dbits(fp); - - bool nan = (bits & expmask) == expmask; - - if(!nan) { - return 0; - } - - if(bits & fracmask) { - dest[0] = 'n'; dest[1] = 'a'; dest[2] = 'n'; - - } else { - dest[0] = 'i'; dest[1] = 'n'; dest[2] = 'f'; - } - - return 3; + if (fp == 0.0) + { + dest[0] = '0'; + return 1; + } + + uint64_t bits = get_dbits(fp); + + bool nan = (bits & expmask) == expmask; + + if (!nan) + { + return 0; + } + + if (bits & fracmask) + { + dest[0] = 'n'; + dest[1] = 'a'; + dest[2] = 'n'; + } + else + { + dest[0] = 'i'; + dest[1] = 'n'; + dest[2] = 'f'; + } + + return 3; } static inline int fpconv_dtoa(double d, char dest[24]) { - char digits[18]; + char digits[18]; - int str_len = 0; - bool neg = false; + int str_len = 0; + bool neg = false; - if(get_dbits(d) & signmask) { - dest[0] = '-'; - str_len++; - neg = true; - } + if (get_dbits(d) & signmask) + { + dest[0] = '-'; + str_len++; + neg = true; + } - int spec = filter_special(d, dest + str_len); + int spec = filter_special(d, dest + str_len); - if(spec) { - return str_len + spec; - } + if (spec) + { + return str_len + spec; + } - int K = 0; - int ndigits = grisu2(d, digits, &K); + int K = 0; + int ndigits = grisu2(d, digits, &K); - str_len += emit_digits(digits, ndigits, dest + str_len, K, neg); + str_len += emit_digits(digits, ndigits, dest + str_len, K, neg); - return str_len; + return str_len; } diff --git a/libraries/clio/include/clio/from_bin.hpp b/libraries/clio/include/clio/from_bin.hpp index 011bb1e9e..9a0f74246 100644 --- a/libraries/clio/include/clio/from_bin.hpp +++ b/libraries/clio/include/clio/from_bin.hpp @@ -1,288 +1,350 @@ #pragma once -#include #include #include +#include #include #include #include #include +#include #include #include #include -#include -namespace clio { - -template -void from_bin(T& obj, S& stream); - -template -void varuint32_from_bin(uint32_t& dest, S& stream) { - dest = 0; - int shift = 0; - uint8_t b = 0; - do { - if (shift >= 35) - throw_error(stream_error::invalid_varuint_encoding); - from_bin(b, stream); - dest |= uint32_t(b & 0x7f) << shift; - shift += 7; - } while (b & 0x80); -} +namespace clio +{ + template + void from_bin(T& obj, S& stream); + + template + void varuint32_from_bin(uint32_t& dest, S& stream) + { + dest = 0; + int shift = 0; + uint8_t b = 0; + do + { + if (shift >= 35) + throw_error(stream_error::invalid_varuint_encoding); + from_bin(b, stream); + dest |= uint32_t(b & 0x7f) << shift; + shift += 7; + } while (b & 0x80); + } -template -void varuint64_from_bin(uint64_t& dest, S& stream) { - dest = 0; - int shift = 0; - uint8_t b = 0; - do { - if (shift >= 70) - throw_error(stream_error::invalid_varuint_encoding); - from_bin(b, stream); - dest |= uint64_t(b & 0x7f) << shift; - shift += 7; - } while (b & 0x80); -} + template + void varuint64_from_bin(uint64_t& dest, S& stream) + { + dest = 0; + int shift = 0; + uint8_t b = 0; + do + { + if (shift >= 70) + throw_error(stream_error::invalid_varuint_encoding); + from_bin(b, stream); + dest |= uint64_t(b & 0x7f) << shift; + shift += 7; + } while (b & 0x80); + } -template -void varint32_from_bin(int32_t& result, S& stream) { - uint32_t v; - varuint32_from_bin(v, stream); - if (v & 1) - result = ((~v) >> 1) | 0x8000'0000; - else - result = v >> 1; -} + template + void varint32_from_bin(int32_t& result, S& stream) + { + uint32_t v; + varuint32_from_bin(v, stream); + if (v & 1) + result = ((~v) >> 1) | 0x8000'0000; + else + result = v >> 1; + } -template -void from_bin_assoc(T& v, S& stream) { - uint32_t size; - varuint32_from_bin(size, stream); - for (size_t i = 0; i < size; ++i) { - typename T::value_type elem; - from_bin(elem, stream); - v.emplace(elem); + template + void from_bin_assoc(T& v, S& stream) + { + uint32_t size; + varuint32_from_bin(size, stream); + for (size_t i = 0; i < size; ++i) + { + typename T::value_type elem; + from_bin(elem, stream); + v.emplace(elem); + } } -} -template -void from_bin_sequence(T& v, S& stream) { - uint32_t size; - varuint32_from_bin(size, stream); - for (size_t i = 0; i < size; ++i) { - v.emplace_back(); - from_bin(v.back(), stream); + template + void from_bin_sequence(T& v, S& stream) + { + uint32_t size; + varuint32_from_bin(size, stream); + for (size_t i = 0; i < size; ++i) + { + v.emplace_back(); + from_bin(v.back(), stream); + } } -} -template -void from_bin(T (&v)[N], S& stream) { - uint32_t size; - varuint32_from_bin(size, stream); - if (size != N) - throw_error(stream_error::array_size_mismatch); - if constexpr (has_bitwise_serialization()) { - stream.read(reinterpret_cast(v), size * sizeof(T)); - } else { - for (size_t i = 0; i < size; ++i) { OUTCOME_TRY(from_bin(v[i], stream)); } + template + void from_bin(T (&v)[N], S& stream) + { + uint32_t size; + varuint32_from_bin(size, stream); + if (size != N) + throw_error(stream_error::array_size_mismatch); + if constexpr (has_bitwise_serialization()) + { + stream.read(reinterpret_cast(v), size * sizeof(T)); + } + else + { + for (size_t i = 0; i < size; ++i) + { + OUTCOME_TRY(from_bin(v[i], stream)); + } + } } -} -template -void from_bin(std::vector& v, S& stream) { - if constexpr (has_bitwise_serialization()) { - if constexpr (sizeof(size_t) >= 8) { - uint64_t size; - varuint64_from_bin(size, stream); - stream.check_available(size * sizeof(T)); - v.resize(size); - stream.read(reinterpret_cast(v.data()), size * sizeof(T)); - } else { + template + void from_bin(std::vector& v, S& stream) + { + if constexpr (has_bitwise_serialization()) + { + if constexpr (sizeof(size_t) >= 8) + { + uint64_t size; + varuint64_from_bin(size, stream); + stream.check_available(size * sizeof(T)); + v.resize(size); + stream.read(reinterpret_cast(v.data()), size * sizeof(T)); + } + else + { + uint32_t size; + varuint32_from_bin(size, stream); + stream.check_available(size * sizeof(T)); + v.resize(size); + stream.read(reinterpret_cast(v.data()), size * sizeof(T)); + } + } + else + { uint32_t size; varuint32_from_bin(size, stream); - stream.check_available(size * sizeof(T)); v.resize(size); - stream.read(reinterpret_cast(v.data()), size * sizeof(T)); + for (size_t i = 0; i < size; ++i) + { + from_bin(v[i], stream); + } } - } else { + } + + template + void from_bin(std::set& v, S& stream) + { + from_bin_assoc(v, stream); + } + + template + void from_bin(std::map& v, S& stream) + { uint32_t size; varuint32_from_bin(size, stream); - v.resize(size); - for (size_t i = 0; i < size; ++i) { - from_bin(v[i], stream); + for (size_t i = 0; i < size; ++i) + { + std::pair elem; + from_bin(elem, stream); + v.emplace(elem); } } -} -template -void from_bin(std::set& v, S& stream) { - from_bin_assoc(v, stream); -} + template + void from_bin(std::deque& v, S& stream) + { + from_bin_sequence(v, stream); + } -template -void from_bin(std::map& v, S& stream) { - uint32_t size; - varuint32_from_bin(size, stream); - for (size_t i = 0; i < size; ++i) { - std::pair elem; - from_bin(elem, stream); - v.emplace(elem); + template + void from_bin(std::list& v, S& stream) + { + from_bin_sequence(v, stream); } -} -template -void from_bin(std::deque& v, S& stream) { - from_bin_sequence(v, stream); -} + template + void from_bin(input_stream& obj, S& stream) + { + if constexpr (sizeof(size_t) >= 8) + { + uint64_t size; + varuint64_from_bin(size, stream); + stream.check_available(size); + stream.read_reuse_storage(obj.pos, size); + obj.end = obj.pos + size; + } + else + { + uint32_t size; + varuint32_from_bin(size, stream); + stream.check_available(size); + stream.read_reuse_storage(obj.pos, size); + obj.end = obj.pos + size; + } + } -template -void from_bin(std::list& v, S& stream) { - from_bin_sequence(v, stream); -} + template + void from_bin(std::pair& obj, S& stream) + { + from_bin(obj.first, stream); + from_bin(obj.second, stream); + } -template -void from_bin(input_stream& obj, S& stream) { - if constexpr (sizeof(size_t) >= 8) { - uint64_t size; - varuint64_from_bin(size, stream); - stream.check_available(size); - stream.read_reuse_storage(obj.pos, size); - obj.end = obj.pos + size; - } else { + template + inline void from_bin(std::string& obj, S& stream) + { uint32_t size; varuint32_from_bin(size, stream); - stream.check_available(size); - stream.read_reuse_storage(obj.pos, size); - obj.end = obj.pos + size; + obj.resize(size); + stream.read(obj.data(), obj.size()); } -} - -template -void from_bin(std::pair& obj, S& stream) { - from_bin(obj.first, stream); - from_bin(obj.second, stream); -} - -template -inline void from_bin(std::string& obj, S& stream) { - uint32_t size; - varuint32_from_bin(size, stream); - obj.resize(size); - stream.read(obj.data(), obj.size()); -} -template -inline void from_bin(std::string_view& obj, S& stream) { - uint32_t size; - varuint32_from_bin(size, stream); - obj = std::string_view(stream.get_pos(),size); - stream.skip(size); -} + template + inline void from_bin(std::string_view& obj, S& stream) + { + uint32_t size; + varuint32_from_bin(size, stream); + obj = std::string_view(stream.get_pos(), size); + stream.skip(size); + } -template -void from_bin(std::optional& obj, S& stream) { - bool present; - from_bin(present, stream); - if (!present) { - obj.reset(); - return; + template + void from_bin(std::optional& obj, S& stream) + { + bool present; + from_bin(present, stream); + if (!present) + { + obj.reset(); + return; + } + obj.emplace(); + from_bin(*obj, stream); } - obj.emplace(); - from_bin(*obj, stream); -} -template -void variant_from_bin(std::variant& v, uint32_t i, S& stream) { - if constexpr (I < std::variant_size_v>) { - if (i == I) { - auto& x = v.template emplace(); - from_bin(x, stream); - } else { - variant_from_bin(v, i, stream); + template + void variant_from_bin(std::variant& v, uint32_t i, S& stream) + { + if constexpr (I < std::variant_size_v>) + { + if (i == I) + { + auto& x = v.template emplace(); + from_bin(x, stream); + } + else + { + variant_from_bin(v, i, stream); + } + } + else + { + throw_error(stream_error::bad_variant_index); } - } else { - throw_error(stream_error::bad_variant_index); } -} -template -void from_bin(std::variant& obj, S& stream) { - uint32_t u; - varuint32_from_bin(u, stream); - variant_from_bin<0>(obj, u, stream); -} + template + void from_bin(std::variant& obj, S& stream) + { + uint32_t u; + varuint32_from_bin(u, stream); + variant_from_bin<0>(obj, u, stream); + } -template -void from_bin(std::array& obj, S& stream) { - for (T& elem : obj) { from_bin(elem, stream); } -} + template + void from_bin(std::array& obj, S& stream) + { + for (T& elem : obj) + { + from_bin(elem, stream); + } + } -template -void from_bin_tuple(T& obj, S& stream) { - if constexpr (N < std::tuple_size_v) { - from_bin(std::get(obj), stream); - from_bin_tuple(obj, stream); - }} + template + void from_bin_tuple(T& obj, S& stream) + { + if constexpr (N < std::tuple_size_v) + { + from_bin(std::get(obj), stream); + from_bin_tuple(obj, stream); + } + } -template -void from_bin(std::tuple& obj, S& stream) { - from_bin_tuple<0>(obj, stream); -} + template + void from_bin(std::tuple& obj, S& stream) + { + from_bin_tuple<0>(obj, stream); + } -template -void from_bin(T& obj, S& stream) { - if constexpr (has_bitwise_serialization()) { - stream.read(reinterpret_cast(&obj), sizeof(T)); - } else { //if constexpr (std::is_same_v, void>) { - reflect::for_each( [&]( const clio::meta&, auto m ) { - if constexpr( not std::is_member_function_pointer_v ) { - from_bin( obj.*m, stream ); - } - }); - } /*else { + template + void from_bin(T& obj, S& stream) + { + if constexpr (has_bitwise_serialization()) + { + stream.read(reinterpret_cast(&obj), sizeof(T)); + } + else + { // if constexpr (std::is_same_v, void>) { + reflect::for_each([&](const clio::meta&, auto m) { + if constexpr (not std::is_member_function_pointer_v) + { + from_bin(obj.*m, stream); + } + }); + } /*else { // TODO: This can operate in place for standard serializers decltype(serialize_as(obj)) temp; OUTCOME_TRY(from_bin(temp, stream)); convert(temp, obj, choose_first); return outcome::success(); } */ -} - -template -void from_bin_vec(T& obj, const std::vector& bin) { - input_stream stream{ bin }; - from_bin(obj, stream); -} -template -void from_bin_vec(T& obj, std::vector& bin) { - input_stream stream{ bin }; - from_bin(obj, stream); -} - -template -void from_bin_vec(T& obj, const bytes& bin) { - from_bin_vec(obj, bin.data); -} -template -void from_bin_vec(T& obj, bytes& bin) { - from_bin_vec(obj, bin.data); -} - -template -T from_bin(const std::vector& bin) { - T obj; - from_bin_vec(obj, bin); - return obj; -} + } -template -T from_bin(S& stream) { - T obj; - from_bin(obj, stream); - return obj; -} + template + void from_bin_vec(T& obj, const std::vector& bin) + { + input_stream stream{bin}; + from_bin(obj, stream); + } + template + void from_bin_vec(T& obj, std::vector& bin) + { + input_stream stream{bin}; + from_bin(obj, stream); + } + template + void from_bin_vec(T& obj, const bytes& bin) + { + from_bin_vec(obj, bin.data); + } + template + void from_bin_vec(T& obj, bytes& bin) + { + from_bin_vec(obj, bin.data); + } + template + T from_bin(const std::vector& bin) + { + T obj; + from_bin_vec(obj, bin); + return obj; + } + template + T from_bin(S& stream) + { + T obj; + from_bin(obj, stream); + return obj; + } -} // namespace clio +} // namespace clio diff --git a/libraries/clio/include/clio/from_bin/varint.hpp b/libraries/clio/include/clio/from_bin/varint.hpp index 08e0b3314..29326e1e9 100644 --- a/libraries/clio/include/clio/from_bin/varint.hpp +++ b/libraries/clio/include/clio/from_bin/varint.hpp @@ -3,16 +3,18 @@ #include #include -namespace clio { +namespace clio +{ + template + void from_bin(varuint32& obj, S& stream) + { + varuint32_from_bin(obj.value, stream); + } -template -void from_bin(varuint32& obj, S& stream) { - varuint32_from_bin(obj.value, stream); -} + template + void from_bin(varint32& obj, S& stream) + { + varint32_from_bin(obj.value, stream); + } -template -void from_bin(varint32& obj, S& stream) { - varint32_from_bin(obj.value, stream); -} - -} /// namespace clio +} // namespace clio diff --git a/libraries/clio/include/clio/from_json.hpp b/libraries/clio/include/clio/from_json.hpp index 25ff07ad8..c7b79f9dd 100644 --- a/libraries/clio/include/clio/from_json.hpp +++ b/libraries/clio/include/clio/from_json.hpp @@ -1,487 +1,548 @@ #pragma once -#include +#include #include +#include #include #include -#include -#include -#include #include +#include +#include + +namespace clio +{ + enum class from_json_error + { + no_error, + + expected_end, + expected_null, + expected_bool, + expected_string, + expected_hex_string, + hex_string_incorrect_length, + invalid_signature, + invalid_name, + expected_start_object, + expected_key, + expected_end_object, + expected_start_array, + expected_end_array, + expected_positive_uint, + expected_field, + expected_variant, + expected_public_key, + expected_private_key, + expected_signature, + expected_number, + expected_int, + expected_time_point, + expected_symbol_code, + expected_symbol, + expected_asset, + invalid_type_for_variant, + unexpected_field, + number_out_of_range, + from_json_no_pair, + + // These are from rapidjson: + document_empty, + document_root_not_singular, + value_invalid, + object_miss_name, + object_miss_colon, + object_miss_comma_or_curly_bracket, + array_miss_comma_or_square_bracket, + string_unicode_escape_invalid_hex, + string_unicode_surrogate_invalid, + string_escape_invalid, + string_miss_quotation_mark, + string_invalid_encoding, + number_too_big, + number_miss_fraction, + number_miss_exponent, + terminated, + unspecific_syntax_error, + }; // from_json_error +} // namespace clio + +namespace std +{ + template <> + struct is_error_code_enum : true_type + { + }; +} // namespace std + +namespace clio +{ + class from_json_error_category_type : public std::error_category + { + public: + const char* name() const noexcept override final { return "ConversionError"; } -namespace clio { -enum class from_json_error { - no_error, - - expected_end, - expected_null, - expected_bool, - expected_string, - expected_hex_string, - hex_string_incorrect_length, - invalid_signature, - invalid_name, - expected_start_object, - expected_key, - expected_end_object, - expected_start_array, - expected_end_array, - expected_positive_uint, - expected_field, - expected_variant, - expected_public_key, - expected_private_key, - expected_signature, - expected_number, - expected_int, - expected_time_point, - expected_symbol_code, - expected_symbol, - expected_asset, - invalid_type_for_variant, - unexpected_field, - number_out_of_range, - from_json_no_pair, - - // These are from rapidjson: - document_empty, - document_root_not_singular, - value_invalid, - object_miss_name, - object_miss_colon, - object_miss_comma_or_curly_bracket, - array_miss_comma_or_square_bracket, - string_unicode_escape_invalid_hex, - string_unicode_surrogate_invalid, - string_escape_invalid, - string_miss_quotation_mark, - string_invalid_encoding, - number_too_big, - number_miss_fraction, - number_miss_exponent, - terminated, - unspecific_syntax_error, -}; // from_json_error -} // namespace clio - -namespace std { -template <> -struct is_error_code_enum : true_type {}; -} // namespace std - -namespace clio { - -class from_json_error_category_type : public std::error_category { - public: - const char* name() const noexcept override final { return "ConversionError"; } - - std::string message(int c) const override final { - switch (static_cast(c)) { + std::string message(int c) const override final + { + switch (static_cast(c)) + { // clang-format off - case from_json_error::no_error: return "No error"; - - case from_json_error::expected_end: return "Expected end of json"; - case from_json_error::expected_null: return "Expected null"; - case from_json_error::expected_bool: return "Expected true or false"; - case from_json_error::expected_string: return "Expected string"; - case from_json_error::expected_hex_string: return "Expected string containing hex"; - case from_json_error::hex_string_incorrect_length: return "Hex string has incorrect length"; - case from_json_error::invalid_signature: return "Invalid signature format"; - case from_json_error::invalid_name: return "Invalid name"; - case from_json_error::expected_start_object: return "Expected {"; - case from_json_error::expected_key: return "Expected key"; - case from_json_error::expected_end_object: return "Expected }"; - case from_json_error::expected_start_array: return "Expected ["; - case from_json_error::expected_end_array: return "Expected ]"; - case from_json_error::expected_positive_uint: return "Expected positive integer"; - case from_json_error::expected_field: return "Expected field"; - case from_json_error::expected_variant: return R"(Expected variant: ["type", value])"; - case from_json_error::expected_public_key: return "Expected public key"; - case from_json_error::expected_private_key: return "Expected private key"; - case from_json_error::expected_signature: return "Expected signature"; - case from_json_error::expected_number: return "Expected number or boolean"; - case from_json_error::expected_int: return "Expected integer"; - case from_json_error::expected_time_point: return "Expected time point"; - case from_json_error::expected_symbol_code: return "Expected symbol code"; - case from_json_error::expected_symbol: return "Expected symbol"; - case from_json_error::expected_asset: return "Expected asset"; - case from_json_error::invalid_type_for_variant: return "Invalid type for variant"; - case from_json_error::unexpected_field: return "Unexpected field"; - case from_json_error::number_out_of_range: return "number is out of range"; - case from_json_error::from_json_no_pair: return "from_json does not support std::pair"; - - case from_json_error::document_empty: return "The document is empty"; - case from_json_error::document_root_not_singular: return "The document root must not follow by other values"; - case from_json_error::value_invalid: return "Invalid value"; - case from_json_error::object_miss_name: return "Missing a name for object member"; - case from_json_error::object_miss_colon: return "Missing a colon after a name of object member"; - case from_json_error::object_miss_comma_or_curly_bracket: return "Missing a comma or '}' after an object member"; - case from_json_error::array_miss_comma_or_square_bracket: return "Missing a comma or ']' after an array element"; - case from_json_error::string_unicode_escape_invalid_hex: return "Incorrect hex digit after \\u escape in string"; - case from_json_error::string_unicode_surrogate_invalid: return "The surrogate pair in string is invalid"; - case from_json_error::string_escape_invalid: return "Invalid escape character in string"; - case from_json_error::string_miss_quotation_mark: return "Missing a closing quotation mark in string"; - case from_json_error::string_invalid_encoding: return "Invalid encoding in string"; - case from_json_error::number_too_big: return "Number too big to be stored in double"; - case from_json_error::number_miss_fraction: return "Miss fraction part in number"; - case from_json_error::number_miss_exponent: return "Miss exponent in number"; - case from_json_error::terminated: return "Parsing was terminated"; - case from_json_error::unspecific_syntax_error: return "Unspecific syntax error"; + case from_json_error::no_error: return "No error"; + + case from_json_error::expected_end: return "Expected end of json"; + case from_json_error::expected_null: return "Expected null"; + case from_json_error::expected_bool: return "Expected true or false"; + case from_json_error::expected_string: return "Expected string"; + case from_json_error::expected_hex_string: return "Expected string containing hex"; + case from_json_error::hex_string_incorrect_length: return "Hex string has incorrect length"; + case from_json_error::invalid_signature: return "Invalid signature format"; + case from_json_error::invalid_name: return "Invalid name"; + case from_json_error::expected_start_object: return "Expected {"; + case from_json_error::expected_key: return "Expected key"; + case from_json_error::expected_end_object: return "Expected }"; + case from_json_error::expected_start_array: return "Expected ["; + case from_json_error::expected_end_array: return "Expected ]"; + case from_json_error::expected_positive_uint: return "Expected positive integer"; + case from_json_error::expected_field: return "Expected field"; + case from_json_error::expected_variant: return R"(Expected variant: ["type", value])"; + case from_json_error::expected_public_key: return "Expected public key"; + case from_json_error::expected_private_key: return "Expected private key"; + case from_json_error::expected_signature: return "Expected signature"; + case from_json_error::expected_number: return "Expected number or boolean"; + case from_json_error::expected_int: return "Expected integer"; + case from_json_error::expected_time_point: return "Expected time point"; + case from_json_error::expected_symbol_code: return "Expected symbol code"; + case from_json_error::expected_symbol: return "Expected symbol"; + case from_json_error::expected_asset: return "Expected asset"; + case from_json_error::invalid_type_for_variant: return "Invalid type for variant"; + case from_json_error::unexpected_field: return "Unexpected field"; + case from_json_error::number_out_of_range: return "number is out of range"; + case from_json_error::from_json_no_pair: return "from_json does not support std::pair"; + + case from_json_error::document_empty: return "The document is empty"; + case from_json_error::document_root_not_singular: return "The document root must not follow by other values"; + case from_json_error::value_invalid: return "Invalid value"; + case from_json_error::object_miss_name: return "Missing a name for object member"; + case from_json_error::object_miss_colon: return "Missing a colon after a name of object member"; + case from_json_error::object_miss_comma_or_curly_bracket: return "Missing a comma or '}' after an object member"; + case from_json_error::array_miss_comma_or_square_bracket: return "Missing a comma or ']' after an array element"; + case from_json_error::string_unicode_escape_invalid_hex: return "Incorrect hex digit after \\u escape in string"; + case from_json_error::string_unicode_surrogate_invalid: return "The surrogate pair in string is invalid"; + case from_json_error::string_escape_invalid: return "Invalid escape character in string"; + case from_json_error::string_miss_quotation_mark: return "Missing a closing quotation mark in string"; + case from_json_error::string_invalid_encoding: return "Invalid encoding in string"; + case from_json_error::number_too_big: return "Number too big to be stored in double"; + case from_json_error::number_miss_fraction: return "Miss fraction part in number"; + case from_json_error::number_miss_exponent: return "Miss exponent in number"; + case from_json_error::terminated: return "Parsing was terminated"; + case from_json_error::unspecific_syntax_error: return "Unspecific syntax error"; + // clang-format on + + default: + return "unknown"; + } + } + }; // from_json_error_category_type + + inline const from_json_error_category_type& from_json_error_category() + { + static from_json_error_category_type c; + return c; + } + + inline std::error_code make_error_code(from_json_error e) + { + return {static_cast(e), from_json_error_category()}; + } + + inline from_json_error convert_error(rapidjson::ParseErrorCode err) + { + switch (err) + { + // clang-format off + case rapidjson::kParseErrorNone: return from_json_error::no_error; + case rapidjson::kParseErrorDocumentEmpty: return from_json_error::document_empty; + case rapidjson::kParseErrorDocumentRootNotSingular: return from_json_error::document_root_not_singular; + case rapidjson::kParseErrorValueInvalid: return from_json_error::value_invalid; + case rapidjson::kParseErrorObjectMissName: return from_json_error::object_miss_name; + case rapidjson::kParseErrorObjectMissColon: return from_json_error::object_miss_colon; + case rapidjson::kParseErrorObjectMissCommaOrCurlyBracket: return from_json_error::object_miss_comma_or_curly_bracket; + case rapidjson::kParseErrorArrayMissCommaOrSquareBracket: return from_json_error::array_miss_comma_or_square_bracket; + case rapidjson::kParseErrorStringUnicodeEscapeInvalidHex: return from_json_error::string_unicode_escape_invalid_hex; + case rapidjson::kParseErrorStringUnicodeSurrogateInvalid: return from_json_error::string_unicode_surrogate_invalid; + case rapidjson::kParseErrorStringEscapeInvalid: return from_json_error::string_escape_invalid; + case rapidjson::kParseErrorStringMissQuotationMark: return from_json_error::string_miss_quotation_mark; + case rapidjson::kParseErrorStringInvalidEncoding: return from_json_error::string_invalid_encoding; + case rapidjson::kParseErrorNumberTooBig: return from_json_error::number_too_big; + case rapidjson::kParseErrorNumberMissFraction: return from_json_error::number_miss_fraction; + case rapidjson::kParseErrorNumberMissExponent: return from_json_error::number_miss_exponent; + case rapidjson::kParseErrorTermination: return from_json_error::terminated; + case rapidjson::kParseErrorUnspecificSyntaxError: return from_json_error::unspecific_syntax_error; // clang-format on - default: return "unknown"; + default: + return from_json_error::unspecific_syntax_error; } } -}; // from_json_error_category_type -inline const from_json_error_category_type& from_json_error_category() { - static from_json_error_category_type c; - return c; -} + enum class json_token_type + { + type_unread, + type_null, + type_bool, + type_string, + type_start_object, + type_key, + type_end_object, + type_start_array, + type_end_array, + }; -inline std::error_code make_error_code(from_json_error e) { - return { static_cast(e), from_json_error_category() }; -} + struct json_token + { + json_token_type type = {}; + std::string_view key = {}; + bool value_bool = {}; + std::string_view value_string = {}; + }; -inline from_json_error convert_error(rapidjson::ParseErrorCode err) { - switch (err) { - // clang-format off - case rapidjson::kParseErrorNone: return from_json_error::no_error; - case rapidjson::kParseErrorDocumentEmpty: return from_json_error::document_empty; - case rapidjson::kParseErrorDocumentRootNotSingular: return from_json_error::document_root_not_singular; - case rapidjson::kParseErrorValueInvalid: return from_json_error::value_invalid; - case rapidjson::kParseErrorObjectMissName: return from_json_error::object_miss_name; - case rapidjson::kParseErrorObjectMissColon: return from_json_error::object_miss_colon; - case rapidjson::kParseErrorObjectMissCommaOrCurlyBracket: return from_json_error::object_miss_comma_or_curly_bracket; - case rapidjson::kParseErrorArrayMissCommaOrSquareBracket: return from_json_error::array_miss_comma_or_square_bracket; - case rapidjson::kParseErrorStringUnicodeEscapeInvalidHex: return from_json_error::string_unicode_escape_invalid_hex; - case rapidjson::kParseErrorStringUnicodeSurrogateInvalid: return from_json_error::string_unicode_surrogate_invalid; - case rapidjson::kParseErrorStringEscapeInvalid: return from_json_error::string_escape_invalid; - case rapidjson::kParseErrorStringMissQuotationMark: return from_json_error::string_miss_quotation_mark; - case rapidjson::kParseErrorStringInvalidEncoding: return from_json_error::string_invalid_encoding; - case rapidjson::kParseErrorNumberTooBig: return from_json_error::number_too_big; - case rapidjson::kParseErrorNumberMissFraction: return from_json_error::number_miss_fraction; - case rapidjson::kParseErrorNumberMissExponent: return from_json_error::number_miss_exponent; - case rapidjson::kParseErrorTermination: return from_json_error::terminated; - case rapidjson::kParseErrorUnspecificSyntaxError: return from_json_error::unspecific_syntax_error; - // clang-format on - - default: return from_json_error::unspecific_syntax_error; - } -} + class json_token_stream + : public rapidjson::BaseReaderHandler, json_token_stream> + { + private: + rapidjson::Reader reader; + rapidjson::InsituStringStream ss; -enum class json_token_type { - type_unread, - type_null, - type_bool, - type_string, - type_start_object, - type_key, - type_end_object, - type_start_array, - type_end_array, -}; - -struct json_token { - json_token_type type = {}; - std::string_view key = {}; - bool value_bool = {}; - std::string_view value_string = {}; -}; - -class json_token_stream : public rapidjson::BaseReaderHandler, json_token_stream> { - private: - rapidjson::Reader reader; - rapidjson::InsituStringStream ss; - - public: - json_token current_token; - - // This modifies json (ok... why?) - json_token_stream(char* json) : ss{ json } { reader.IterativeParseInit(); } - - bool complete() { return reader.IterativeParseComplete(); } - - std::reference_wrapper peek_token() { - if (current_token.type != json_token_type::type_unread) - return current_token; - else if (reader.IterativeParseNext(ss, - *this)) - return current_token; - else - throw_error(convert_error(reader.GetParseErrorCode())); - } + public: + json_token current_token; - void eat_token() { current_token.type = json_token_type::type_unread; } + // This modifies json (ok... why?) + json_token_stream(char* json) : ss{json} { reader.IterativeParseInit(); } - void get_end() { - if (current_token.type != json_token_type::type_unread || !complete()) - throw_error(from_json_error::expected_end); - } + bool complete() { return reader.IterativeParseComplete(); } - void get_null() { - auto t = peek_token(); - if (t.get().type != json_token_type::type_null) - throw_error( from_json_error::expected_null ); - eat_token(); - } + std::reference_wrapper peek_token() + { + if (current_token.type != json_token_type::type_unread) + return current_token; + else if (reader.IterativeParseNext< + rapidjson::kParseInsituFlag | rapidjson::kParseValidateEncodingFlag | + rapidjson::kParseIterativeFlag | rapidjson::kParseNumbersAsStringsFlag>( + ss, *this)) + return current_token; + else + throw_error(convert_error(reader.GetParseErrorCode())); + } - bool get_bool() { - auto t = peek_token(); - if (t.get().type != json_token_type::type_bool) - throw_error( from_json_error::expected_bool); - eat_token(); - return t.get().value_bool; - } + void eat_token() { current_token.type = json_token_type::type_unread; } - std::string_view get_string() { - auto t = peek_token(); - if (t.get().type != json_token_type::type_string) - throw_error(from_json_error::expected_string); - eat_token(); - return t.get().value_string; - } + void get_end() + { + if (current_token.type != json_token_type::type_unread || !complete()) + throw_error(from_json_error::expected_end); + } - void get_start_object() { - auto t = peek_token(); - if (t.get().type != json_token_type::type_start_object) - throw_error(from_json_error::expected_start_object); - eat_token(); - } + void get_null() + { + auto t = peek_token(); + if (t.get().type != json_token_type::type_null) + throw_error(from_json_error::expected_null); + eat_token(); + } - std::string_view get_key() { - auto t = peek_token(); - if (t.get().type != json_token_type::type_key) - throw_error(from_json_error::expected_key); - eat_token(); - return t.get().key; - } + bool get_bool() + { + auto t = peek_token(); + if (t.get().type != json_token_type::type_bool) + throw_error(from_json_error::expected_bool); + eat_token(); + return t.get().value_bool; + } - void get_end_object() { - auto t = peek_token(); - if (t.get().type != json_token_type::type_end_object) - throw_error(from_json_error::expected_end_object); - eat_token(); - } + std::string_view get_string() + { + auto t = peek_token(); + if (t.get().type != json_token_type::type_string) + throw_error(from_json_error::expected_string); + eat_token(); + return t.get().value_string; + } - void get_start_array() { - auto t = peek_token(); - if (t.get().type != json_token_type::type_start_array) - throw_error(from_json_error::expected_start_array); - eat_token(); - } + void get_start_object() + { + auto t = peek_token(); + if (t.get().type != json_token_type::type_start_object) + throw_error(from_json_error::expected_start_object); + eat_token(); + } - void get_end_array() { - auto t = peek_token(); - if (t.get().type != json_token_type::type_end_array) - throw_error(from_json_error::expected_end_array); - eat_token(); - } + std::string_view get_key() + { + auto t = peek_token(); + if (t.get().type != json_token_type::type_key) + throw_error(from_json_error::expected_key); + eat_token(); + return t.get().key; + } - // BaseReaderHandler methods - bool Null() { - current_token.type = json_token_type::type_null; - return true; - } - bool Bool(bool v) { - current_token.type = json_token_type::type_bool; - current_token.value_bool = v; - return true; - } - bool RawNumber(const char* v, rapidjson::SizeType length, bool copy) { return String(v, length, copy); } - bool Int(int v) { return false; } - bool Uint(unsigned v) { return false; } - bool Int64(int64_t v) { return false; } - bool Uint64(uint64_t v) { return false; } - bool Double(double v) { return false; } - bool String(const char* v, rapidjson::SizeType length, bool) { - current_token.type = json_token_type::type_string; - current_token.value_string = { v, length }; - return true; - } - bool StartObject() { - current_token.type = json_token_type::type_start_object; - return true; - } - bool Key(const char* v, rapidjson::SizeType length, bool) { - current_token.key = { v, length }; - current_token.type = json_token_type::type_key; - return true; - } - bool EndObject(rapidjson::SizeType) { - current_token.type = json_token_type::type_end_object; - return true; - } - bool StartArray() { - current_token.type = json_token_type::type_start_array; - return true; - } - bool EndArray(rapidjson::SizeType) { - current_token.type = json_token_type::type_end_array; - return true; - } -}; // json_token_stream - -template -[[nodiscard]] bool unhex(DestIt dest, SrcIt begin, SrcIt end) { - auto get_digit = [&](uint8_t& nibble) { - if (*begin >= '0' && *begin <= '9') - nibble = *begin++ - '0'; - else if (*begin >= 'a' && *begin <= 'f') - nibble = *begin++ - 'a' + 10; - else if (*begin >= 'A' && *begin <= 'F') - nibble = *begin++ - 'A' + 10; - else - return false; - return true; - }; - while (begin != end) { - uint8_t h, l; - if (!get_digit(h) || !get_digit(l)) - return false; - *dest++ = (h << 4) | l; - } - return true; -} + void get_end_object() + { + auto t = peek_token(); + if (t.get().type != json_token_type::type_end_object) + throw_error(from_json_error::expected_end_object); + eat_token(); + } -/// \exclude -template -void from_json(T& result, S& stream); - -/// \group from_json_explicit Parse JSON (Explicit Types) -/// Parse JSON and convert to `result`. These overloads handle specified types. -template -void from_json(std::string_view& result, S& stream) { - result = stream.get_string(); -} + void get_start_array() + { + auto t = peek_token(); + if (t.get().type != json_token_type::type_start_array) + throw_error(from_json_error::expected_start_array); + eat_token(); + } -/// \group from_json_explicit Parse JSON (Explicit Types) -/// Parse JSON and convert to `result`. These overloads handle specified types. -template -void from_json(std::string& result, S& stream) { - result = stream.get_string(); -} + void get_end_array() + { + auto t = peek_token(); + if (t.get().type != json_token_type::type_end_array) + throw_error(from_json_error::expected_end_array); + eat_token(); + } -/// \exclude -template -void from_json_int(T& result, S& stream) { - auto r = stream.get_string(); - auto pos = r.data(); - auto end = pos + r.size(); - bool found = false; - result = 0; - T limit; - T sign; - if (std::is_signed_v && pos != end && *pos == '-') { - ++pos; - sign = -1; - limit = std::numeric_limits::min(); - } else { - sign = 1; - limit = std::numeric_limits::max(); + // BaseReaderHandler methods + bool Null() + { + current_token.type = json_token_type::type_null; + return true; + } + bool Bool(bool v) + { + current_token.type = json_token_type::type_bool; + current_token.value_bool = v; + return true; + } + bool RawNumber(const char* v, rapidjson::SizeType length, bool copy) + { + return String(v, length, copy); + } + bool Int(int v) { return false; } + bool Uint(unsigned v) { return false; } + bool Int64(int64_t v) { return false; } + bool Uint64(uint64_t v) { return false; } + bool Double(double v) { return false; } + bool String(const char* v, rapidjson::SizeType length, bool) + { + current_token.type = json_token_type::type_string; + current_token.value_string = {v, length}; + return true; + } + bool StartObject() + { + current_token.type = json_token_type::type_start_object; + return true; + } + bool Key(const char* v, rapidjson::SizeType length, bool) + { + current_token.key = {v, length}; + current_token.type = json_token_type::type_key; + return true; + } + bool EndObject(rapidjson::SizeType) + { + current_token.type = json_token_type::type_end_object; + return true; + } + bool StartArray() + { + current_token.type = json_token_type::type_start_array; + return true; + } + bool EndArray(rapidjson::SizeType) + { + current_token.type = json_token_type::type_end_array; + return true; + } + }; // json_token_stream + + template + [[nodiscard]] bool unhex(DestIt dest, SrcIt begin, SrcIt end) + { + auto get_digit = [&](uint8_t& nibble) { + if (*begin >= '0' && *begin <= '9') + nibble = *begin++ - '0'; + else if (*begin >= 'a' && *begin <= 'f') + nibble = *begin++ - 'a' + 10; + else if (*begin >= 'A' && *begin <= 'F') + nibble = *begin++ - 'A' + 10; + else + return false; + return true; + }; + while (begin != end) + { + uint8_t h, l; + if (!get_digit(h) || !get_digit(l)) + return false; + *dest++ = (h << 4) | l; + } + return true; } - while (pos != end && *pos >= '0' && *pos <= '9') { - T digit = (*pos++ - '0'); - // abs(result) can overflow. Use -abs(result) instead. - if (std::is_signed_v && (-sign * limit + digit) / 10 > -sign * result) - throw_error(from_json_error::number_out_of_range); - if (!std::is_signed_v && (limit - digit) / 10 < result) - throw_error(from_json_error::number_out_of_range); - result = result * 10 + sign * digit; - found = true; + + /// \exclude + template + void from_json(T& result, S& stream); + + /// \group from_json_explicit Parse JSON (Explicit Types) + /// Parse JSON and convert to `result`. These overloads handle specified types. + template + void from_json(std::string_view& result, S& stream) + { + result = stream.get_string(); + } + + /// \group from_json_explicit Parse JSON (Explicit Types) + /// Parse JSON and convert to `result`. These overloads handle specified types. + template + void from_json(std::string& result, S& stream) + { + result = stream.get_string(); + } + + /// \exclude + template + void from_json_int(T& result, S& stream) + { + auto r = stream.get_string(); + auto pos = r.data(); + auto end = pos + r.size(); + bool found = false; + result = 0; + T limit; + T sign; + if (std::is_signed_v && pos != end && *pos == '-') + { + ++pos; + sign = -1; + limit = std::numeric_limits::min(); + } + else + { + sign = 1; + limit = std::numeric_limits::max(); + } + while (pos != end && *pos >= '0' && *pos <= '9') + { + T digit = (*pos++ - '0'); + // abs(result) can overflow. Use -abs(result) instead. + if (std::is_signed_v && (-sign * limit + digit) / 10 > -sign * result) + throw_error(from_json_error::number_out_of_range); + if (!std::is_signed_v && (limit - digit) / 10 < result) + throw_error(from_json_error::number_out_of_range); + result = result * 10 + sign * digit; + found = true; + } + if (pos != end || !found) + throw_error(from_json_error::expected_int); } - if (pos != end || !found) - throw_error(from_json_error::expected_int); -} -/// \group from_json_explicit -template -void from_json(uint8_t& result, S& stream) { - return from_json_int(result, stream); -} + /// \group from_json_explicit + template + void from_json(uint8_t& result, S& stream) + { + return from_json_int(result, stream); + } -/// \group from_json_explicit -template -void from_json(uint16_t& result, S& stream) { - return from_json_int(result, stream); -} + /// \group from_json_explicit + template + void from_json(uint16_t& result, S& stream) + { + return from_json_int(result, stream); + } -/// \group from_json_explicit -template -void from_json(uint32_t& result, S& stream) { - return from_json_int(result, stream); -} + /// \group from_json_explicit + template + void from_json(uint32_t& result, S& stream) + { + return from_json_int(result, stream); + } -/// \group from_json_explicit -template -void from_json(uint64_t& result, S& stream) { - return from_json_int(result, stream); -} + /// \group from_json_explicit + template + void from_json(uint64_t& result, S& stream) + { + return from_json_int(result, stream); + } -/// \group from_json_explicit -template -void from_json(unsigned __int128& result, S& stream) { - return from_json_int(result, stream); -} + /// \group from_json_explicit + template + void from_json(unsigned __int128& result, S& stream) + { + return from_json_int(result, stream); + } -/// \group from_json_explicit -template -void from_json(int8_t& result, S& stream) { - return from_json_int(result, stream); -} + /// \group from_json_explicit + template + void from_json(int8_t& result, S& stream) + { + return from_json_int(result, stream); + } -/// \group from_json_explicit -template -void from_json(int16_t& result, S& stream) { - return from_json_int(result, stream); -} + /// \group from_json_explicit + template + void from_json(int16_t& result, S& stream) + { + return from_json_int(result, stream); + } -/// \group from_json_explicit -template -void from_json(int32_t& result, S& stream) { - return from_json_int(result, stream); -} + /// \group from_json_explicit + template + void from_json(int32_t& result, S& stream) + { + return from_json_int(result, stream); + } -/// \group from_json_explicit -template -void from_json(int64_t& result, S& stream) { - return from_json_int(result, stream); -} + /// \group from_json_explicit + template + void from_json(int64_t& result, S& stream) + { + return from_json_int(result, stream); + } -/// \group from_json_explicit -template -void from_json(__int128& result, S& stream) { - return from_json_int(result, stream); -} + /// \group from_json_explicit + template + void from_json(__int128& result, S& stream) + { + return from_json_int(result, stream); + } -template -void from_json(float& result, S& stream) { - auto sv = stream.get_string(); - if (sv.empty()) - throw_error(from_json_error::expected_number); - std::string s(sv); // strtof expects a null-terminated string - errno = 0; - char* end; - result = std::strtof(s.c_str(), &end); - if (errno || end != s.c_str() + s.size()) - throw_error(from_json_error::expected_number); -} + template + void from_json(float& result, S& stream) + { + auto sv = stream.get_string(); + if (sv.empty()) + throw_error(from_json_error::expected_number); + std::string s(sv); // strtof expects a null-terminated string + errno = 0; + char* end; + result = std::strtof(s.c_str(), &end); + if (errno || end != s.c_str() + s.size()) + throw_error(from_json_error::expected_number); + } -template -void from_json(double& result, S& stream) { - auto sv = stream.get_string(); - if (sv.empty()) - throw_error( from_json_error::expected_number ); - std::string s(sv); - errno = 0; - char* end; - result = std::strtod(s.c_str(), &end); - if (errno || end != s.c_str() + s.size()) - throw_error(from_json_error::expected_number); -} + template + void from_json(double& result, S& stream) + { + auto sv = stream.get_string(); + if (sv.empty()) + throw_error(from_json_error::expected_number); + std::string s(sv); + errno = 0; + char* end; + result = std::strtod(s.c_str(), &end); + if (errno || end != s.c_str() + s.size()) + throw_error(from_json_error::expected_number); + } -/* + /* /// \group from_json_explicit template void from_json(int32_t& result, S& stream) { @@ -511,171 +572,202 @@ void from_json(int32_t& result, S& stream) { result = -result; } */ -/// \group from_json_explicit -template -void from_json(bool& result, S& stream) { - result = stream.get_bool(); -} - -/// \group from_json_explicit -template -void from_json(std::vector& result, S& stream) { - stream.get_start_array(); - result.clear(); - while (true) { - auto t = stream.peek_token(); - if (t.get().type == json_token_type::type_end_array) - break; - result.emplace_back(); - from_json(result.back(), stream); - } - stream.get_end_array(); -} - -/// \group from_json_explicit -template -void from_json(std::optional& result, S& stream) { - result = std::optional(); - if (stream.get_null()) { - result = std::nullopt; - } else { - result.emplace(); - from_json(*result, stream); + /// \group from_json_explicit + template + void from_json(bool& result, S& stream) + { + result = stream.get_bool(); + } + + /// \group from_json_explicit + template + void from_json(std::vector& result, S& stream) + { + stream.get_start_array(); + result.clear(); + while (true) + { + auto t = stream.peek_token(); + if (t.get().type == json_token_type::type_end_array) + break; + result.emplace_back(); + from_json(result.back(), stream); + } + stream.get_end_array(); } -} -template -bool set_variant_impl(std::variant& result, uint32_t type) { - if (type == N) { - result.template emplace(); - return true; - } else if constexpr (N + 1 < sizeof...(T)) { - return set_variant_impl(result, type); + /// \group from_json_explicit + template + void from_json(std::optional& result, S& stream) + { + result = std::optional(); + if (stream.get_null()) + { + result = std::nullopt; + } + else + { + result.emplace(); + from_json(*result, stream); + } } - return false; -} - -/// TODO: this is attempting to parse variant as [ type, obj ], -/// but to_json is encoding it as { type: "", value: } -/// \group from_json_explicit -template -void from_json(std::variant& result, S& stream) { - stream.get_start_array(); - std::string_view type; - from_json(type, stream); - const char* const type_names[] = { get_type_name((T*)nullptr)... }; - uint32_t type_idx = std::find(type_names, type_names + sizeof...(T), type) - type_names; - if (type_idx >= sizeof...(T)) - throw_error(from_json_error::invalid_type_for_variant); - if( set_variant_impl(result, type_idx) ) { - std::visit([&](auto& x) { return from_json(x, stream); }, result); - } - stream.get_end_array(); -} -/** - * Accepts null as default value for all tuple elements - * Accepts [] with 0 to N as the first n elements and errors if there are too many elements - * accepts anything else as the presumed first element - * - * TODO: make robust against adding elements to the tuple by droping all remaining values - */ -template -void from_json(std::tuple& result, S& stream) { - result = std::tuple(); - auto t = stream.peek_token(); - if( t.get().type == json_token_type::type_null ) { - return; - } - - if constexpr( sizeof...(T) > 0 ) { - if( t.get().type != json_token_type::type_start_array ) { - from_json( std::get<0>(result), stream ); - } - } - - stream.get_start_array(); - tuple_for_each( result, [&]( int idx, auto& item ) { - auto t = stream.peek_token(); - if( t.get().type == json_token_type::type_end_array ) - return; - from_json( item, stream ); - }); - stream.get_end_array(); -} + template + bool set_variant_impl(std::variant& result, uint32_t type) + { + if (type == N) + { + result.template emplace(); + return true; + } + else if constexpr (N + 1 < sizeof...(T)) + { + return set_variant_impl(result, type); + } + return false; + } + + /// TODO: this is attempting to parse variant as [ type, obj ], + /// but to_json is encoding it as { type: "", value: } + /// \group from_json_explicit + template + void from_json(std::variant& result, S& stream) + { + stream.get_start_array(); + std::string_view type; + from_json(type, stream); + const char* const type_names[] = {get_type_name((T*)nullptr)...}; + uint32_t type_idx = std::find(type_names, type_names + sizeof...(T), type) - type_names; + if (type_idx >= sizeof...(T)) + throw_error(from_json_error::invalid_type_for_variant); + if (set_variant_impl(result, type_idx)) + { + std::visit([&](auto& x) { return from_json(x, stream); }, result); + } + stream.get_end_array(); + } + + /** + * Accepts null as default value for all tuple elements + * Accepts [] with 0 to N as the first n elements and errors if there are too many elements + * accepts anything else as the presumed first element + * + * TODO: make robust against adding elements to the tuple by droping all remaining values + */ + template + void from_json(std::tuple& result, S& stream) + { + result = std::tuple(); + auto t = stream.peek_token(); + if (t.get().type == json_token_type::type_null) + { + return; + } -/// \group from_json_explicit -template -void from_json_hex(std::vector& result, S& stream) { - auto s = stream.get_string(); - if (s.size() & 1) - throw_error(from_json_error::expected_hex_string); - result.clear(); - result.reserve(s.size() / 2); - if (!unhex(std::back_inserter(result), s.begin(), s.end())) - throw_error(from_json_error::expected_hex_string); -} + if constexpr (sizeof...(T) > 0) + { + if (t.get().type != json_token_type::type_start_array) + { + from_json(std::get<0>(result), stream); + } + } -/// \exclude -template -inline void from_json_object(S& stream, F f) { - stream.get_start_object(); - while (true) { - auto t = stream.peek_token(); - if (t.get().type == json_token_type::type_end_object) + stream.get_start_array(); + tuple_for_each(result, [&](int idx, auto& item) { + auto t = stream.peek_token(); + if (t.get().type == json_token_type::type_end_array) + return; + from_json(item, stream); + }); + stream.get_end_array(); + } + + /// \group from_json_explicit + template + void from_json_hex(std::vector& result, S& stream) + { + auto s = stream.get_string(); + if (s.size() & 1) + throw_error(from_json_error::expected_hex_string); + result.clear(); + result.reserve(s.size() / 2); + if (!unhex(std::back_inserter(result), s.begin(), s.end())) + throw_error(from_json_error::expected_hex_string); + } + + /// \exclude + template + inline void from_json_object(S& stream, F f) + { + stream.get_start_object(); + while (true) { - break; + auto t = stream.peek_token(); + if (t.get().type == json_token_type::type_end_object) + { + break; + } + auto k = stream.get_key(); + f(k); } - auto k = stream.get_key(); - f(k); + stream.get_end_object(); } - stream.get_end_object(); -} -template -void from_json_skip_value(S& stream) { - uint64_t depth = 0; - do { - auto t = stream.peek_token(); - auto type = t.get().type; - if (type == json_token_type::type_start_object || type == json_token_type::type_start_array) - ++depth; - else if (type == json_token_type::type_end_object || type == json_token_type::type_end_array) - --depth; - stream.eat_token(); - } while (depth); -} - -/// \output_section Parse JSON (Reflected Objects) -/// Parse JSON and convert to `obj`. This overload works with -/// [reflected objects](standardese://reflection/). -template -void from_json(T& obj, S& stream) { - if constexpr( reflect::is_struct ) { - from_json_object(stream, [&](std::string_view key) -> void { - bool found = false; - reflect::get( key, [&]( auto member ){ - if constexpr( not std::is_member_function_pointer_v< decltype(member) > ) + template + void from_json_skip_value(S& stream) + { + uint64_t depth = 0; + do + { + auto t = stream.peek_token(); + auto type = t.get().type; + if (type == json_token_type::type_start_object || + type == json_token_type::type_start_array) + ++depth; + else if (type == json_token_type::type_end_object || + type == json_token_type::type_end_array) + --depth; + stream.eat_token(); + } while (depth); + } + + /// \output_section Parse JSON (Reflected Objects) + /// Parse JSON and convert to `obj`. This overload works with + /// [reflected objects](standardese://reflection/). + template + void from_json(T& obj, S& stream) + { + if constexpr (reflect::is_struct) + { + from_json_object(stream, [&](std::string_view key) -> void { + bool found = false; + reflect::get(key, [&](auto member) { + if constexpr (not std::is_member_function_pointer_v) { - from_json( obj.*member, stream ); - found = true; - } else { - } - }); - if( not found ) { - from_json_skip_value(stream); - } - }); - } else { - T::reflection_not_defined; + from_json(obj.*member, stream); + found = true; + } + else + { + } + }); + if (not found) + { + from_json_skip_value(stream); + } + }); + } + else + { + T::reflection_not_defined; + } } -} -/// not implimented, so don't link -template -void from_json(std::pair& obj, S& stream); + /// not implimented, so don't link + template + void from_json(std::pair& obj, S& stream); -/* + /* /// \output_section Convenience Wrappers /// Parse JSON and return result. This overload wraps the other `to_json` overloads. template @@ -702,22 +794,23 @@ T from_json(std::string_view s) { } */ -/// Parse JSON and return result. This overload wraps the other `to_json` overloads. -template -T from_json(S& stream) { - T x; - from_json(x, stream); - return x; -} - -template -T from_json( std::string json) { - json_token_stream stream( json.data() ); - return from_json(stream); -} + /// Parse JSON and return result. This overload wraps the other `to_json` overloads. + template + T from_json(S& stream) + { + T x; + from_json(x, stream); + return x; + } + template + T from_json(std::string json) + { + json_token_stream stream(json.data()); + return from_json(stream); + } -/* + /* /// \exclude template __attribute__((noinline)) void parse_named_variant_impl(tagged_variant& v, size_t i, @@ -759,4 +852,4 @@ __attribute__((noinline)) void from_json(tagged_variant& } */ -} // namespace clio +} // namespace clio diff --git a/libraries/clio/include/clio/from_json/varint.hpp b/libraries/clio/include/clio/from_json/varint.hpp index fa4961695..756e23a4a 100644 --- a/libraries/clio/include/clio/from_json/varint.hpp +++ b/libraries/clio/include/clio/from_json/varint.hpp @@ -3,16 +3,18 @@ #include #include -namespace clio { +namespace clio +{ + template + void from_json(varuint32& obj, S& stream) + { + from_json(obj.value, stream); + } -template -void from_json(varuint32& obj, S& stream) { - from_json(obj.value, stream); -} + template + void from_json(varint32& obj, S& stream) + { + from_json(obj.value, stream); + } -template -void from_json(varint32& obj, S& stream) { - from_json(obj.value, stream); -} - -} /// namespace clio +} // namespace clio diff --git a/libraries/clio/include/clio/from_protobuf.hpp b/libraries/clio/include/clio/from_protobuf.hpp index e05bd9fdf..f651ec32a 100644 --- a/libraries/clio/include/clio/from_protobuf.hpp +++ b/libraries/clio/include/clio/from_protobuf.hpp @@ -2,186 +2,224 @@ #include #include -namespace clio { - -template -void from_protobuf_object( T& obj, S& stream ); - -template -bool set_variant(std::variant& result, uint32_t type) { - if (type == N) { - result.template emplace(); - return true; - } else if constexpr (N + 1 < sizeof...(T)) { - return set_variant(result, type); +namespace clio +{ + template + void from_protobuf_object(T& obj, S& stream); + + template + bool set_variant(std::variant& result, uint32_t type) + { + if (type == N) + { + result.template emplace(); + return true; + } + else if constexpr (N + 1 < sizeof...(T)) + { + return set_variant(result, type); + } + return false; } - return false; -} -template -void skip_member( uint16_t wire_type, S& stream ) { - uint32_t temp = 0; - switch( wire_type ) { - case 0: - varuint32_from_bin( temp, stream ); + template + void skip_member(uint16_t wire_type, S& stream) + { + uint32_t temp = 0; + switch (wire_type) + { + case 0: + varuint32_from_bin(temp, stream); break; - case 1: - stream.skip(8); - break; - case 2: - varuint32_from_bin( temp, stream ); + case 1: + stream.skip(8); + break; + case 2: + varuint32_from_bin(temp, stream); stream.skip(temp); break; - case 5: + case 5: stream.skip(4); - break; - } -} - - -template -void from_protobuf_object(std::variant& obj, S& stream) -{ - uint32_t key = 0; - varuint32_from_bin( key, stream ); - uint32_t field = key >> 3; - - if( set_variant( obj, field-1 ) ) { - std::visit([&](auto& x) { - return from_protobuf_member(x, stream); - }, obj); - } else { - skip_member( uint8_t(key) & 0x07 , stream ); - obj = std::variant(); /// reset to default - } -} -template -void from_protobuf_object(std::tuple& obj, S& stream) -{ - while( stream.remaining() ) { - uint32_t key = 0; - varuint32_from_bin( key, stream ); - uint16_t wire_type = uint8_t(key) & 0x07; - uint32_t number = key >> 3; - - bool skip_it = true; - tuple_get( obj, number-1, [&]( auto& member ) { - from_protobuf_member( member, stream ); - skip_it = false; - }); - if( skip_it ) skip_member( wire_type, stream ); - } -} + break; + } + } + template + void from_protobuf_object(std::variant& obj, S& stream) + { + uint32_t key = 0; + varuint32_from_bin(key, stream); + uint32_t field = key >> 3; + + if (set_variant(obj, field - 1)) + { + std::visit([&](auto& x) { return from_protobuf_member(x, stream); }, obj); + } + else + { + skip_member(uint8_t(key) & 0x07, stream); + obj = std::variant(); /// reset to default + } + } + template + void from_protobuf_object(std::tuple& obj, S& stream) + { + while (stream.remaining()) + { + uint32_t key = 0; + varuint32_from_bin(key, stream); + uint16_t wire_type = uint8_t(key) & 0x07; + uint32_t number = key >> 3; + + bool skip_it = true; + tuple_get(obj, number - 1, [&](auto& member) { + from_protobuf_member(member, stream); + skip_it = false; + }); + if (skip_it) + skip_member(wire_type, stream); + } + } -template -void from_protobuf_object( std::vector& obj, S& stream ) { - obj.clear(); - if constexpr( std::is_arithmetic_v ) { - uint32_t key = 0; - varuint32_from_bin( key, stream ); - uint16_t wire_type = uint8_t(key) & 0x07; - uint32_t number = key >> 3; - if( number != 1 ) - skip_member( wire_type, stream ); - else { + template + void from_protobuf_object(std::vector& obj, S& stream) + { + obj.clear(); + if constexpr (std::is_arithmetic_v) + { + uint32_t key = 0; + varuint32_from_bin(key, stream); + uint16_t wire_type = uint8_t(key) & 0x07; + uint32_t number = key >> 3; + if (number != 1) + skip_member(wire_type, stream); + else + { uint32_t size = 0; - varuint32_from_bin( size, stream ); - if( size > stream.remaining() ) - throw_error(stream_error::overrun); - obj.resize( size / sizeof( T ) ); - stream.read( (char*)obj.data(), size ); - } - } else { - while( stream.remaining() ) { + varuint32_from_bin(size, stream); + if (size > stream.remaining()) + throw_error(stream_error::overrun); + obj.resize(size / sizeof(T)); + stream.read((char*)obj.data(), size); + } + } + else + { + while (stream.remaining()) + { uint32_t key = 0; - varuint32_from_bin( key, stream ); + varuint32_from_bin(key, stream); uint16_t wire_type = uint8_t(key) & 0x07; - uint32_t number = key >> 3; + uint32_t number = key >> 3; bool skip_it = true; - if( number == 1 ) { - skip_it = false; - obj.resize(obj.size()+1); - from_protobuf_member( obj.back(), stream ); - } else { - skip_member( wire_type, stream ); + if (number == 1) + { + skip_it = false; + obj.resize(obj.size() + 1); + from_protobuf_member(obj.back(), stream); + } + else + { + skip_member(wire_type, stream); } - } - } -} + } + } + } -template -void from_protobuf_member( T& obj, S& stream ) { - if constexpr( std::is_same_v< T, std::string> ) { - uint32_t size = 0; - varuint32_from_bin( size, stream ); - obj.resize(size); - stream.read( obj.data(), size ); - } else if constexpr( std::is_same_v< T, bytes> ) { - uint32_t size = 0; - varuint32_from_bin( size, stream ); - obj.data.resize(size); - stream.read( obj.data.data(), size ); - } - else if constexpr( reflect::is_struct || is_std_variant::value || is_std_vector::value || is_std_tuple::value ) { - uint32_t size = 0; - varuint32_from_bin( size, stream ); - if( size > stream.remaining() ) + template + void from_protobuf_member(T& obj, S& stream) + { + if constexpr (std::is_same_v) + { + uint32_t size = 0; + varuint32_from_bin(size, stream); + obj.resize(size); + stream.read(obj.data(), size); + } + else if constexpr (std::is_same_v) + { + uint32_t size = 0; + varuint32_from_bin(size, stream); + obj.data.resize(size); + stream.read(obj.data.data(), size); + } + else if constexpr (reflect::is_struct || is_std_variant::value || + is_std_vector::value || is_std_tuple::value) + { + uint32_t size = 0; + varuint32_from_bin(size, stream); + if (size > stream.remaining()) throw_error(stream_error::overrun); - input_stream objstream( stream.pos, size ); - stream.skip(size); - from_protobuf_object( obj, objstream ); - } else if constexpr ( 5 == wire_type::value or 1 == wire_type::value ) { - from_bin( obj, stream ); - } else if constexpr ( 0 == wire_type::value ) { - uint32_t v; - varuint32_from_bin( v, stream ); - obj = v; - } else { - T::from_protobuf_is_not_defined; /// used to generate useful compile error - } -} - -template -void from_protobuf_object( T& obj, S& stream ) { - while( stream.remaining() ) { - uint32_t key = 0; - varuint32_from_bin( key, stream ); - uint16_t wire_type = uint8_t(key) & 0x07; - uint32_t number = key >> 3; + input_stream objstream(stream.pos, size); + stream.skip(size); + from_protobuf_object(obj, objstream); + } + else if constexpr (5 == wire_type::value or 1 == wire_type::value) + { + from_bin(obj, stream); + } + else if constexpr (0 == wire_type::value) + { + uint32_t v; + varuint32_from_bin(v, stream); + obj = v; + } + else + { + T::from_protobuf_is_not_defined; /// used to generate useful compile error + } + } - //bool skip_it = true; - if( not reflect::get( number, [&]( auto m ) { - if constexpr ( std::is_member_function_pointer_v ) { - skip_member( wire_type, stream ); /// we cannot store return value of functions in T - } else { - //using member_type = std::decay_t; - //auto& member = obj.*m; - from_protobuf_member( obj.*m, stream ); - } - }) ) { // if not reflect::get(number...) - skip_member( wire_type, stream ); - } - } -} + template + void from_protobuf_object(T& obj, S& stream) + { + while (stream.remaining()) + { + uint32_t key = 0; + varuint32_from_bin(key, stream); + uint16_t wire_type = uint8_t(key) & 0x07; + uint32_t number = key >> 3; + + // bool skip_it = true; + if (not reflect::get(number, [&](auto m) { + if constexpr (std::is_member_function_pointer_v) + { + skip_member(wire_type, + stream); /// we cannot store return value of functions in T + } + else + { + // using member_type = std::decay_t; + // auto& member = obj.*m; + from_protobuf_member(obj.*m, stream); + } + })) + { // if not reflect::get(number...) + skip_member(wire_type, stream); + } + } + } -template -T from_protobuf( S& stream ) { - T tmp; - from_protobuf_object( tmp, stream ); - return tmp; -} + template + T from_protobuf(S& stream) + { + T tmp; + from_protobuf_object(tmp, stream); + return tmp; + } -template -T from_protobuf( const std::vector& in ) { - clio::input_stream stream(in.data(), in.size() ); - return clio::from_protobuf(stream); -} -template -T from_protobuf( std::vector& in ) { - clio::input_stream stream(in.data(), in.size() ); - return clio::from_protobuf(stream); -} + template + T from_protobuf(const std::vector& in) + { + clio::input_stream stream(in.data(), in.size()); + return clio::from_protobuf(stream); + } + template + T from_protobuf(std::vector& in) + { + clio::input_stream stream(in.data(), in.size()); + return clio::from_protobuf(stream); + } -} // namespace clio +} // namespace clio diff --git a/libraries/clio/include/clio/get_type_name.hpp b/libraries/clio/include/clio/get_type_name.hpp index 55c82d144..9225d7301 100644 --- a/libraries/clio/include/clio/get_type_name.hpp +++ b/libraries/clio/include/clio/get_type_name.hpp @@ -4,169 +4,205 @@ #include #include #include -#include #include +#include #include #include -namespace clio { - -#define CLIO_REFLECT_TYPENAME( T ) \ - constexpr const char* get_type_name( const T* ) { return BOOST_PP_STRINGIZE(T); } - -#define CLIO_REFLECT_TYPENAME_CUSTOM( T, CUSTOM ) \ - constexpr const char* get_type_name( const T* ) { return BOOST_PP_STRINGIZE(CUSTOM); } - -constexpr const char* get_type_name(const bool*) { return "bool"; } -constexpr const char* get_type_name(const int8_t*) { return "int8"; } -constexpr const char* get_type_name(const uint8_t*) { return "uint8"; } -constexpr const char* get_type_name(const int16_t*) { return "int16"; } -constexpr const char* get_type_name(const uint16_t*) { return "uint16"; } -constexpr const char* get_type_name(const int32_t*) { return "int32"; } -constexpr const char* get_type_name(const uint32_t*) { return "uint32"; } -constexpr const char* get_type_name(const int64_t*) { return "int64"; } -constexpr const char* get_type_name(const uint64_t*) { return "uint64"; } -constexpr const char* get_type_name(const float*) { return "float32"; } -constexpr const char* get_type_name(const double*) { return "double"; } -constexpr const char* get_type_name(const char*) { return "char"; } -constexpr const char* get_type_name(const std::string*) { return "string"; } -constexpr const char* get_type_name(const __int128*) { return "int128"; } -constexpr const char* get_type_name(const unsigned __int128*) { return "uint128"; } +namespace clio +{ +#define CLIO_REFLECT_TYPENAME(T) \ + constexpr const char* get_type_name(const T*) { return BOOST_PP_STRINGIZE(T); } + +#define CLIO_REFLECT_TYPENAME_CUSTOM(T, CUSTOM) \ + constexpr const char* get_type_name(const T*) { return BOOST_PP_STRINGIZE(CUSTOM); } + + constexpr const char* get_type_name(const bool*) { return "bool"; } + constexpr const char* get_type_name(const int8_t*) { return "int8"; } + constexpr const char* get_type_name(const uint8_t*) { return "uint8"; } + constexpr const char* get_type_name(const int16_t*) { return "int16"; } + constexpr const char* get_type_name(const uint16_t*) { return "uint16"; } + constexpr const char* get_type_name(const int32_t*) { return "int32"; } + constexpr const char* get_type_name(const uint32_t*) { return "uint32"; } + constexpr const char* get_type_name(const int64_t*) { return "int64"; } + constexpr const char* get_type_name(const uint64_t*) { return "uint64"; } + constexpr const char* get_type_name(const float*) { return "float32"; } + constexpr const char* get_type_name(const double*) { return "double"; } + constexpr const char* get_type_name(const char*) { return "char"; } + constexpr const char* get_type_name(const std::string*) { return "string"; } + constexpr const char* get_type_name(const __int128*) { return "int128"; } + constexpr const char* get_type_name(const unsigned __int128*) { return "uint128"; } #ifdef __eosio_cdt__ -constexpr const char* get_type_name(const long double*) { return "float128"; } + constexpr const char* get_type_name(const long double*) { return "float128"; } #endif -template -constexpr std::array array_cat(std::array lhs, std::array rhs) { - std::array result{}; - for (int i = 0; i < N; ++i) { result[i] = lhs[i]; } - for (int i = 0; i < M; ++i) { result[i + N] = rhs[i]; } - return result; -} - -template -constexpr std::array to_array(std::string_view s) { - std::array result{}; - for (int i = 0; i < N; ++i) { result[i] = s[i]; } - return result; -} - -template -constexpr auto append_type_name(const char (&suffix)[N]) { - constexpr std::string_view name = get_type_name((T*)nullptr); - return array_cat(to_array(name), to_array({ suffix, N })); -} - -template -constexpr auto vector_type_name = append_type_name("[]"); - -template -constexpr auto optional_type_name = append_type_name("?"); - -template -constexpr const char* get_type_name(const std::vector*) { - return vector_type_name.data(); -} - -template -constexpr const char* get_type_name(const std::optional*) { - return optional_type_name.data(); -} - -struct variant_type_appender { - char* buf; - constexpr variant_type_appender operator+(std::string_view s) { - *buf++ = '_'; - for (auto ch : s) *buf++ = ch; - return *this; + template + constexpr std::array array_cat(std::array lhs, std::array rhs) + { + std::array result{}; + for (int i = 0; i < N; ++i) + { + result[i] = lhs[i]; + } + for (int i = 0; i < M; ++i) + { + result[i + N] = rhs[i]; + } + return result; } -}; - -template -constexpr auto get_variant_type_name() { - constexpr std::size_t size = sizeof("variant") + ((std::string_view(get_type_name((T*)nullptr)).size() + 1) + ...); - std::array buffer{ 'v', 'a', 'r', 'i', 'a', 'n', 't' }; - (variant_type_appender{ buffer.data() + 7 } + ... + std::string_view(get_type_name((T*)nullptr))); - buffer[buffer.size() - 1] = '\0'; - return buffer; -} - -template -constexpr auto get_tuple_type_name() { - constexpr std::size_t size = sizeof("tuple") + ((std::string_view(get_type_name((T*)nullptr)).size() + 1) + ...); - std::array buffer{ 't', 'u', 'p', 'l', 'e' }; - (variant_type_appender{ buffer.data() + 5 } + ... + std::string_view(get_type_name((T*)nullptr))); - buffer[buffer.size() - 1] = '\0'; - return buffer; -} - -template -constexpr auto variant_type_name = get_variant_type_name(); - -template -constexpr const char* get_type_name(const std::variant*) { - return variant_type_name.data(); -} - -template -constexpr const char* get_type_name() { - return get_type_name( (const T*)nullptr ); -} - -[[nodiscard]] inline constexpr bool char_to_name_digit_strict(char c, uint64_t& result) { - if (c >= 'a' && c <= 'z') { - result = (c - 'a') + 6; - return true; + + template + constexpr std::array to_array(std::string_view s) + { + std::array result{}; + for (int i = 0; i < N; ++i) + { + result[i] = s[i]; + } + return result; } - if (c >= '1' && c <= '5') { - result = (c - '1') + 1; - return true; + + template + constexpr auto append_type_name(const char (&suffix)[N]) + { + constexpr std::string_view name = get_type_name((T*)nullptr); + return array_cat(to_array(name), to_array({suffix, N})); } - if (c == '.') { - result = 0; - return true; + + template + constexpr auto vector_type_name = append_type_name("[]"); + + template + constexpr auto optional_type_name = append_type_name("?"); + + template + constexpr const char* get_type_name(const std::vector*) + { + return vector_type_name.data(); } - return false; -} - -[[nodiscard]] inline constexpr bool string_to_name_strict(uint64_t& name, std::string_view str) { - name = 0; - unsigned i = 0; - for (; i < str.size() && i < 12; ++i) { - uint64_t x = 0; - // - this is not safe in const expression OUTCOME_TRY(char_to_name_digit_strict(str[i], x)); - auto r = char_to_name_digit_strict(str[i], x); - if( !r ) return false; - name |= (x & 0x1f) << (64 - 5 * (i + 1)); + + template + constexpr const char* get_type_name(const std::optional*) + { + return optional_type_name.data(); } - if (i < str.size() && i == 12) { - uint64_t x = 0; - // - this is not safe in const expression OUTCOME_TRY(char_to_name_digit_strict(str[i], x)); - auto r = char_to_name_digit_strict(str[i], x); - if( !r ) return false; - if (x != (x & 0xf)) - return false; - name |= x; - ++i; + struct variant_type_appender + { + char* buf; + constexpr variant_type_appender operator+(std::string_view s) + { + *buf++ = '_'; + for (auto ch : s) + *buf++ = ch; + return *this; + } + }; + + template + constexpr auto get_variant_type_name() + { + constexpr std::size_t size = + sizeof("variant") + ((std::string_view(get_type_name((T*)nullptr)).size() + 1) + ...); + std::array buffer{'v', 'a', 'r', 'i', 'a', 'n', 't'}; + (variant_type_appender{buffer.data() + 7} + ... + + std::string_view(get_type_name((T*)nullptr))); + buffer[buffer.size() - 1] = '\0'; + return buffer; + } + + template + constexpr auto get_tuple_type_name() + { + constexpr std::size_t size = + sizeof("tuple") + ((std::string_view(get_type_name((T*)nullptr)).size() + 1) + ...); + std::array buffer{'t', 'u', 'p', 'l', 'e'}; + (variant_type_appender{buffer.data() + 5} + ... + + std::string_view(get_type_name((T*)nullptr))); + buffer[buffer.size() - 1] = '\0'; + return buffer; + } + + template + constexpr auto variant_type_name = get_variant_type_name(); + + template + constexpr const char* get_type_name(const std::variant*) + { + return variant_type_name.data(); + } + + template + constexpr const char* get_type_name() + { + return get_type_name((const T*)nullptr); } - if (i < str.size()) + + [[nodiscard]] inline constexpr bool char_to_name_digit_strict(char c, uint64_t& result) + { + if (c >= 'a' && c <= 'z') + { + result = (c - 'a') + 6; + return true; + } + if (c >= '1' && c <= '5') + { + result = (c - '1') + 1; + return true; + } + if (c == '.') + { + result = 0; + return true; + } return false; - return true; -} - -inline constexpr uint64_t hash_name( std::string_view str ) { - uint64_t r = 0; - if( not string_to_name_strict(r, str) ) - return murmur64( str.data(), str.size() ); - return r; -} - -template -constexpr uint64_t get_type_hashname() { - return hash_name( get_type_name() ); -} - -} // namespace clio + } + + [[nodiscard]] inline constexpr bool string_to_name_strict(uint64_t& name, std::string_view str) + { + name = 0; + unsigned i = 0; + for (; i < str.size() && i < 12; ++i) + { + uint64_t x = 0; + // - this is not safe in const expression OUTCOME_TRY(char_to_name_digit_strict(str[i], x)); + auto r = char_to_name_digit_strict(str[i], x); + if (!r) + return false; + name |= (x & 0x1f) << (64 - 5 * (i + 1)); + } + if (i < str.size() && i == 12) + { + uint64_t x = 0; + // - this is not safe in const expression OUTCOME_TRY(char_to_name_digit_strict(str[i], x)); + auto r = char_to_name_digit_strict(str[i], x); + if (!r) + return false; + + if (x != (x & 0xf)) + return false; + name |= x; + ++i; + } + if (i < str.size()) + return false; + return true; + } + + inline constexpr uint64_t hash_name(std::string_view str) + { + uint64_t r = 0; + if (not string_to_name_strict(r, str)) + return murmur64(str.data(), str.size()); + return r; + } + + template + constexpr uint64_t get_type_hashname() + { + return hash_name(get_type_name()); + } + +} // namespace clio diff --git a/libraries/clio/include/clio/gql.hpp b/libraries/clio/include/clio/gql.hpp index 5211db444..0e17afca7 100644 --- a/libraries/clio/include/clio/gql.hpp +++ b/libraries/clio/include/clio/gql.hpp @@ -1,96 +1,103 @@ #pragma once -#include #include +#include -#include #include +#include -namespace clio { - - /** - * The concept of our graph-ql like interface is that a graph-ql query is - * parsed and converted into a gql query object and then flattened into something - * that requires no unpacking inorder to dispatch. This means converting gql strings - * into base32 integer representations and/or hashes if he name does not fit in - * base32. - * - * We use base32 (eg eosio::name) based 64bit integers as the key values because they - * map to swtich(key) statements for high performance dispatch by our resolvers. - */ - namespace gql { - - struct null_t{}; - struct entry; - struct scalar; - using object = std::vector; - using array = std::vector; - - - /** - * The scalar is the base value type, it is effectively a JSON object that uses - * base32 names for object keys and supports int64_t and uint64_t types in addition - * to double, string, object, array, and null. - * - * In principle you can convert from JSON to Scalar without any schema, and if you know - * the original keys for your objects, you can query the scalar for the value. In rare - * circumstances two keys may result in a hash collision, this should be detected at compile - * time and can be resolved by changing a field name. - * - * The flat view of a scalar is the same as a variant: - * [char_type][uint64_value] inline. If the type is a string,object,or array then the - * value field is a offset_ptr<> - * - */ - class scalar { - public: - scalar(){} - - template - scalar( const T& v ):value(v){} - - using value_type = std::variant; - - value_type value; - }; - using scalar_value = scalar::value_type; - CLIO_REFLECT_TYPENAME_CUSTOM( null_t, null ) - CLIO_REFLECT_TYPENAME_CUSTOM( scalar_value, scalar ) - - /** - * The key/value pair of an object - */ - struct entry { - uint64_t key; - scalar value; - }; - - - struct query; - struct query_filter; - - /** - * Defines what fields to select from the result of a query, - * if type is non-zero then the filter only applies if the type - * of the return value matches the variant type. - */ - struct query_filter { - std::vector filter; - /// the null type matches everything - uint64_t type = 0; - }; - - struct query { - uint64_t key; - uint64_t alias; - object args; - /// for each type... define the filter... - std::vector filter; - }; - CLIO_REFLECT( scalar, value ) - CLIO_REFLECT( entry, key, value ) - CLIO_REFLECT( query_filter, filter, type ) - CLIO_REFLECT( query, key, alias, args, filter ) - - } // namespace graphql - -} // namespace clio +namespace clio +{ + /** + * The concept of our graph-ql like interface is that a graph-ql query is + * parsed and converted into a gql query object and then flattened into something + * that requires no unpacking inorder to dispatch. This means converting gql strings + * into base32 integer representations and/or hashes if he name does not fit in + * base32. + * + * We use base32 (eg eosio::name) based 64bit integers as the key values because they + * map to swtich(key) statements for high performance dispatch by our resolvers. + */ + namespace gql + { + struct null_t + { + }; + struct entry; + struct scalar; + using object = std::vector; + using array = std::vector; + + /** + * The scalar is the base value type, it is effectively a JSON object that uses + * base32 names for object keys and supports int64_t and uint64_t types in addition + * to double, string, object, array, and null. + * + * In principle you can convert from JSON to Scalar without any schema, and if you know + * the original keys for your objects, you can query the scalar for the value. In rare + * circumstances two keys may result in a hash collision, this should be detected at compile + * time and can be resolved by changing a field name. + * + * The flat view of a scalar is the same as a variant: + * [char_type][uint64_value] inline. If the type is a string,object,or array then the + * value field is a offset_ptr<> + * + */ + class scalar + { + public: + scalar() {} + + template + scalar(const T& v) : value(v) + { + } + + using value_type = + std::variant; + + value_type value; + }; + using scalar_value = scalar::value_type; + CLIO_REFLECT_TYPENAME_CUSTOM(null_t, null) + CLIO_REFLECT_TYPENAME_CUSTOM(scalar_value, scalar) + + /** + * The key/value pair of an object + */ + struct entry + { + uint64_t key; + scalar value; + }; + + struct query; + struct query_filter; + + /** + * Defines what fields to select from the result of a query, + * if type is non-zero then the filter only applies if the type + * of the return value matches the variant type. + */ + struct query_filter + { + std::vector filter; + /// the null type matches everything + uint64_t type = 0; + }; + + struct query + { + uint64_t key; + uint64_t alias; + object args; + /// for each type... define the filter... + std::vector filter; + }; + CLIO_REFLECT(scalar, value) + CLIO_REFLECT(entry, key, value) + CLIO_REFLECT(query_filter, filter, type) + CLIO_REFLECT(query, key, alias, args, filter) + + } // namespace gql + +} // namespace clio diff --git a/libraries/clio/include/clio/json/any.hpp b/libraries/clio/include/clio/json/any.hpp index e2e8bf84e..8f9f6587e 100644 --- a/libraries/clio/include/clio/json/any.hpp +++ b/libraries/clio/include/clio/json/any.hpp @@ -1,188 +1,232 @@ #pragma once +#include #include #include -#include -namespace clio { - - namespace json { - struct any; - struct entry; - struct null_t{};// uint8_t none = 0; }; - struct error_t{ std::string what; }; - - using any_object = std::vector; - using any_array = std::vector; - - struct any { - public: - any(){}; - template - any( const T& v ):_value(v){} - - template - const T* get_if()const { return std::get_if(&_value); } - - template - const T& as()const { - if( auto p = std::get_if(&_value) ) - return *p; - - assert( !"invalid type cast from any" ); - // static T dummy; /// used to prevent warnings about no return - // return dummy; - } - template - T& as(){ - if( auto p = std::get_if(&_value) ) - return *p; - - assert( !"invalid type cast from any" ); - __builtin_unreachable(); - } - - const auto& value()const { return _value; } - auto& value(){ return _value; } - - inline any operator[]( const std::string_view& key )const; - any operator[]( uint32_t index )const { - if( auto p = get_if() ) { - if( index >= p->size() ) return error_t{ "index out of bounds" }; - return p->at(index); - } - return error_t{ "not an array" }; - } - - template - void visit( Lambda&& l )const { - std::visit( std::forward(l), _value ); - } - private: - std::variant _value; - - template - friend void to_bin(const any& obj, S& stream) { - to_bin( obj._value, stream ); - } - template - friend void from_bin( any& obj, S& stream) { - from_bin( obj._value, stream ); - } - }; - - struct entry { - std::string key; - any value; - }; - CLIO_REFLECT( entry, key, value ) - CLIO_REFLECT_TYPENAME( null_t ) - CLIO_REFLECT( error_t, what ) - - - inline any any::operator[]( const std::string_view& key )const { - if( auto p = get_if() ) { - for( const auto& i : *p ) { - if( i.key == key ) return i.value; - } - } - return error_t{"not an object"}; - } - - template - void to_json(const any& obj, S& stream) { - std::visit( [&]( const auto& val ) { - using T = std::decay_t; - if constexpr ( std::is_same_v ) { - stream.write( "null", 4 ); - } - else if constexpr ( std::is_same_v ) { - stream.write( '{' ); - int size = val.size(); - for( const auto& e : val ) { - if( size == val.size() ) { - increase_indent(stream); - } - write_newline(stream); - to_json(e.key, stream); - write_colon(stream); - to_json(e.value, stream); - if( --size ) { - stream.write( ',' ); - } - } - if( !val.empty() ) { - decrease_indent(stream); - write_newline(stream); - } - stream.write( '}' ); - } else { - to_json( val, stream ); - } - }, obj.value() ); - } - - template - void from_json(any& obj, S& stream); - - template - void from_json(any_object& obj, S& stream) { - stream.get_start_object(); - while( true ) { - auto t = stream.peek_token(); - if (t.get().type == json_token_type::type_end_object) - break; - auto k = stream.get_key(); - obj.push_back( { .key = std::string(k) } ); - from_json( obj.back().value, stream ); - } - return stream.get_end_object(); - } - - template - void from_json(any& obj, S& stream) { - while( true ) { - auto t = stream.peek_token(); - switch( t.get().type ) { - case json_token_type::type_null: - stream.eat_token(); - obj = null_t{}; - return; - case json_token_type::type_bool: - obj = t.get().value_bool; - stream.eat_token(); - return; - case json_token_type::type_string: - obj = std::string(t.get().value_string); - stream.eat_token(); - return; - case json_token_type::type_start_object: { - obj = any_object(); - from_json( obj.as(), stream ); - return; +namespace clio +{ + namespace json + { + struct any; + struct entry; + struct null_t + { + }; // uint8_t none = 0; }; + struct error_t + { + std::string what; + }; + + using any_object = std::vector; + using any_array = std::vector; + + struct any + { + public: + any(){}; + template + any(const T& v) : _value(v) + { + } + + template + const T* get_if() const + { + return std::get_if(&_value); + } + + template + const T& as() const + { + if (auto p = std::get_if(&_value)) + return *p; + + assert(!"invalid type cast from any"); + // static T dummy; /// used to prevent warnings about no return + // return dummy; + } + template + T& as() + { + if (auto p = std::get_if(&_value)) + return *p; + + assert(!"invalid type cast from any"); + __builtin_unreachable(); + } + + const auto& value() const { return _value; } + auto& value() { return _value; } + + inline any operator[](const std::string_view& key) const; + any operator[](uint32_t index) const + { + if (auto p = get_if()) + { + if (index >= p->size()) + return error_t{"index out of bounds"}; + return p->at(index); + } + return error_t{"not an array"}; + } + + template + void visit(Lambda&& l) const + { + std::visit(std::forward(l), _value); + } + + private: + std::variant + _value; + + template + friend void to_bin(const any& obj, S& stream) + { + to_bin(obj._value, stream); + } + template + friend void from_bin(any& obj, S& stream) + { + from_bin(obj._value, stream); + } + }; + + struct entry + { + std::string key; + any value; + }; + CLIO_REFLECT(entry, key, value) + CLIO_REFLECT_TYPENAME(null_t) + CLIO_REFLECT(error_t, what) + + inline any any::operator[](const std::string_view& key) const + { + if (auto p = get_if()) + { + for (const auto& i : *p) + { + if (i.key == key) + return i.value; + } + } + return error_t{"not an object"}; + } + + template + void to_json(const any& obj, S& stream) + { + std::visit( + [&](const auto& val) { + using T = std::decay_t; + if constexpr (std::is_same_v) + { + stream.write("null", 4); + } + else if constexpr (std::is_same_v) + { + stream.write('{'); + int size = val.size(); + for (const auto& e : val) + { + if (size == val.size()) + { + increase_indent(stream); + } + write_newline(stream); + to_json(e.key, stream); + write_colon(stream); + to_json(e.value, stream); + if (--size) + { + stream.write(','); + } } - case json_token_type::type_start_array: { - obj = any_array(); - from_json( obj.as(), stream ); - return; + if (!val.empty()) + { + decrease_indent(stream); + write_newline(stream); } - case json_token_type::type_end_array: - case json_token_type::type_end_object: - case json_token_type::type_key: - default: - stream.eat_token(); - throw_error(clio::from_json_error::value_invalid); - }; - } - } - - } /// namespace json - -} /// clio + stream.write('}'); + } + else + { + to_json(val, stream); + } + }, + obj.value()); + } + + template + void from_json(any& obj, S& stream); + + template + void from_json(any_object& obj, S& stream) + { + stream.get_start_object(); + while (true) + { + auto t = stream.peek_token(); + if (t.get().type == json_token_type::type_end_object) + break; + auto k = stream.get_key(); + obj.push_back({.key = std::string(k)}); + from_json(obj.back().value, stream); + } + return stream.get_end_object(); + } + + template + void from_json(any& obj, S& stream) + { + while (true) + { + auto t = stream.peek_token(); + switch (t.get().type) + { + case json_token_type::type_null: + stream.eat_token(); + obj = null_t{}; + return; + case json_token_type::type_bool: + obj = t.get().value_bool; + stream.eat_token(); + return; + case json_token_type::type_string: + obj = std::string(t.get().value_string); + stream.eat_token(); + return; + case json_token_type::type_start_object: + { + obj = any_object(); + from_json(obj.as(), stream); + return; + } + case json_token_type::type_start_array: + { + obj = any_array(); + from_json(obj.as(), stream); + return; + } + case json_token_type::type_end_array: + case json_token_type::type_end_object: + case json_token_type::type_key: + default: + stream.eat_token(); + throw_error(clio::from_json_error::value_invalid); + }; + } + } + + } // namespace json + +} // namespace clio diff --git a/libraries/clio/include/clio/member_proxy.hpp b/libraries/clio/include/clio/member_proxy.hpp index b1a2a8deb..6e080daba 100644 --- a/libraries/clio/include/clio/member_proxy.hpp +++ b/libraries/clio/include/clio/member_proxy.hpp @@ -1,86 +1,107 @@ #pragma once -namespace clio { +namespace clio +{ + template + struct member_proxy + { + /** + * This object is created on a type created by the macro, I represents the member number in + * the containing type. The containing type's first member is a ProxyObject. This function + * does the pointer math necessary to find the ProxyObject. + * + * Alternatively the macro code would have to initialize every member_proxy with this, which + * would bloat the size of the member_proxy object + * + * flat_view => reflect::member_proxy + * + * struct member_proxy { + * proxy_impl proxy___; + * member_proxy<0, ptr, proxy_impl> member_zero + * member_proxy<1, ptr, proxy_impl> member_one + * member_proxy<2, ptr, proxy_impl> member_two + * } + * a packed flat buffer. + * + * Let char* buf = point to a flat buffer; + * Let reinterpet buf as memper_proxy*, this makes the address of proxy__ equal + * to the address of member_proxy because it is the first element. + * + * because member_proxy has no values it takes 1 byte in member_proxy and the value of that + * byte is never read by member_proxy... member_proxy always gets the address of proxy___ and + * then does offset math. + * + */ + constexpr auto proxyptr() const + { + return (reinterpret_cast(reinterpret_cast(this) - + sizeof(*this) * (I + 1))); + } + constexpr auto proxyptr() + { + return (reinterpret_cast(reinterpret_cast(this) - + sizeof(*this) * (I + 1))); + } + constexpr const auto& get() const { return *(proxyptr()->template get()); } + constexpr auto& get() { return *(proxyptr()->template get()); } - template - struct member_proxy { - /** - * This object is created on a type created by the macro, I represents the member number in the - * containing type. The containing type's first member is a ProxyObject. This function does the - * pointer math necessary to find the ProxyObject. - * - * Alternatively the macro code would have to initialize every member_proxy with this, which would - * bloat the size of the member_proxy object - * - * flat_view => reflect::member_proxy - * - * struct member_proxy { - * proxy_impl proxy___; - * member_proxy<0, ptr, proxy_impl> member_zero - * member_proxy<1, ptr, proxy_impl> member_one - * member_proxy<2, ptr, proxy_impl> member_two - * } - * a packed flat buffer. - * - * Let char* buf = point to a flat buffer; - * Let reinterpet buf as memper_proxy*, this makes the address of proxy__ equal to - * the address of member_proxy because it is the first element. - * - * because member_proxy has no values it takes 1 byte in member_proxy and the value of that byte - * is never read by member_proxy... member_proxy always gets the address of proxy___ and then - * does offset math. - * - */ - constexpr auto proxyptr()const { - return (reinterpret_cast( reinterpret_cast(this)-sizeof(*this)*(I+1)) ); - } - constexpr auto proxyptr(){ - return (reinterpret_cast( reinterpret_cast(this)-sizeof(*this)*(I+1)) ); - } - constexpr const auto& get()const { return *(proxyptr()->template get()); } - constexpr auto& get() { return *(proxyptr()->template get()); } + template + constexpr auto operator()(Ts&&... args) + { + return proxyptr()->template call(std::forward(args)...); + } + template + constexpr auto operator()(Ts&&... args) const + { + return proxyptr()->template call(std::forward(args)...); + } - template - constexpr auto operator()(Ts&&... args) { - return proxyptr()->template call( std::forward(args)...); - } - template - constexpr auto operator()(Ts&&... args)const { - return proxyptr()->template call( std::forward(args)...); - } + constexpr auto operator->() { return (proxyptr()->template get()); } + constexpr const auto operator->() const + { + return (proxyptr()->template get()); + } - constexpr auto operator->(){ return (proxyptr()->template get()); } - constexpr const auto operator->()const { return (proxyptr()->template get()); } + constexpr auto& operator*() { return get(); } + constexpr const auto& operator*() const { return get(); } - constexpr auto& operator*(){ return get(); } - constexpr const auto& operator*()const { return get(); } + template + constexpr auto& operator[](T&& k) + { + return get()[std::forward(k)]; + } - template - constexpr auto& operator[]( T&& k ) { return get()[ std::forward(k)]; } + template + constexpr const auto& operator[](T&& k) const + { + return get()[std::forward(k)]; + } - template - constexpr const auto& operator[]( T&& k )const { return get()[ std::forward(k)]; } + template + friend S& operator<<(S& stream, const member_proxy& member) + { + return stream << member.get(); + } - template - friend S& operator << ( S& stream, const member_proxy& member) { - return stream << member.get(); - } + template + auto operator=(R&& r) + { + get() = std::forward(r); + return *this; + } - template - auto operator=( R&& r ) { - get() = std::forward(r); - return *this; - } + template + operator R() const + { + return get(); + } - template - operator R ()const { return get(); } - - /* + /* operator decltype( ((ProxyObject*)nullptr)->template get())() { return get(); } operator decltype( ((const ProxyObject*)nullptr)->template get()) ()const { return get(); } */ - }; + }; -} /// clio +} // namespace clio diff --git a/libraries/clio/include/clio/murmur.hpp b/libraries/clio/include/clio/murmur.hpp index 9a231afa7..df273e19e 100644 --- a/libraries/clio/include/clio/murmur.hpp +++ b/libraries/clio/include/clio/murmur.hpp @@ -1,55 +1,58 @@ -#pragma once -namespace clio { -namespace { - inline constexpr uint64_t unaligned_load(const char* p) - { - uint64_t r = 0; - for( uint32_t i = 0; i < 8; ++i ) { - r |= p[i]; - r <<= 8; - } - return r; - } +#pragma once +namespace clio +{ + namespace + { + inline constexpr uint64_t unaligned_load(const char* p) + { + uint64_t r = 0; + for (uint32_t i = 0; i < 8; ++i) + { + r |= p[i]; + r <<= 8; + } + return r; + } - // Loads n bytes, where 1 <= n < 8. - inline constexpr uint64_t load_bytes(const char* p, int n) - { - uint64_t result = 0; - --n; - do - result = (result << 8) + (unsigned char)(p[n]); - while (--n >= 0); - return result; - } + // Loads n bytes, where 1 <= n < 8. + inline constexpr uint64_t load_bytes(const char* p, int n) + { + uint64_t result = 0; + --n; + do + result = (result << 8) + (unsigned char)(p[n]); + while (--n >= 0); + return result; + } - inline constexpr uint64_t shift_mix(std::uint64_t v) - { return v ^ (v >> 47);} -} // namespace + inline constexpr uint64_t shift_mix(std::uint64_t v) { return v ^ (v >> 47); } + } // namespace -// Implementation of Murmur hash for 64-bit size_t. - inline constexpr uint64_t murmur64(const char* ptr, uint64_t len, uint64_t seed = 0xbadd00d00) - { - const uint64_t mul = (((uint64_t) 0xc6a4a793UL) << 32UL) - + (uint64_t) 0x5bd1e995UL; - const char* const buf = ptr; + // Implementation of Murmur hash for 64-bit size_t. + inline constexpr uint64_t murmur64(const char* ptr, uint64_t len, uint64_t seed = 0xbadd00d00) + { + const uint64_t mul = (((uint64_t)0xc6a4a793UL) << 32UL) + (uint64_t)0x5bd1e995UL; + const char* const buf = ptr; - // Remove the bytes not divisible by the sizeof(uint64_t). This - // allows the main loop to process the data as 64-bit integers. - const uint64_t len_aligned = len & ~(uint64_t)0x7; - const char* const end = buf + len_aligned; - uint64_t hash = seed ^ (len * mul); - for (const char* p = buf; p != end; p += 8) { - const uint64_t data = shift_mix(unaligned_load(p) * mul) * mul; - hash ^= data; - hash *= mul; - } - if ((len & 0x7) != 0) { - const uint64_t data = load_bytes(end, len & 0x7); - hash ^= data; - hash *= mul; - } - hash = shift_mix(hash) * mul; - hash = shift_mix(hash); - return hash; - } -} // namespace eosio + // Remove the bytes not divisible by the sizeof(uint64_t). This + // allows the main loop to process the data as 64-bit integers. + const uint64_t len_aligned = len & ~(uint64_t)0x7; + const char* const end = buf + len_aligned; + uint64_t hash = seed ^ (len * mul); + for (const char* p = buf; p != end; p += 8) + { + const uint64_t data = shift_mix(unaligned_load(p) * mul) * mul; + hash ^= data; + hash *= mul; + } + if ((len & 0x7) != 0) + { + const uint64_t data = load_bytes(end, len & 0x7); + hash ^= data; + hash *= mul; + } + hash = shift_mix(hash) * mul; + hash = shift_mix(hash); + return hash; + } +} // namespace clio diff --git a/libraries/clio/include/clio/name.hpp b/libraries/clio/include/clio/name.hpp index b52050973..724953c2f 100644 --- a/libraries/clio/include/clio/name.hpp +++ b/libraries/clio/include/clio/name.hpp @@ -1,198 +1,236 @@ #pragma once -#include #include #include +#include -namespace clio { +namespace clio +{ + inline std::string name_to_string(uint64_t name) + { + static const char* charmap = ".12345abcdefghijklmnopqrstuvwxyz"; + std::string str(13, '.'); + + uint64_t tmp = name; + for (uint32_t i = 0; i <= 12; ++i) + { + char c = charmap[tmp & (i == 0 ? 0x0f : 0x1f)]; + str[12 - i] = c; + tmp >>= (i == 0 ? 4 : 5); + } -inline std::string name_to_string(uint64_t name) { - static const char* charmap = ".12345abcdefghijklmnopqrstuvwxyz"; - std::string str(13, '.'); + const auto last = str.find_last_not_of('.'); + return str.substr(0, last + 1); + } - uint64_t tmp = name; - for (uint32_t i = 0; i <= 12; ++i) { - char c = charmap[tmp & (i == 0 ? 0x0f : 0x1f)]; - str[12 - i] = c; - tmp >>= (i == 0 ? 4 : 5); + inline constexpr uint64_t char_to_name_digit(char c) + { + if (c >= 'a' && c <= 'z') + return (c - 'a') + 6; + if (c >= '1' && c <= '5') + return (c - '1') + 1; + return 0; } - const auto last = str.find_last_not_of('.'); - return str.substr(0, last + 1); -} - -inline constexpr uint64_t char_to_name_digit(char c) { - if (c >= 'a' && c <= 'z') - return (c - 'a') + 6; - if (c >= '1' && c <= '5') - return (c - '1') + 1; - return 0; -} - -inline constexpr uint64_t string_to_name(const char* str, int size) { - uint64_t name = 0; - int i = 0; - for (; i < size && i < 12; ++i) name |= (char_to_name_digit(str[i]) & 0x1f) << (64 - 5 * (i + 1)); - if (i < size) - name |= char_to_name_digit(str[i]) & 0x0F; - return name; -} - -inline constexpr uint64_t string_to_name(const char* str) { - int len = 0; - while (str[len]) ++len; - return string_to_name(str, len); -} - -inline constexpr uint64_t string_to_name(std::string_view str) { return string_to_name(str.data(), str.size()); } - -struct name { - enum class raw : uint64_t {}; - uint64_t value = 0; - - constexpr name() = default; - constexpr explicit name(uint64_t value) : value{ value } {} - constexpr explicit name(name::raw value) : value{ static_cast(value) } {} - constexpr explicit name(std::string_view str) { - if( not string_to_name_strict( value, str ) ) { - // TODO: throw_error - } + inline constexpr uint64_t string_to_name(const char* str, int size) + { + uint64_t name = 0; + int i = 0; + for (; i < size && i < 12; ++i) + name |= (char_to_name_digit(str[i]) & 0x1f) << (64 - 5 * (i + 1)); + if (i < size) + name |= char_to_name_digit(str[i]) & 0x0F; + return name; } + inline constexpr uint64_t string_to_name(const char* str) + { + int len = 0; + while (str[len]) + ++len; + return string_to_name(str, len); + } - constexpr name(const name&) = default; - - constexpr operator raw() const { return static_cast(value); } - explicit operator std::string() const { return name_to_string(value); } - std::string to_string() const { return std::string(*this); } - /** - * Explicit cast to bool of the uint64_t value of the name - * - * @return Returns true if the name is set to the default value of 0 else true. - */ - constexpr explicit operator bool() const { return value != 0; } - - /** - * Converts a %name Base32 symbol into its corresponding value - * - * @param c - Character to be converted - * @return constexpr char - Converted value - */ - static constexpr uint8_t char_to_value(char c) { - if (c == '.') - return 0; - else if (c >= '1' && c <= '5') - return (c - '1') + 1; - else if (c >= 'a' && c <= 'z') - return (c - 'a') + 6; - return 0; // control flow will never reach here; just added to suppress warning + inline constexpr uint64_t string_to_name(std::string_view str) + { + return string_to_name(str.data(), str.size()); } - /** - * Returns the length of the %name - */ - constexpr uint8_t length() const { - constexpr uint64_t mask = 0xF800000000000000ull; + struct name + { + enum class raw : uint64_t + { + }; + uint64_t value = 0; + + constexpr name() = default; + constexpr explicit name(uint64_t value) : value{value} {} + constexpr explicit name(name::raw value) : value{static_cast(value)} {} + constexpr explicit name(std::string_view str) + { + if (not string_to_name_strict(value, str)) + { + // TODO: throw_error + } + } - if (value == 0) - return 0; + constexpr name(const name&) = default; + + constexpr operator raw() const { return static_cast(value); } + explicit operator std::string() const { return name_to_string(value); } + std::string to_string() const { return std::string(*this); } + /** + * Explicit cast to bool of the uint64_t value of the name + * + * @return Returns true if the name is set to the default value of 0 else true. + */ + constexpr explicit operator bool() const { return value != 0; } + + /** + * Converts a %name Base32 symbol into its corresponding value + * + * @param c - Character to be converted + * @return constexpr char - Converted value + */ + static constexpr uint8_t char_to_value(char c) + { + if (c == '.') + return 0; + else if (c >= '1' && c <= '5') + return (c - '1') + 1; + else if (c >= 'a' && c <= 'z') + return (c - 'a') + 6; + return 0; // control flow will never reach here; just added to suppress warning + } - uint8_t l = 0; - uint8_t i = 0; - for (auto v = value; i < 13; ++i, v <<= 5) { - if ((v & mask) > 0) { - l = i; + /** + * Returns the length of the %name + */ + constexpr uint8_t length() const + { + constexpr uint64_t mask = 0xF800000000000000ull; + + if (value == 0) + return 0; + + uint8_t l = 0; + uint8_t i = 0; + for (auto v = value; i < 13; ++i, v <<= 5) + { + if ((v & mask) > 0) + { + l = i; + } } + + return l + 1; } - return l + 1; - } + /** + * Returns the suffix of the %name + */ + constexpr name suffix() const + { + uint32_t remaining_bits_after_last_actual_dot = 0; + uint32_t tmp = 0; + for (int32_t remaining_bits = 59; remaining_bits >= 4; remaining_bits -= 5) + { // Note: remaining_bits must remain signed integer + // Get characters one-by-one in name in order from left to right (not including the 13th + // character) + auto c = (value >> remaining_bits) & 0x1Full; + if (!c) + { // if this character is a dot + tmp = static_cast(remaining_bits); + } + else + { // if this character is not a dot + remaining_bits_after_last_actual_dot = tmp; + } + } - /** - * Returns the suffix of the %name - */ - constexpr name suffix() const { - uint32_t remaining_bits_after_last_actual_dot = 0; - uint32_t tmp = 0; - for (int32_t remaining_bits = 59; remaining_bits >= 4; - remaining_bits -= 5) { // Note: remaining_bits must remain signed integer - // Get characters one-by-one in name in order from left to right (not including the 13th character) - auto c = (value >> remaining_bits) & 0x1Full; - if (!c) { // if this character is a dot - tmp = static_cast(remaining_bits); - } else { // if this character is not a dot + uint64_t thirteenth_character = value & 0x0Full; + if (thirteenth_character) + { // if 13th character is not a dot remaining_bits_after_last_actual_dot = tmp; } - } - uint64_t thirteenth_character = value & 0x0Full; - if (thirteenth_character) { // if 13th character is not a dot - remaining_bits_after_last_actual_dot = tmp; - } + if (remaining_bits_after_last_actual_dot == + 0) // there is no actual dot in the %name other than potentially leading dots + return name{value}; - if (remaining_bits_after_last_actual_dot == - 0) // there is no actual dot in the %name other than potentially leading dots - return name{ value }; + // At this point remaining_bits_after_last_actual_dot has to be within the range of 4 to 59 + // (and restricted to increments of 5). - // At this point remaining_bits_after_last_actual_dot has to be within the range of 4 to 59 (and restricted to - // increments of 5). + // Mask for remaining bits corresponding to characters after last actual dot, except for 4 + // least significant bits (corresponds to 13th character). + uint64_t mask = (1ull << remaining_bits_after_last_actual_dot) - 16; + uint32_t shift = 64 - remaining_bits_after_last_actual_dot; - // Mask for remaining bits corresponding to characters after last actual dot, except for 4 least significant bits - // (corresponds to 13th character). - uint64_t mask = (1ull << remaining_bits_after_last_actual_dot) - 16; - uint32_t shift = 64 - remaining_bits_after_last_actual_dot; + return name{((value & mask) << shift) + (thirteenth_character << (shift - 1))}; + } - return name{ ((value & mask) << shift) + (thirteenth_character << (shift - 1)) }; - } + /** + * Returns the prefix of the %name + */ + constexpr name prefix() const + { + uint64_t result = value; + bool not_dot_character_seen = false; + uint64_t mask = 0xFull; + + // Get characters one-by-one in name in order from right to left + for (int32_t offset = 0; offset <= 59;) + { + auto c = (value >> offset) & mask; + + if (!c) + { // if this character is a dot + if (not_dot_character_seen) + { // we found the rightmost dot character + result = (value >> offset) << offset; + break; + } + } + else + { + not_dot_character_seen = true; + } - /** - * Returns the prefix of the %name - */ - constexpr name prefix() const { - uint64_t result = value; - bool not_dot_character_seen = false; - uint64_t mask = 0xFull; - - // Get characters one-by-one in name in order from right to left - for (int32_t offset = 0; offset <= 59;) { - auto c = (value >> offset) & mask; - - if (!c) { // if this character is a dot - if (not_dot_character_seen) { // we found the rightmost dot character - result = (value >> offset) << offset; - break; + if (offset == 0) + { + offset += 4; + mask = 0x1Full; + } + else + { + offset += 5; } - } else { - not_dot_character_seen = true; } - if (offset == 0) { - offset += 4; - mask = 0x1Full; - } else { - offset += 5; - } + return name{result}; } - - return name{ result }; + }; + CLIO_REFLECT(name, value) + + template + void from_json(name& obj, S& stream) + { + auto r = stream.get_string(); + obj = name(hash_name(r)); } -}; -CLIO_REFLECT( name, value ) - -template -void from_json(name& obj, S& stream) { - auto r = stream.get_string(); - obj = name(hash_name(r)); -} - -template -void to_json(const name& obj, S& stream) { - to_json(name_to_string(obj.value), stream); -} + template + void to_json(const name& obj, S& stream) + { + to_json(name_to_string(obj.value), stream); + } -inline namespace literals { - inline constexpr name operator""_n(const char* s, size_t) { return name( std::string_view(s) ); } - inline constexpr name operator""_h(const char* s, size_t) { return name( hash_name(s) ); } -} // namespace literals + inline namespace literals + { + inline constexpr name operator""_n(const char* s, size_t) + { + return name(std::string_view(s)); + } + inline constexpr name operator""_h(const char* s, size_t) { return name(hash_name(s)); } + } // namespace literals -} // namespace clio +} // namespace clio diff --git a/libraries/clio/include/clio/powers.h b/libraries/clio/include/clio/powers.h index d2acf6707..c4e249592 100644 --- a/libraries/clio/include/clio/powers.h +++ b/libraries/clio/include/clio/powers.h @@ -14,58 +14,62 @@ #define expmax -32 #define expmin -60 -typedef struct Fp { +typedef struct Fp +{ uint64_t frac; - int exp; + int exp; } Fp; static constexpr Fp powers_ten[] = { - { 18054884314459144840U, -1220 }, { 13451937075301367670U, -1193 }, { 10022474136428063862U, -1166 }, - { 14934650266808366570U, -1140 }, { 11127181549972568877U, -1113 }, { 16580792590934885855U, -1087 }, - { 12353653155963782858U, -1060 }, { 18408377700990114895U, -1034 }, { 13715310171984221708U, -1007 }, - { 10218702384817765436U, -980 }, { 15227053142812498563U, -954 }, { 11345038669416679861U, -927 }, - { 16905424996341287883U, -901 }, { 12595523146049147757U, -874 }, { 9384396036005875287U, -847 }, - { 13983839803942852151U, -821 }, { 10418772551374772303U, -794 }, { 15525180923007089351U, -768 }, - { 11567161174868858868U, -741 }, { 17236413322193710309U, -715 }, { 12842128665889583758U, -688 }, - { 9568131466127621947U, -661 }, { 14257626930069360058U, -635 }, { 10622759856335341974U, -608 }, - { 15829145694278690180U, -582 }, { 11793632577567316726U, -555 }, { 17573882009934360870U, -529 }, - { 13093562431584567480U, -502 }, { 9755464219737475723U, -475 }, { 14536774485912137811U, -449 }, - { 10830740992659433045U, -422 }, { 16139061738043178685U, -396 }, { 12024538023802026127U, -369 }, - { 17917957937422433684U, -343 }, { 13349918974505688015U, -316 }, { 9946464728195732843U, -289 }, - { 14821387422376473014U, -263 }, { 11042794154864902060U, -236 }, { 16455045573212060422U, -210 }, - { 12259964326927110867U, -183 }, { 18268770466636286478U, -157 }, { 13611294676837538539U, -130 }, - { 10141204801825835212U, -103 }, { 15111572745182864684U, -77 }, { 11258999068426240000U, -50 }, - { 16777216000000000000U, -24 }, { 12500000000000000000U, 3 }, { 9313225746154785156U, 30 }, - { 13877787807814456755U, 56 }, { 10339757656912845936U, 83 }, { 15407439555097886824U, 109 }, - { 11479437019748901445U, 136 }, { 17105694144590052135U, 162 }, { 12744735289059618216U, 189 }, - { 9495567745759798747U, 216 }, { 14149498560666738074U, 242 }, { 10542197943230523224U, 269 }, - { 15709099088952724970U, 295 }, { 11704190886730495818U, 322 }, { 17440603504673385349U, 348 }, - { 12994262207056124023U, 375 }, { 9681479787123295682U, 402 }, { 14426529090290212157U, 428 }, - { 10748601772107342003U, 455 }, { 16016664761464807395U, 481 }, { 11933345169920330789U, 508 }, - { 17782069995880619868U, 534 }, { 13248674568444952270U, 561 }, { 9871031767461413346U, 588 }, - { 14708983551653345445U, 614 }, { 10959046745042015199U, 641 }, { 16330252207878254650U, 667 }, - { 12166986024289022870U, 694 }, { 18130221999122236476U, 720 }, { 13508068024458167312U, 747 }, - { 10064294952495520794U, 774 }, { 14996968138956309548U, 800 }, { 11173611982879273257U, 827 }, - { 16649979327439178909U, 853 }, { 12405201291620119593U, 880 }, { 9242595204427927429U, 907 }, - { 13772540099066387757U, 933 }, { 10261342003245940623U, 960 }, { 15290591125556738113U, 986 }, - { 11392378155556871081U, 1013 }, { 16975966327722178521U, 1039 }, { 12648080533535911531U, 1066 } -}; + {18054884314459144840U, -1220}, {13451937075301367670U, -1193}, {10022474136428063862U, -1166}, + {14934650266808366570U, -1140}, {11127181549972568877U, -1113}, {16580792590934885855U, -1087}, + {12353653155963782858U, -1060}, {18408377700990114895U, -1034}, {13715310171984221708U, -1007}, + {10218702384817765436U, -980}, {15227053142812498563U, -954}, {11345038669416679861U, -927}, + {16905424996341287883U, -901}, {12595523146049147757U, -874}, {9384396036005875287U, -847}, + {13983839803942852151U, -821}, {10418772551374772303U, -794}, {15525180923007089351U, -768}, + {11567161174868858868U, -741}, {17236413322193710309U, -715}, {12842128665889583758U, -688}, + {9568131466127621947U, -661}, {14257626930069360058U, -635}, {10622759856335341974U, -608}, + {15829145694278690180U, -582}, {11793632577567316726U, -555}, {17573882009934360870U, -529}, + {13093562431584567480U, -502}, {9755464219737475723U, -475}, {14536774485912137811U, -449}, + {10830740992659433045U, -422}, {16139061738043178685U, -396}, {12024538023802026127U, -369}, + {17917957937422433684U, -343}, {13349918974505688015U, -316}, {9946464728195732843U, -289}, + {14821387422376473014U, -263}, {11042794154864902060U, -236}, {16455045573212060422U, -210}, + {12259964326927110867U, -183}, {18268770466636286478U, -157}, {13611294676837538539U, -130}, + {10141204801825835212U, -103}, {15111572745182864684U, -77}, {11258999068426240000U, -50}, + {16777216000000000000U, -24}, {12500000000000000000U, 3}, {9313225746154785156U, 30}, + {13877787807814456755U, 56}, {10339757656912845936U, 83}, {15407439555097886824U, 109}, + {11479437019748901445U, 136}, {17105694144590052135U, 162}, {12744735289059618216U, 189}, + {9495567745759798747U, 216}, {14149498560666738074U, 242}, {10542197943230523224U, 269}, + {15709099088952724970U, 295}, {11704190886730495818U, 322}, {17440603504673385349U, 348}, + {12994262207056124023U, 375}, {9681479787123295682U, 402}, {14426529090290212157U, 428}, + {10748601772107342003U, 455}, {16016664761464807395U, 481}, {11933345169920330789U, 508}, + {17782069995880619868U, 534}, {13248674568444952270U, 561}, {9871031767461413346U, 588}, + {14708983551653345445U, 614}, {10959046745042015199U, 641}, {16330252207878254650U, 667}, + {12166986024289022870U, 694}, {18130221999122236476U, 720}, {13508068024458167312U, 747}, + {10064294952495520794U, 774}, {14996968138956309548U, 800}, {11173611982879273257U, 827}, + {16649979327439178909U, 853}, {12405201291620119593U, 880}, {9242595204427927429U, 907}, + {13772540099066387757U, 933}, {10261342003245940623U, 960}, {15290591125556738113U, 986}, + {11392378155556871081U, 1013}, {16975966327722178521U, 1039}, {12648080533535911531U, 1066}}; -static inline constexpr Fp find_cachedpow10(int exp, int* k) { +static inline constexpr Fp find_cachedpow10(int exp, int* k) +{ const double one_log_ten = 0.30102999566398114; int approx = -(exp + npowers) * one_log_ten; - int idx = (approx - firstpower) / steppowers; + int idx = (approx - firstpower) / steppowers; - while (1) { + while (1) + { int current = exp + powers_ten[idx].exp + 64; - if (current < expmin) { + if (current < expmin) + { idx++; continue; } - if (current > expmax) { + if (current > expmax) + { idx--; continue; } diff --git a/libraries/clio/include/clio/protobuf/any.hpp b/libraries/clio/include/clio/protobuf/any.hpp index 75823b8bd..43b7a02c8 100644 --- a/libraries/clio/include/clio/protobuf/any.hpp +++ b/libraries/clio/include/clio/protobuf/any.hpp @@ -1,143 +1,149 @@ #pragma once -#include -#include #include +#include +#include -namespace clio { - namespace protobuf { - enum wire_type_enum { - varint = 0, - fixed64 = 1, - buffer = 2, - fixed32 = 5 - }; - - struct entry { - typedef std::variant value_type; +namespace clio +{ + namespace protobuf + { + enum wire_type_enum + { + varint = 0, + fixed64 = 1, + buffer = 2, + fixed32 = 5 + }; - entry(){} - template - entry( uint32_t n, T&& v ) - :number(n),value( std::forward(v) ){} + struct entry + { + typedef std::variant value_type; - uint32_t number; - value_type value; - }; - CLIO_REFLECT( entry, number, value ) + entry() {} + template + entry(uint32_t n, T&& v) : number(n), value(std::forward(v)) + { + } - /** - * This class can be used to hold the - * deserialized contents of any protobuf stream - */ - struct any { - std::vector< entry > members; + uint32_t number; + value_type value; + }; + CLIO_REFLECT(entry, number, value) - void add( uint32_t field, const std::string& s ) { - members.push_back( entry{ field, bytes{ std::vector(s.begin(), s.end()) } } ); - } - template::value > > - void add( uint32_t field, const std::vector& s ) { - members.push_back( entry{ field, - bytes{ std::vector( (char*)s.data(), (char*)(s.data()+s.size())) } } ); - } - void add( uint32_t field, std::vector&& s ) { - members.push_back( entry{ field, bytes{ std::move(s) } } ); - } - void add( uint32_t field, varuint32 s ) { - members.push_back( entry{ field, s } ); - } - void add( uint32_t field, uint64_t s ) { - members.push_back( entry{ field, static_cast(s) } ); - } - void add( uint32_t field, int64_t s ) { - members.push_back( entry{ field, s } ); - } - void add( uint32_t field, int32_t s ) { - members.push_back( entry{ field, s } ); - } - void add( uint32_t field, int16_t s ) { - members.push_back( entry{ field, varuint32(s) } ); - } - void add( uint32_t field, uint16_t s ) { - members.push_back( entry{ field, varuint32(s) } ); - } - void add( uint32_t field, int8_t s ) { - members.push_back( entry{ field, varuint32(s) } ); - } - void add( uint32_t field, uint8_t s ) { - members.push_back( entry{ field, varuint32(s) } ); - } - void add( uint32_t field, bool s ) { - members.push_back( entry{ field, varuint32(s) } ); - } - void add( uint32_t field, char s ) { - members.push_back( entry{ field, varuint32(s) } ); - } - void add( uint32_t field, double s ) { - members.push_back( entry{ field, *reinterpret_cast(&s) } ); - } - void add( uint32_t field, float s ) { - members.push_back( entry{ field, *reinterpret_cast(&s) } ); - } - }; - CLIO_REFLECT( any, members ) + /** + * This class can be used to hold the + * deserialized contents of any protobuf stream + */ + struct any + { + std::vector members; + void add(uint32_t field, const std::string& s) + { + members.push_back(entry{field, bytes{std::vector(s.begin(), s.end())}}); + } + template ::value> > + void add(uint32_t field, const std::vector& s) + { + members.push_back(entry{ + field, bytes{std::vector((char*)s.data(), (char*)(s.data() + s.size()))}}); + } + void add(uint32_t field, std::vector&& s) + { + members.push_back(entry{field, bytes{std::move(s)}}); + } + void add(uint32_t field, varuint32 s) { members.push_back(entry{field, s}); } + void add(uint32_t field, uint64_t s) + { + members.push_back(entry{field, static_cast(s)}); + } + void add(uint32_t field, int64_t s) { members.push_back(entry{field, s}); } + void add(uint32_t field, int32_t s) { members.push_back(entry{field, s}); } + void add(uint32_t field, int16_t s) { members.push_back(entry{field, varuint32(s)}); } + void add(uint32_t field, uint16_t s) { members.push_back(entry{field, varuint32(s)}); } + void add(uint32_t field, int8_t s) { members.push_back(entry{field, varuint32(s)}); } + void add(uint32_t field, uint8_t s) { members.push_back(entry{field, varuint32(s)}); } + void add(uint32_t field, bool s) { members.push_back(entry{field, varuint32(s)}); } + void add(uint32_t field, char s) { members.push_back(entry{field, varuint32(s)}); } + void add(uint32_t field, double s) + { + members.push_back(entry{field, *reinterpret_cast(&s)}); + } + void add(uint32_t field, float s) + { + members.push_back(entry{field, *reinterpret_cast(&s)}); + } + }; + CLIO_REFLECT(any, members) - template - void to_bin( const any& a, Stream& s ) { - for( const auto& e : a.members ) { - uint32_t key = e.number << 3; - key |= e.value.index() == 3 ? wire_type_enum::fixed32 : e.value.index(); - varuint32_to_bin( key, s ); + template + void to_bin(const any& a, Stream& s) + { + for (const auto& e : a.members) + { + uint32_t key = e.number << 3; + key |= e.value.index() == 3 ? wire_type_enum::fixed32 : e.value.index(); + varuint32_to_bin(key, s); - std::visit( [&]( const auto& v ){ - to_bin( v, s ); - }, e.value ); - } - } + std::visit([&](const auto& v) { to_bin(v, s); }, e.value); + } + } - template - void from_bin( any& a, Stream& s ) { - while( s.remaining() ) { - uint32_t key = 0; - varuint32_from_bin( key, s ); - wire_type_enum type = wire_type_enum(uint8_t(key) & 0x07); - uint32_t number = key >> 3; + template + void from_bin(any& a, Stream& s) + { + while (s.remaining()) + { + uint32_t key = 0; + varuint32_from_bin(key, s); + wire_type_enum type = wire_type_enum(uint8_t(key) & 0x07); + uint32_t number = key >> 3; - switch( type ) { - case wire_type_enum::varint: { - varuint32 val; - from_bin( val, s ); - a.members.push_back( {number, val} ); - } break; - case wire_type_enum::fixed64: { - int64_t val; - from_bin( val, s ); - a.members.push_back( {number, val} ); - } break; - case wire_type_enum::buffer: { - bytes val; - from_bin( val, s ); - a.members.emplace_back( number, std::move(val) ); - } break; - case wire_type_enum::fixed32: { - int32_t val; - from_bin( val, s ); - a.members.push_back( {number, val} ); - } break; + switch (type) + { + case wire_type_enum::varint: + { + varuint32 val; + from_bin(val, s); + a.members.push_back({number, val}); + } + break; + case wire_type_enum::fixed64: + { + int64_t val; + from_bin(val, s); + a.members.push_back({number, val}); + } + break; + case wire_type_enum::buffer: + { + bytes val; + from_bin(val, s); + a.members.emplace_back(number, std::move(val)); + } + break; + case wire_type_enum::fixed32: + { + int32_t val; + from_bin(val, s); + a.members.push_back({number, val}); } + break; } - } + } + } - } /// namespace protobuf + } // namespace protobuf - template - void to_protobuf( const protobuf::any& a, Stream& s ) { - to_bin( a, s ); - } - template - void from_protobuf( protobuf::any& a, Stream& s ) { - from_bin( a, s ); - } + template + void to_protobuf(const protobuf::any& a, Stream& s) + { + to_bin(a, s); + } + template + void from_protobuf(protobuf::any& a, Stream& s) + { + from_bin(a, s); + } -} /// namespace clio +} // namespace clio diff --git a/libraries/clio/include/clio/protobuf/json.hpp b/libraries/clio/include/clio/protobuf/json.hpp index 8ddd585c7..f9ca75051 100644 --- a/libraries/clio/include/clio/protobuf/json.hpp +++ b/libraries/clio/include/clio/protobuf/json.hpp @@ -1,172 +1,203 @@ #pragma once -#include #include #include #include #include +#include -namespace clio { - - namespace protobuf { - - struct json_field_query; - struct json_query { - std::vector fields; - }; - struct json_field_query { - std::string name; - json::any args; /// args as json encoded tuple, or named object - json_query filter; /// apply to result - }; - CLIO_REFLECT( json_field_query, name, args, filter ) - CLIO_REFLECT( json_query, fields ) - +namespace clio +{ + namespace protobuf + { + struct json_field_query; + struct json_query + { + std::vector fields; + }; + struct json_field_query + { + std::string name; + json::any args; /// args as json encoded tuple, or named object + json_query filter; /// apply to result + }; + CLIO_REFLECT(json_field_query, name, args, filter) + CLIO_REFLECT(json_query, fields) - /** - * Given a json description of the query and an assumed reflected type (used to interpret names to numbers), - * this function will convert the JSON-query into a protobuf-style query. - */ - template - protobuf::query from_json_query( const json_query& jq ) { - protobuf::query result; - result.fields.reserve( jq.fields.size() ); + /** + * Given a json description of the query and an assumed reflected type (used to interpret + * names to numbers), this function will convert the JSON-query into a protobuf-style query. + */ + template + protobuf::query from_json_query(const json_query& jq) + { + protobuf::query result; + result.fields.reserve(jq.fields.size()); - if constexpr( reflect::is_struct ) { - for( const auto& field : jq.fields ) { - reflect::for_each( [&](const meta& ref, auto mptr ) { - if constexpr( not std::is_member_function_pointer_v< decltype(mptr) > ) { - if( ref.name == field.name ) { - if( field.filter.fields.size() ) { - result.fields.push_back({ - .number = ref.number, - .filter = from_json_query(nullptr)->*mptr)>>( field.filter ) - }); - } - else { - result.fields.push_back( field_query{ ref.number } ); - } - } - } else { /// member function ptr - if( ref.name == field.name ) { - /// TODO: do a direct conversion from json::any to pb bytes instead of - /// any->json_string->native->pb - using args_tuple_type = decltype( args_as_tuple( mptr ) ); - std::string json = convert_to_json( field.args ); - auto args_value = clio::from_json(std::move(json)); + if constexpr (reflect::is_struct) + { + for (const auto& field : jq.fields) + { + reflect::for_each([&](const meta& ref, auto mptr) { + if constexpr (not std::is_member_function_pointer_v) + { + if (ref.name == field.name) + { + if (field.filter.fields.size()) + { + result.fields.push_back( + {.number = ref.number, + .filter = from_json_query< + std::decay_t(nullptr)->*mptr)>>( + field.filter)}); + } + else + { + result.fields.push_back(field_query{ref.number}); + } + } + } + else + { /// member function ptr + if (ref.name == field.name) + { + /// TODO: do a direct conversion from json::any to pb bytes instead of + /// any->json_string->native->pb + using args_tuple_type = decltype(args_as_tuple(mptr)); + std::string json = convert_to_json(field.args); + auto args_value = clio::from_json(std::move(json)); - if( field.filter.fields.size() ) { - result.fields.push_back({ - .number = ref.number, - .args = to_protobuf( args_value ), - .filter = from_json_query< decltype( result_of(mptr) )>( field.filter ) - }); - } else { - result.fields.push_back( field_query{ - .number = ref.number, - .args = to_protobuf( args_value ) - }); - } - } + if (field.filter.fields.size()) + { + result.fields.push_back( + {.number = ref.number, + .args = to_protobuf(args_value), + .filter = + from_json_query(field.filter)}); } - }); - } + else + { + result.fields.push_back( + field_query{.number = ref.number, .args = to_protobuf(args_value)}); + } + } + } + }); } - return result; - } + } + return result; + } - /** - * Given a protobuf query and a type, convert it into a friendly JSON Query - */ - template - json_query to_json_query( const protobuf::query& pbuf ) { - json_query result; + /** + * Given a protobuf query and a type, convert it into a friendly JSON Query + */ + template + json_query to_json_query(const protobuf::query& pbuf) + { + json_query result; - if constexpr( reflect::is_struct ) { - for( const auto& entry : pbuf.fields ) { - reflect::for_each( [&](const meta& ref, auto mptr ) { - if constexpr( not std::is_member_function_pointer_v< decltype(mptr) > ) { - if( ref.number == entry.number.value ) { - if( entry.filter.fields.size() ) { - result.fields.push_back({ - .name = ref.name, - .filter = to_json_query(nullptr)->*mptr)>>( entry.filter ) - }); - } - else { - result.fields.push_back( { ref.name} ); - } - } - } else { /// member function ptr - if( ref.number == entry.number.value ) { - /// TODO: do a direct conversion from PB to json::any instead of - /// pb->native->json_string->any - using args_tuple_type = decltype( args_as_tuple( mptr ) ); - auto pbargs = from_protobuf( entry.args.data ); - auto jargs = to_json( pbargs ); + if constexpr (reflect::is_struct) + { + for (const auto& entry : pbuf.fields) + { + reflect::for_each([&](const meta& ref, auto mptr) { + if constexpr (not std::is_member_function_pointer_v) + { + if (ref.number == entry.number.value) + { + if (entry.filter.fields.size()) + { + result.fields.push_back( + {.name = ref.name, + .filter = to_json_query< + std::decay_t(nullptr)->*mptr)>>( + entry.filter)}); + } + else + { + result.fields.push_back({ref.name}); + } + } + } + else + { /// member function ptr + if (ref.number == entry.number.value) + { + /// TODO: do a direct conversion from PB to json::any instead of + /// pb->native->json_string->any + using args_tuple_type = decltype(args_as_tuple(mptr)); + auto pbargs = from_protobuf(entry.args.data); + auto jargs = to_json(pbargs); - if( entry.filter.fields.size() ) { - result.fields.push_back({ - .name = ref.name, - .args = from_json( jargs ), - .filter = to_json_query< decltype( result_of(mptr) )>( entry.filter ) - }); - } else { - result.fields.push_back( { - .name = ref.name, - .args = from_json( jargs ) - }); - } - } + if (entry.filter.fields.size()) + { + result.fields.push_back( + {.name = ref.name, + .args = from_json(jargs), + .filter = to_json_query(entry.filter)}); + } + else + { + result.fields.push_back( + {.name = ref.name, .args = from_json(jargs)}); } - }); - } + } + } + }); } - return result; - } + } + return result; + } - } /// namespace protobuf - template - void to_json(const protobuf::json_query& s, S& stream) { - to_json(s.fields, stream); - } - template - void from_json( protobuf::json_query& s, S& stream) { - from_json(s.fields, stream); - } - template - void to_json(const protobuf::json_field_query& s, S& stream) { - stream.write('{'); - increase_indent(stream); + } // namespace protobuf + template + void to_json(const protobuf::json_query& s, S& stream) + { + to_json(s.fields, stream); + } + template + void from_json(protobuf::json_query& s, S& stream) + { + from_json(s.fields, stream); + } + template + void to_json(const protobuf::json_field_query& s, S& stream) + { + stream.write('{'); + increase_indent(stream); + write_newline(stream); + stream.write("\"name\""); + write_colon(stream); + to_json(s.name, stream); + if (auto* a = s.args.get_if()) + { + if (a->size()) + { + stream.write(','); + write_newline(stream); + stream.write("\"args\""); + write_colon(stream); + to_json(s.args, stream); + } + } + else if (not s.args.get_if()) + { + stream.write(','); write_newline(stream); - stream.write("\"name\""); + stream.write("\"args\""); write_colon(stream); - to_json( s.name, stream ); - if( auto* a = s.args.get_if() ) { - if( a->size() ) { - stream.write(','); - write_newline(stream); - stream.write("\"args\""); - write_colon(stream); - to_json( s.args, stream ); - } - } - else if( not s.args.get_if() ) { - stream.write(','); - write_newline(stream); - stream.write("\"args\""); - write_colon(stream); - to_json( s.args, stream ); - } - if( s.filter.fields.size() ) { - stream.write(','); - write_newline(stream); - stream.write("\"filter\""); - write_colon(stream); - to_json( s.filter, stream ); - } - decrease_indent(stream); + to_json(s.args, stream); + } + if (s.filter.fields.size()) + { + stream.write(','); write_newline(stream); - stream.write('}'); - } + stream.write("\"filter\""); + write_colon(stream); + to_json(s.filter, stream); + } + decrease_indent(stream); + write_newline(stream); + stream.write('}'); + } -} /// clio +} // namespace clio diff --git a/libraries/clio/include/clio/protobuf/query.hpp b/libraries/clio/include/clio/protobuf/query.hpp index 7147172e1..a26719a21 100644 --- a/libraries/clio/include/clio/protobuf/query.hpp +++ b/libraries/clio/include/clio/protobuf/query.hpp @@ -2,200 +2,231 @@ #include #include -namespace clio { - - namespace protobuf { - - struct field_query; - struct variant_query; - - struct variant_query { - varuint32 type; - std::vector fields; - }; - - struct query { - std::vector fields; - std::vector variant_fields; /// only apply these fields when the type has them - }; - - struct field_query { - varuint32 number; - bytes args; /// args as protobuf encoded tuple - query filter; /// apply to result - }; - - CLIO_REFLECT( variant_query, type, fields ) - CLIO_REFLECT( field_query, number, args, filter ) - CLIO_REFLECT( query, fields, variant_fields ) - - - - template - any dispatch( T&& obj, const query& q ) { - any result; - result.members.reserve( q.fields.size() ); - - for( const auto& field : q.fields ) { - reflect::get( int64_t(field.number), [&]( auto mptr ){ - using member_ptr_type = decltype(mptr); - if constexpr ( std::is_member_function_pointer_v ) { - using return_type = decltype( result_of(mptr) ); - using param_type = decltype( args_as_tuple(mptr) ); - // std::cerr<< boost::core::demangle(typeid(param_type).name()) <<"\n"; - - param_type params; - input_stream in( field.args.data.data(), field.args.data.size() ); - (void) from_protobuf_object( params, in ); - std::apply( [&]( auto... args ){ - if constexpr( is_std_variant::value ) { - /// TODO: - } else if constexpr( not reflect::is_struct ) - result.add( field.number, (obj.*mptr)( args... ) ); - else { - any field_data = dispatch( (obj.*mptr)( args...) , field.filter ); - result.add( field.number, clio::to_bin( field_data ) ); - } - }, params ); - - } else if constexpr ( not std::is_member_function_pointer_v ) { - using member_type = std::decay_t; - - if constexpr( is_std_tuple::value ) { - /// TODO: - } - else if constexpr( is_std_variant::value ) { - /// TODO: - } - else if constexpr ( is_std_vector::value ) - { - using value_type = typename is_std_vector::value_type; - if constexpr( is_std_variant::value ) { - /// TODO: - } else if constexpr ( std::is_arithmetic_v< value_type > ) { - result.add( field.number, obj.*mptr ); - } else if constexpr( reflect::is_struct ) { - for( const auto& item : obj.*mptr ) { - any field_data = dispatch( item, field.filter ); - result.add( field.number, clio::to_bin( field_data ) ); - } - } else { - for( const auto& item : obj.*mptr ) { - result.add( field.number, item ); - } - } - } - else if constexpr ( not reflect::is_struct ) +namespace clio +{ + namespace protobuf + { + struct field_query; + struct variant_query; + + struct variant_query + { + varuint32 type; + std::vector fields; + }; + + struct query + { + std::vector fields; + std::vector + variant_fields; /// only apply these fields when the type has them + }; + + struct field_query + { + varuint32 number; + bytes args; /// args as protobuf encoded tuple + query filter; /// apply to result + }; + + CLIO_REFLECT(variant_query, type, fields) + CLIO_REFLECT(field_query, number, args, filter) + CLIO_REFLECT(query, fields, variant_fields) + + template + any dispatch(T&& obj, const query& q) + { + any result; + result.members.reserve(q.fields.size()); + + for (const auto& field : q.fields) + { + reflect::get(int64_t(field.number), [&](auto mptr) { + using member_ptr_type = decltype(mptr); + if constexpr (std::is_member_function_pointer_v) + { + using return_type = decltype(result_of(mptr)); + using param_type = decltype(args_as_tuple(mptr)); + // std::cerr<< boost::core::demangle(typeid(param_type).name()) <<"\n"; + + param_type params; + input_stream in(field.args.data.data(), field.args.data.size()); + (void)from_protobuf_object(params, in); + std::apply( + [&](auto... args) { + if constexpr (is_std_variant::value) + { + /// TODO: + } + else if constexpr (not reflect::is_struct) + result.add(field.number, (obj.*mptr)(args...)); + else + { + any field_data = dispatch((obj.*mptr)(args...), field.filter); + result.add(field.number, clio::to_bin(field_data)); + } + }, + params); + } + else if constexpr (not std::is_member_function_pointer_v) + { + using member_type = std::decay_t; + + if constexpr (is_std_tuple::value) + { + /// TODO: + } + else if constexpr (is_std_variant::value) + { + /// TODO: + } + else if constexpr (is_std_vector::value) + { + using value_type = typename is_std_vector::value_type; + if constexpr (is_std_variant::value) + { + /// TODO: + } + else if constexpr (std::is_arithmetic_v) + { + result.add(field.number, obj.*mptr); + } + else if constexpr (reflect::is_struct) + { + for (const auto& item : obj.*mptr) { - result.add( field.number, obj.*mptr ); + any field_data = dispatch(item, field.filter); + result.add(field.number, clio::to_bin(field_data)); } - else + } + else + { + for (const auto& item : obj.*mptr) { - any field_data = dispatch( obj.*mptr, field.filter ); - result.add( field.number, clio::to_bin( field_data ) ); + result.add(field.number, item); } - - } - }); - } - return result; - } - - struct query_proxy { - protobuf::query q; - - template - query_proxy& call( Args&&... args ) { - return *this; - } - template - void get() { - } - - template - void get()const { - } - - template - query_proxy& call( const meta& ref, Mptr mptr, Filter&& filter ) { - using result_type = decltype( result_of(mptr) ); - using args_tuple = decltype(args_as_tuple( mptr )); - - if constexpr ( std::tuple_size::value == 1 ) { - auto params = args_tuple( std::forward(filter) ); - q.fields.push_back( { - .number = ref.number, - .args = to_protobuf( params ), - }); - } else { - typename reflect::template proxy rp; - filter( rp ); - q.fields.push_back( { - .number = ref.number, - .filter = std::move(rp->q) - }); + } + } + else if constexpr (not reflect::is_struct) + { + result.add(field.number, obj.*mptr); + } + else + { + any field_data = dispatch(obj.*mptr, field.filter); + result.add(field.number, clio::to_bin(field_data)); + } } - return *this; - } - - template - query_proxy& call( const meta& ref, Mptr mptr ) { - q.fields.push_back( { + }); + } + return result; + } + + struct query_proxy + { + protobuf::query q; + + template + query_proxy& call(Args&&... args) + { + return *this; + } + template + void get() + { + } + + template + void get() const + { + } + + template + query_proxy& call(const meta& ref, Mptr mptr, Filter&& filter) + { + using result_type = decltype(result_of(mptr)); + using args_tuple = decltype(args_as_tuple(mptr)); + + if constexpr (std::tuple_size::value == 1) + { + auto params = args_tuple(std::forward(filter)); + q.fields.push_back({ .number = ref.number, + .args = to_protobuf(params), }); - return *this; - } - - - template - query_proxy& call( const meta& ref, Mptr mptr, Filter&& filter, Args&&... args ) { - using result_type = decltype( result_of(mptr) ); - using args_tuple = decltype(args_as_tuple( mptr )); - /** test to see whether the user passed a filter by comparing the number of args, - * if the number of var args equals the number of paramters then a filter was passed, - * otherwise the Filter is being interpreted as the first argument. This means that in - * order to use a filter you must pass all args, if you don't want a filter on the result - * then you can skip trailing args. - */ - if constexpr ( std::tuple_size::value == sizeof...(Args) ) { - auto params = args_tuple( std::forward(args)... ); - - if constexpr( reflect::is_struct ) { - typename reflect::template proxy rp; - filter( rp ); - q.fields.push_back( { - .number = ref.number, - .args = to_protobuf( params ), - .filter = std::move(rp->q) - }); - } else { - q.fields.push_back( { - .number = ref.number, - .args = to_protobuf( params ), - }); - } - } else { - auto params = args_tuple( std::forward(filter), std::forward(args)... ); - - if constexpr( reflect::is_struct ) { - typename reflect::template proxy rp; - q.fields.push_back( { - .number = ref.number, - .args = to_protobuf( params ), - .filter = std::move(rp->q) - }); - } else { - q.fields.push_back( { - .number = ref.number, - .args = to_protobuf( params ), - }); - } + } + else + { + typename reflect::template proxy rp; + filter(rp); + q.fields.push_back({.number = ref.number, .filter = std::move(rp->q)}); + } + return *this; + } + + template + query_proxy& call(const meta& ref, Mptr mptr) + { + q.fields.push_back({ + .number = ref.number, + }); + return *this; + } + + template + query_proxy& call(const meta& ref, Mptr mptr, Filter&& filter, Args&&... args) + { + using result_type = decltype(result_of(mptr)); + using args_tuple = decltype(args_as_tuple(mptr)); + /** test to see whether the user passed a filter by comparing the number of args, + * if the number of var args equals the number of paramters then a filter was passed, + * otherwise the Filter is being interpreted as the first argument. This means that in + * order to use a filter you must pass all args, if you don't want a filter on the + * result then you can skip trailing args. + */ + if constexpr (std::tuple_size::value == sizeof...(Args)) + { + auto params = args_tuple(std::forward(args)...); + + if constexpr (reflect::is_struct) + { + typename reflect::template proxy rp; + filter(rp); + q.fields.push_back({.number = ref.number, + .args = to_protobuf(params), + .filter = std::move(rp->q)}); } - return *this; - } - }; - + else + { + q.fields.push_back({ + .number = ref.number, + .args = to_protobuf(params), + }); + } + } + else + { + auto params = args_tuple(std::forward(filter), std::forward(args)...); + + if constexpr (reflect::is_struct) + { + typename reflect::template proxy rp; + q.fields.push_back({.number = ref.number, + .args = to_protobuf(params), + .filter = std::move(rp->q)}); + } + else + { + q.fields.push_back({ + .number = ref.number, + .args = to_protobuf(params), + }); + } + } + return *this; + } + }; - } // namespace protobuf + } // namespace protobuf -} /// namespace clio +} // namespace clio diff --git a/libraries/clio/include/clio/protobuf/schema.hpp b/libraries/clio/include/clio/protobuf/schema.hpp index d19575dab..09a332f40 100644 --- a/libraries/clio/include/clio/protobuf/schema.hpp +++ b/libraries/clio/include/clio/protobuf/schema.hpp @@ -1,37 +1,47 @@ #pragma once #include -namespace clio { - - /** - * Writes a protobuf schem file to stream given a schema object - */ - template - void to_protobuf_schema( const schema& sch, S& stream ) { - - stream.write( "syntax = \"proto3\";\n" ); - - auto convert_variant_name = []( string name ) { - for( auto& c : name ) { - if( c == '|' ) c = '_'; - if( c == '?' ) c = 'O'; - if( c == '[' ) c = '_'; - if( c == ']' ) c = 'a'; - } - return "variant_" + name.substr( 0, name.size()-1); - }; - auto convert_tuple_name = []( string name ) { - for( auto& c : name ) { - if( c == '&' ) c = '_'; - if( c == '?' ) c = 'O'; - if( c == '[' ) c = '_'; - if( c == ']' ) c = 'a'; - } - return "tuple_" + name.substr( 0, name.size()-1); - }; - - auto convert_map_name = []( string name ) { - /* //TODO... +namespace clio +{ + /** + * Writes a protobuf schem file to stream given a schema object + */ + template + void to_protobuf_schema(const schema& sch, S& stream) + { + stream.write("syntax = \"proto3\";\n"); + + auto convert_variant_name = [](string name) { + for (auto& c : name) + { + if (c == '|') + c = '_'; + if (c == '?') + c = 'O'; + if (c == '[') + c = '_'; + if (c == ']') + c = 'a'; + } + return "variant_" + name.substr(0, name.size() - 1); + }; + auto convert_tuple_name = [](string name) { + for (auto& c : name) + { + if (c == '&') + c = '_'; + if (c == '?') + c = 'O'; + if (c == '[') + c = '_'; + if (c == ']') + c = 'a'; + } + return "tuple_" + name.substr(0, name.size() - 1); + }; + + auto convert_map_name = [](string name) { + /* //TODO... for( auto& c : name ) { if( c == '&' ) c = '_'; if( c == '?' ) c = 'O'; @@ -39,65 +49,90 @@ namespace clio { if( c == ']' ) c = 'a'; } */ - return name; - }; - - auto decay_type = [&]( const std::string& str, bool& is_repeated ) -> std::string { - if( str.back() == ']' ) { - is_repeated = true; - return str.substr( 0, str.size()-2); - } - else if( str.back() == '?' ) return str.substr( 0, str.size()-1); - else if( str.back() == '|' ) return convert_variant_name( str ); - else if( str.back() == '&' ) return convert_tuple_name( str ); - else if( str.back() == '>' ) return convert_map_name( str ); - else return str; - }; - - auto convert_to_pb_type = [&]( const string& str, bool& is_repeated, bool& is_prim ) -> std::string { - auto decay = decay_type( str, is_repeated ); - is_prim = true; - if( decay == "int16_t" ) return "int32"; - else if( decay == "char" ) return "int32"; - else if( decay == "int8_t" ) return "int32"; - else if( decay == "uint16_t" ) return "int32"; - else if( decay == "uint8_t" ) return "int32"; - else if( decay == "int32_t" ) return "sfixed32"; - else if( decay == "uint32_t" ) return "fixed32"; - else if( decay == "int64_t" ) return "sfixed64"; - else if( decay == "uint64_t" ) return "fixed64"; - else if( decay == "bool" ) return "bool"; - else if( decay == "double" ) return decay; - else if( decay == "float" ) return decay; - is_prim = false; + return name; + }; + + auto decay_type = [&](const std::string& str, bool& is_repeated) -> std::string { + if (str.back() == ']') + { + is_repeated = true; + return str.substr(0, str.size() - 2); + } + else if (str.back() == '?') + return str.substr(0, str.size() - 1); + else if (str.back() == '|') + return convert_variant_name(str); + else if (str.back() == '&') + return convert_tuple_name(str); + else if (str.back() == '>') + return convert_map_name(str); + else + return str; + }; + + auto convert_to_pb_type = [&](const string& str, bool& is_repeated, + bool& is_prim) -> std::string { + auto decay = decay_type(str, is_repeated); + is_prim = true; + if (decay == "int16_t") + return "int32"; + else if (decay == "char") + return "int32"; + else if (decay == "int8_t") + return "int32"; + else if (decay == "uint16_t") + return "int32"; + else if (decay == "uint8_t") + return "int32"; + else if (decay == "int32_t") + return "sfixed32"; + else if (decay == "uint32_t") + return "fixed32"; + else if (decay == "int64_t") + return "sfixed64"; + else if (decay == "uint64_t") + return "fixed64"; + else if (decay == "bool") + return "bool"; + else if (decay == "double") return decay; - }; - - for( const auto& item : sch.types ) { - std::visit( [&]( auto i ) { - if constexpr ( std::is_same_v ) { - stream.write( "message ", 8 ); - stream.write( item.first ); - stream.write( " {\n" ); - for( auto mem : i.members ) { - bool is_repeated = false; - bool is_prim = false; - auto pbt = convert_to_pb_type( mem.type, is_repeated, is_prim ); - stream.write( " " ); - if( is_repeated ) { - stream.write( "repeated " ); - } - stream.write( pbt ); - stream.write( " " ); - stream.write( mem.name.c_str() ); - stream.write( " = " ); - stream.write( std::to_string( mem.number ).c_str() ); - if( is_repeated && is_prim ) { - stream.write( " [packed=true]" ); - } - stream.write( ";\n" ); - } - /* + else if (decay == "float") + return decay; + is_prim = false; + return decay; + }; + + for (const auto& item : sch.types) + { + std::visit( + [&](auto i) { + if constexpr (std::is_same_v) + { + stream.write("message ", 8); + stream.write(item.first); + stream.write(" {\n"); + for (auto mem : i.members) + { + bool is_repeated = false; + bool is_prim = false; + auto pbt = convert_to_pb_type(mem.type, is_repeated, is_prim); + stream.write(" "); + if (is_repeated) + { + stream.write("repeated "); + } + stream.write(pbt); + stream.write(" "); + stream.write(mem.name.c_str()); + stream.write(" = "); + stream.write(std::to_string(mem.number).c_str()); + if (is_repeated && is_prim) + { + stream.write(" [packed=true]"); + } + stream.write(";\n"); + } + /* for( auto mem : i.methods ) { bool is_repeated = false; bool is_prim = false; @@ -111,77 +146,93 @@ namespace clio { stream.write( std::to_string( mem.number ).c_str() ); stream.write( ";\n" ); }*/ - stream.write( "}\n" ); - } else if constexpr ( std::is_same_v ) { - stream.write( "message " ); - if( item.first.back() == '|' ) { - stream.write( convert_variant_name(item.first) ); - } else { - stream.write( item.first ); - } - - stream.write( " {\n" ); - stream.write( " oneof which {\n" ); - - for( uint32_t idx = 0; idx < i.types.size(); ++idx ) { - bool is_repeated = false; - bool is_prim = false; - auto pbt = convert_to_pb_type( i.types[idx], is_repeated, is_prim ); - - stream.write( " " ); - stream.write( pbt ); - stream.write( " " ); - stream.write( pbt ); - stream.write( "_value = " ); - stream.write( std::to_string(idx+1)); - stream.write( ";\n" ); - } - stream.write( " }\n" ); - stream.write( "}\n" ); - } else if constexpr ( std::is_same_v ) { - stream.write( "message " ); - - if( item.first.back() == '&' ) { - stream.write( convert_tuple_name(item.first) ); - } else { - stream.write( item.first ); - } - - stream.write( " {\n" ); - for( uint32_t idx = 0; idx < i.types.size(); ++idx ) { - stream.write( " " ); - bool is_repeated = false; - bool is_prim = false; - auto pbt = convert_to_pb_type( i.types[idx], is_repeated, is_prim ); - if( is_repeated ) { - stream.write( "repeated " ); - } - stream.write( pbt ); - stream.write( " _" ); - stream.write( std::to_string(idx+1) ); - stream.write( " = " ); - stream.write( std::to_string(idx+1)); - if( is_repeated && is_prim ) { - stream.write( " [packed=true]" ); - } - stream.write( ";\n" ); - } - stream.write( "}\n" ); - } - }, item.second ); - } - } - inline std::string to_protobuf_schema( const schema& s ) { - size_stream ss; - to_protobuf_schema(s, ss); - - std::string result(ss.size, 0); - fixed_buf_stream fbs(result.data(), result.size()); - to_protobuf_schema(s, fbs); - - if (fbs.pos != fbs.end) - throw_error(stream_error::underrun); - - return result; - } -} /// clio + stream.write("}\n"); + } + else if constexpr (std::is_same_v) + { + stream.write("message "); + if (item.first.back() == '|') + { + stream.write(convert_variant_name(item.first)); + } + else + { + stream.write(item.first); + } + + stream.write(" {\n"); + stream.write(" oneof which {\n"); + + for (uint32_t idx = 0; idx < i.types.size(); ++idx) + { + bool is_repeated = false; + bool is_prim = false; + auto pbt = convert_to_pb_type(i.types[idx], is_repeated, is_prim); + + stream.write(" "); + stream.write(pbt); + stream.write(" "); + stream.write(pbt); + stream.write("_value = "); + stream.write(std::to_string(idx + 1)); + stream.write(";\n"); + } + stream.write(" }\n"); + stream.write("}\n"); + } + else if constexpr (std::is_same_v) + { + stream.write("message "); + + if (item.first.back() == '&') + { + stream.write(convert_tuple_name(item.first)); + } + else + { + stream.write(item.first); + } + + stream.write(" {\n"); + for (uint32_t idx = 0; idx < i.types.size(); ++idx) + { + stream.write(" "); + bool is_repeated = false; + bool is_prim = false; + auto pbt = convert_to_pb_type(i.types[idx], is_repeated, is_prim); + if (is_repeated) + { + stream.write("repeated "); + } + stream.write(pbt); + stream.write(" _"); + stream.write(std::to_string(idx + 1)); + stream.write(" = "); + stream.write(std::to_string(idx + 1)); + if (is_repeated && is_prim) + { + stream.write(" [packed=true]"); + } + stream.write(";\n"); + } + stream.write("}\n"); + } + }, + item.second); + } + } + inline std::string to_protobuf_schema(const schema& s) + { + size_stream ss; + to_protobuf_schema(s, ss); + + std::string result(ss.size, 0); + fixed_buf_stream fbs(result.data(), result.size()); + to_protobuf_schema(s, fbs); + + if (fbs.pos != fbs.end) + throw_error(stream_error::underrun); + + return result; + } +} // namespace clio diff --git a/libraries/clio/include/clio/reflect.hpp b/libraries/clio/include/clio/reflect.hpp index 705aa12d7..cc7892526 100644 --- a/libraries/clio/include/clio/reflect.hpp +++ b/libraries/clio/include/clio/reflect.hpp @@ -1,182 +1,182 @@ #pragma once -#include +#include #include #include -#include #include #include -#include +#include +#include #include #include -#include #include +#include #include #include +#include #include #include #include #include -#include -#include #include +#include -namespace clio { - - template - std::tuple...> args_as_tuple( R (C::*)(Args...) ); - template - std::tuple...> args_as_tuple( R (C::*)(Args...)const ); - - template - R result_of( R (C::*)(Args...)const ); - template - R result_of( R (C::*)(Args...) ); - - template - constexpr R result_of_member( R (C::*) ); - template - constexpr C class_of_member( R (C::*) ); - - template - void result_of_member( R (C::*)(Args...)const ); - template - void result_of_member( R (C::*)(Args...) ); - - - struct meta { - const char* name; - int32_t number; - std::initializer_list param_names; - }; - - #define CLIO_REFLECT_ARGS_INTERNAL( r, OP, i, PARAM) \ - BOOST_PP_COMMA_IF(i) BOOST_PP_STRINGIZE(PARAM) - - #define CLIO_REFLECT_ARGS_HELPER( METHOD, PARAM_NAMES ) \ - BOOST_PP_SEQ_FOR_EACH_I( CLIO_REFLECT_ARGS_INTERNAL, METHOD, PARAM_NAMES ) - - - #define CLIO_REFLECT_FILTER_PARAMS(NAME, IDX, ...) { CLIO_REFLECT_ARGS_HELPER( METHOD, BOOST_PP_VARIADIC_TO_SEQ( __VA_ARGS__ ) ) } - #define CLIO_REFLECT_FILTER_NAME(NAME, IDX, ...) NAME - #define CLIO_REFLECT_FILTER_NAME_STR(NAME, IDX, ...) BOOST_PP_STRINGIZE(NAME) - #define CLIO_REFLECT_FILTER_IDX(NAME, IDX, ...) IDX - +namespace clio +{ + template + std::tuple...> args_as_tuple(R (C::*)(Args...)); + template + std::tuple...> args_as_tuple(R (C::*)(Args...) const); + + template + R result_of(R (C::*)(Args...) const); + template + R result_of(R (C::*)(Args...)); + + template + constexpr R result_of_member(R(C::*)); + template + constexpr C class_of_member(R(C::*)); + + template + void result_of_member(R (C::*)(Args...) const); + template + void result_of_member(R (C::*)(Args...)); + + struct meta + { + const char* name; + int32_t number; + std::initializer_list param_names; + }; - #define CLIO_REFLECT_FOREACH_PB_INTERNAL( r, OP, member) \ - lambda( clio::meta{ .number = CLIO_REFLECT_FILTER_IDX member, \ - .name = CLIO_REFLECT_FILTER_NAME_STR member, \ - .param_names = CLIO_REFLECT_FILTER_PARAMS member }, \ - &OP::CLIO_REFLECT_FILTER_NAME member);\ +#define CLIO_REFLECT_ARGS_INTERNAL(r, OP, i, PARAM) BOOST_PP_COMMA_IF(i) BOOST_PP_STRINGIZE(PARAM) - - #define CLIO_REFLECT_FOREACH_INTERNAL( r, OP, i, member) \ - (void)lambda( clio::meta{ \ - .name = BOOST_PP_STRINGIZE(member), \ - .number = i+1 , \ - }, \ - &OP::member);\ +#define CLIO_REFLECT_ARGS_HELPER(METHOD, PARAM_NAMES) \ + BOOST_PP_SEQ_FOR_EACH_I(CLIO_REFLECT_ARGS_INTERNAL, METHOD, PARAM_NAMES) - #define CLIO_REFLECT_MEMBER_BY_STR_INTERNAL( r, OP, member) \ - if( BOOST_PP_STRINGIZE(member) == m ) { \ - (void)lambda( &OP::member ); return true; \ +#define CLIO_REFLECT_FILTER_PARAMS(NAME, IDX, ...) \ + { \ + CLIO_REFLECT_ARGS_HELPER(METHOD, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ } - - #define CLIO_REFLECT_MEMBER_BY_STR_PB_INTERNAL( r, OP, member) \ - if( CLIO_REFLECT_FILTER_NAME_STR member == m ) { \ - (void)lambda( &OP::CLIO_REFLECT_FILTER_NAME member ); return true; \ +#define CLIO_REFLECT_FILTER_NAME(NAME, IDX, ...) NAME +#define CLIO_REFLECT_FILTER_NAME_STR(NAME, IDX, ...) BOOST_PP_STRINGIZE(NAME) +#define CLIO_REFLECT_FILTER_IDX(NAME, IDX, ...) IDX + +#define CLIO_REFLECT_FOREACH_PB_INTERNAL(r, OP, member) \ + lambda(clio::meta{.number = CLIO_REFLECT_FILTER_IDX member, \ + .name = CLIO_REFLECT_FILTER_NAME_STR member, \ + .param_names = CLIO_REFLECT_FILTER_PARAMS member}, \ + &OP::CLIO_REFLECT_FILTER_NAME member); + +#define CLIO_REFLECT_FOREACH_INTERNAL(r, OP, i, member) \ + (void)lambda(clio::meta{ \ + .name = BOOST_PP_STRINGIZE(member), .number = i + 1, \ + }, \ + &OP::member); + +#define CLIO_REFLECT_MEMBER_BY_STR_INTERNAL(r, OP, member) \ + if (BOOST_PP_STRINGIZE(member) == m) \ + { \ + (void)lambda(&OP::member); \ + return true; \ } +#define CLIO_REFLECT_MEMBER_BY_STR_PB_INTERNAL(r, OP, member) \ + if (CLIO_REFLECT_FILTER_NAME_STR member == m) \ + { \ + (void)lambda(&OP::CLIO_REFLECT_FILTER_NAME member); \ + return true; \ + } - #define CLIO_REFLECT_MEMBER_BY_IDX_PB_INTERNAL( r, OP, member) \ - case CLIO_REFLECT_FILTER_IDX member: (void)lambda( &OP:: CLIO_REFLECT_FILTER_NAME member); return true; - -#define CLIO_REFLECT_PROXY_MEMBER_BY_IDX_INTERNAL( r, OP, I, member ) \ - template \ - auto member( Args&&... args ) { \ - return proxy___.call( clio::meta{ .number = I+1, \ - .name = BOOST_PP_STRINGIZE(member)\ - }, \ - &OP::member, std::forward(args)... ); \ - } \ - - -#define CLIO_REFLECT_MPROXY_MEMBER_BY_IDX_INTERNAL( r, OP, I, member ) \ - clio::member_proxy member; - -#define CLIO_REFLECT_MPROXY_MEMBER_BY_IDX_HELPER( QUERY_CLASS, MEMBER_IDXS ) \ - BOOST_PP_SEQ_FOR_EACH_I( CLIO_REFLECT_MPROXY_MEMBER_BY_IDX_INTERNAL, QUERY_CLASS, MEMBER_IDXS ) - - -#define CLIO_REFLECT_PROXY_MEMBER_BY_PB_INTERNAL( r, OP, member ) \ - template \ - auto CLIO_REFLECT_FILTER_NAME member ( Args&&... args ) { \ - return proxy___.call( clio::meta{ .number = CLIO_REFLECT_FILTER_IDX member, \ - .name = CLIO_REFLECT_FILTER_NAME_STR member, \ - .param_names = CLIO_REFLECT_FILTER_PARAMS member }, \ - &OP::CLIO_REFLECT_FILTER_NAME member, std::forward(args)... ); \ - } - - - #define CLIO_REFLECT_FOREACH_MEMBER_HELPER( QUERY_CLASS, MEMBERS ) \ - BOOST_PP_SEQ_FOR_EACH_I( CLIO_REFLECT_FOREACH_INTERNAL, QUERY_CLASS, MEMBERS ) - - #define CLIO_REFLECT_MEMBER_BY_STR_HELPER( QUERY_CLASS, MEMBERS ) \ - BOOST_PP_SEQ_FOR_EACH( CLIO_REFLECT_MEMBER_BY_STR_INTERNAL, QUERY_CLASS, MEMBERS ) - - - #define CLIO_REFLECT_MEMBER_BY_IDX_I_INTERNAL( r, OP, I, member) \ - case I+1: (void)lambda( &OP::member ); return true; - - #define CLIO_REFLECT_MEMBER_BY_IDX_HELPER( QUERY_CLASS, MEMBER_IDXS ) \ - BOOST_PP_SEQ_FOR_EACH_I( CLIO_REFLECT_MEMBER_BY_IDX_I_INTERNAL, QUERY_CLASS, MEMBER_IDXS ) +#define CLIO_REFLECT_MEMBER_BY_IDX_PB_INTERNAL(r, OP, member) \ + case CLIO_REFLECT_FILTER_IDX member: \ + (void)lambda(&OP::CLIO_REFLECT_FILTER_NAME member); \ + return true; + +#define CLIO_REFLECT_PROXY_MEMBER_BY_IDX_INTERNAL(r, OP, I, member) \ + template \ + auto member(Args&&... args) \ + { \ + return proxy___.call(clio::meta{.number = I + 1, .name = BOOST_PP_STRINGIZE(member)}, \ + &OP::member, \ + std::forward(args)...); \ + } +#define CLIO_REFLECT_MPROXY_MEMBER_BY_IDX_INTERNAL(r, OP, I, member) \ + clio::member_proxy \ + member; + +#define CLIO_REFLECT_MPROXY_MEMBER_BY_IDX_HELPER(QUERY_CLASS, MEMBER_IDXS) \ + BOOST_PP_SEQ_FOR_EACH_I(CLIO_REFLECT_MPROXY_MEMBER_BY_IDX_INTERNAL, QUERY_CLASS, MEMBER_IDXS) + +#define CLIO_REFLECT_PROXY_MEMBER_BY_PB_INTERNAL(r, OP, member) \ + template \ + auto CLIO_REFLECT_FILTER_NAME member(Args&&... args) \ + { \ + return proxy___.call(clio::meta{.number = CLIO_REFLECT_FILTER_IDX member, \ + .name = CLIO_REFLECT_FILTER_NAME_STR member, \ + .param_names = CLIO_REFLECT_FILTER_PARAMS member}, \ + &OP::CLIO_REFLECT_FILTER_NAME member, std::forward(args)...); \ + } - #define CLIO_REFLECT_MEMBER_BY_NAME_I_INTERNAL( r, OP, I, member) \ - case clio::hash_name(BOOST_PP_STRINGIZE(member)): (void)lambda( &OP::member ); return true; +#define CLIO_REFLECT_FOREACH_MEMBER_HELPER(QUERY_CLASS, MEMBERS) \ + BOOST_PP_SEQ_FOR_EACH_I(CLIO_REFLECT_FOREACH_INTERNAL, QUERY_CLASS, MEMBERS) - #define CLIO_REFLECT_MEMBER_BY_NAME_HELPER( QUERY_CLASS, MEMBER_NAMES ) \ - BOOST_PP_SEQ_FOR_EACH_I( CLIO_REFLECT_MEMBER_BY_NAME_I_INTERNAL, QUERY_CLASS, MEMBER_NAMES ) +#define CLIO_REFLECT_MEMBER_BY_STR_HELPER(QUERY_CLASS, MEMBERS) \ + BOOST_PP_SEQ_FOR_EACH(CLIO_REFLECT_MEMBER_BY_STR_INTERNAL, QUERY_CLASS, MEMBERS) - #define CLIO_REFLECT_MEMBER_TYPE_BY_IDX_INTERNAL( r, OP, I, member ) \ - BOOST_PP_COMMA_IF( I ) std::decay_t< decltype(clio::result_of_member(&OP::member))> +#define CLIO_REFLECT_MEMBER_BY_IDX_I_INTERNAL(r, OP, I, member) \ + case I + 1: \ + (void)lambda(&OP::member); \ + return true; +#define CLIO_REFLECT_MEMBER_BY_IDX_HELPER(QUERY_CLASS, MEMBER_IDXS) \ + BOOST_PP_SEQ_FOR_EACH_I(CLIO_REFLECT_MEMBER_BY_IDX_I_INTERNAL, QUERY_CLASS, MEMBER_IDXS) - #define CLIO_REFLECT_MEMBER_TYPE_IDX_HELPER( QUERY_CLASS, MEMBER_IDXS ) \ - BOOST_PP_SEQ_FOR_EACH_I( CLIO_REFLECT_MEMBER_TYPE_BY_IDX_INTERNAL, QUERY_CLASS, MEMBER_IDXS ) +#define CLIO_REFLECT_MEMBER_BY_NAME_I_INTERNAL(r, OP, I, member) \ + case clio::hash_name(BOOST_PP_STRINGIZE(member)): \ + (void)lambda(&OP::member); \ + return true; +#define CLIO_REFLECT_MEMBER_BY_NAME_HELPER(QUERY_CLASS, MEMBER_NAMES) \ + BOOST_PP_SEQ_FOR_EACH_I(CLIO_REFLECT_MEMBER_BY_NAME_I_INTERNAL, QUERY_CLASS, MEMBER_NAMES) - #define CLIO_REFLECT_FOREACH_MEMBER_PB_HELPER( QUERY_CLASS, MEMBERS ) \ - BOOST_PP_SEQ_FOR_EACH( CLIO_REFLECT_FOREACH_PB_INTERNAL, QUERY_CLASS, MEMBERS ) +#define CLIO_REFLECT_MEMBER_TYPE_BY_IDX_INTERNAL(r, OP, I, member) \ + BOOST_PP_COMMA_IF(I) std::decay_t - #define CLIO_REFLECT_MEMBER_INDEX_HELPER( QUERY_CLASS, MEMBER_IDXS ) \ - BOOST_PP_SEQ_FOR_EACH_I( CLIO_REFLECT_MEMBER_PB_INTERNAL, QUERY_CLASS, MEMBER_IDXS ) +#define CLIO_REFLECT_MEMBER_TYPE_IDX_HELPER(QUERY_CLASS, MEMBER_IDXS) \ + BOOST_PP_SEQ_FOR_EACH_I(CLIO_REFLECT_MEMBER_TYPE_BY_IDX_INTERNAL, QUERY_CLASS, MEMBER_IDXS) - #define CLIO_REFLECT_MEMBER_BY_STR_PB_HELPER( QUERY_CLASS, MEMBER_IDXS ) \ - BOOST_PP_SEQ_FOR_EACH( CLIO_REFLECT_MEMBER_BY_STR_PB_INTERNAL, QUERY_CLASS, MEMBER_IDXS ) +#define CLIO_REFLECT_FOREACH_MEMBER_PB_HELPER(QUERY_CLASS, MEMBERS) \ + BOOST_PP_SEQ_FOR_EACH(CLIO_REFLECT_FOREACH_PB_INTERNAL, QUERY_CLASS, MEMBERS) - #define CLIO_REFLECT_MEMBER_BY_IDX_PB_HELPER( QUERY_CLASS, MEMBER_IDXS ) \ - BOOST_PP_SEQ_FOR_EACH( CLIO_REFLECT_MEMBER_BY_IDX_PB_INTERNAL, QUERY_CLASS, MEMBER_IDXS ) - -#define CLIO_REFLECT_PROXY_MEMBER_BY_IDX_HELPER( QUERY_CLASS, MEMBER_IDXS ) \ - BOOST_PP_SEQ_FOR_EACH_I( CLIO_REFLECT_PROXY_MEMBER_BY_IDX_INTERNAL, QUERY_CLASS, MEMBER_IDXS ) +#define CLIO_REFLECT_MEMBER_INDEX_HELPER(QUERY_CLASS, MEMBER_IDXS) \ + BOOST_PP_SEQ_FOR_EACH_I(CLIO_REFLECT_MEMBER_PB_INTERNAL, QUERY_CLASS, MEMBER_IDXS) -#define CLIO_REFLECT_PROXY_MEMBER_BY_PB_HELPER( QUERY_CLASS, MEMBER_IDXS ) \ - BOOST_PP_SEQ_FOR_EACH( CLIO_REFLECT_PROXY_MEMBER_BY_PB_INTERNAL, QUERY_CLASS, MEMBER_IDXS ) +#define CLIO_REFLECT_MEMBER_BY_STR_PB_HELPER(QUERY_CLASS, MEMBER_IDXS) \ + BOOST_PP_SEQ_FOR_EACH(CLIO_REFLECT_MEMBER_BY_STR_PB_INTERNAL, QUERY_CLASS, MEMBER_IDXS) +#define CLIO_REFLECT_MEMBER_BY_IDX_PB_HELPER(QUERY_CLASS, MEMBER_IDXS) \ + BOOST_PP_SEQ_FOR_EACH(CLIO_REFLECT_MEMBER_BY_IDX_PB_INTERNAL, QUERY_CLASS, MEMBER_IDXS) +#define CLIO_REFLECT_PROXY_MEMBER_BY_IDX_HELPER(QUERY_CLASS, MEMBER_IDXS) \ + BOOST_PP_SEQ_FOR_EACH_I(CLIO_REFLECT_PROXY_MEMBER_BY_IDX_INTERNAL, QUERY_CLASS, MEMBER_IDXS) - #define CLIO_REFLECT_PARAMS_BY_IDX_PB_INTERNAL( r, OP, i, member) \ - if constexpr ( std::is_member_function_pointer_v ) \ - return OP:: BOOST_PP_CAT( CLIO_REFLECT_FILTER_NAME member ,___PARAM_NAMES); +#define CLIO_REFLECT_PROXY_MEMBER_BY_PB_HELPER(QUERY_CLASS, MEMBER_IDXS) \ + BOOST_PP_SEQ_FOR_EACH(CLIO_REFLECT_PROXY_MEMBER_BY_PB_INTERNAL, QUERY_CLASS, MEMBER_IDXS) - #define CLIO_REFLECT_PARAMS_BY_IDX_PB_HELPER( QUERY_CLASS, MEMBER_IDXS ) \ - BOOST_PP_SEQ_FOR_EACH_I( CLIO_REFLECT_PARAMS_BY_IDX_PB_INTERNAL, QUERY_CLASS, MEMBER_IDXS ) +#define CLIO_REFLECT_PARAMS_BY_IDX_PB_INTERNAL(r, OP, i, member) \ + if constexpr (std::is_member_function_pointer_v) \ + return OP::BOOST_PP_CAT(CLIO_REFLECT_FILTER_NAME member, ___PARAM_NAMES); +#define CLIO_REFLECT_PARAMS_BY_IDX_PB_HELPER(QUERY_CLASS, MEMBER_IDXS) \ + BOOST_PP_SEQ_FOR_EACH_I(CLIO_REFLECT_PARAMS_BY_IDX_PB_INTERNAL, QUERY_CLASS, MEMBER_IDXS) - /* + /* template \ struct proxy { \ template \ @@ -191,109 +191,148 @@ namespace clio { }; \ */ - -#define CLIO_REFLECT(QUERY_CLASS, ...) \ - CLIO_REFLECT_TYPENAME( QUERY_CLASS ) \ - struct reflect_impl_##QUERY_CLASS { \ - static constexpr bool is_defined = true; \ - static constexpr bool is_struct = true; \ - static inline constexpr const char* name() { return BOOST_PP_STRINGIZE(QUERY_CLASS); } \ - typedef std::tuple< \ - CLIO_REFLECT_MEMBER_TYPE_IDX_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) > struct_tuple_type; \ - template constexpr inline static void for_each(L&& lambda) { \ - CLIO_REFLECT_FOREACH_MEMBER_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ - } \ - template inline static bool get(const std::string_view& m, L&& lambda) { \ - CLIO_REFLECT_MEMBER_BY_STR_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ - return false; \ - } \ - template inline static bool get(int64_t m, L&& lambda) { \ - switch (m) { \ - CLIO_REFLECT_MEMBER_BY_IDX_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ - } \ - return false; \ - } \ - template inline static bool get_by_name(uint64_t n, L&& lambda) { \ - switch (n) { \ +#define CLIO_REFLECT(QUERY_CLASS, ...) \ + CLIO_REFLECT_TYPENAME(QUERY_CLASS) \ + struct reflect_impl_##QUERY_CLASS \ + { \ + static constexpr bool is_defined = true; \ + static constexpr bool is_struct = true; \ + static inline constexpr const char* name() { return BOOST_PP_STRINGIZE(QUERY_CLASS); } \ + typedef std::tuple< \ + CLIO_REFLECT_MEMBER_TYPE_IDX_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__))> \ + struct_tuple_type; \ + template \ + constexpr inline static void for_each(L&& lambda) \ + { \ + CLIO_REFLECT_FOREACH_MEMBER_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + } \ + template \ + inline static bool get(const std::string_view& m, L&& lambda) \ + { \ + CLIO_REFLECT_MEMBER_BY_STR_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + return false; \ + } \ + template \ + inline static bool get(int64_t m, L&& lambda) \ + { \ + switch (m) \ + { \ + CLIO_REFLECT_MEMBER_BY_IDX_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + } \ + return false; \ + } \ + template \ + inline static bool get_by_name(uint64_t n, L&& lambda) \ + { \ + switch (n) \ + { \ CLIO_REFLECT_MEMBER_BY_NAME_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ - } \ - return false; \ - } \ - template \ - struct proxy { \ - private: \ - ProxyObject proxy___; \ - public: \ - template \ - proxy( Args&&... args ):proxy___( std::forward(args)... ){}\ - ProxyObject* operator->(){ return &proxy___; } \ - const ProxyObject* operator->()const{ return &proxy___; } \ - ProxyObject& operator*(){ return proxy___; } \ - const ProxyObject& operator*()const{ return proxy___; } \ - CLIO_REFLECT_MPROXY_MEMBER_BY_IDX_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ - }; \ - }; \ + } \ + return false; \ + } \ + template \ + struct proxy \ + { \ + private: \ + ProxyObject proxy___; \ + \ + public: \ + template \ + proxy(Args&&... args) : proxy___(std::forward(args)...) \ + { \ + } \ + ProxyObject* operator->() { return &proxy___; } \ + const ProxyObject* operator->() const { return &proxy___; } \ + ProxyObject& operator*() { return proxy___; } \ + const ProxyObject& operator*() const { return proxy___; } \ + CLIO_REFLECT_MPROXY_MEMBER_BY_IDX_HELPER(QUERY_CLASS, \ + BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + }; \ + }; \ reflect_impl_##QUERY_CLASS get_reflect_impl(const QUERY_CLASS&); +#define CLIO_REFLECT_PB(QUERY_CLASS, ...) \ + CLIO_REFLECT_TYPENAME(QUERY_CLASS) \ + struct reflect_impl_##QUERY_CLASS \ + { \ + static constexpr bool is_defined = true; \ + static constexpr bool is_struct = true; \ + static inline constexpr const char* name() { return BOOST_PP_STRINGIZE(QUERY_CLASS); } \ + template \ + inline static void for_each(L&& lambda) \ + { \ + CLIO_REFLECT_FOREACH_MEMBER_PB_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + } \ + template \ + inline static bool get(const std::string_view& m, L&& lambda) \ + { \ + CLIO_REFLECT_MEMBER_BY_STR_PB_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + return false; \ + } \ + template \ + inline static bool get(int64_t m, L&& lambda) \ + { \ + switch (m) \ + { \ + CLIO_REFLECT_MEMBER_BY_IDX_PB_HELPER(QUERY_CLASS, \ + BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + } \ + return false; \ + } \ + }; \ + reflect_impl_##QUERY_CLASS get_reflect_impl(const QUERY_CLASS&); -#define CLIO_REFLECT_PB(QUERY_CLASS, ...) \ - CLIO_REFLECT_TYPENAME( QUERY_CLASS ) \ - struct reflect_impl_##QUERY_CLASS { \ - static constexpr bool is_defined = true; \ - static constexpr bool is_struct = true; \ - static inline constexpr const char* name() { return BOOST_PP_STRINGIZE(QUERY_CLASS); } \ - template inline static void for_each(L&& lambda) { \ - CLIO_REFLECT_FOREACH_MEMBER_PB_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ - } \ - template inline static bool get(const std::string_view& m, L&& lambda) { \ - CLIO_REFLECT_MEMBER_BY_STR_PB_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ - return false; \ - } \ - template inline static bool get(int64_t m, L&& lambda) { \ - switch (m) { \ - CLIO_REFLECT_MEMBER_BY_IDX_PB_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ - } \ - return false; \ - } \ - }; \ - reflect_impl_##QUERY_CLASS get_reflect_impl(const QUERY_CLASS&); \ - - -#define CLIO_REFLECT_TEMPLATE_OBJECT(QUERY_CLASS, TPARAM, ...) \ - template struct reflect_impl_##QUERY_CLASS { \ - static constexpr bool is_defined = true; \ - static constexpr bool is_struct = true; \ - static inline const char* name() { return BOOST_PP_STRINGIZE(QUERY_CLASS) "<" BOOST_PP_STRINGIZE(TPARAM) ">"; } \ - template inline static void for_each(L&& lambda) { \ - CLIO_REFLECT_FOREACH_MEMBER_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ - } \ - template inline static void get(const std::string_view& m, L&& lambda) { \ - CLIO_REFLECT_MEMBER_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ - return false; \ - } \ - template inline static void get(int64_t m, L&& lambda) { \ - switch (m) { \ - CLIO_REFLECT_MEMBER_INDEX_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ - } \ - return false; \ - } \ - using tuple_type = std::tuple< \ - CLIO_REFLECT_MEMBER_TYPE_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) > \ - }; \ - template reflect_impl_##QUERY_CLASS get_reflect_impl(const QUERY_CLASS&); \ - constexpr const char* get_type_name( QUERY_CLASS* ) { return reflect_impl_##QUERY_CLASS :: name(); } - - template - struct reflect_undefined { +#define CLIO_REFLECT_TEMPLATE_OBJECT(QUERY_CLASS, TPARAM, ...) \ + template \ + struct reflect_impl_##QUERY_CLASS \ + { \ + static constexpr bool is_defined = true; \ + static constexpr bool is_struct = true; \ + static inline const char* name() \ + { \ + return BOOST_PP_STRINGIZE(QUERY_CLASS) "<" BOOST_PP_STRINGIZE(TPARAM) ">"; \ + } \ + template \ + inline static void for_each(L&& lambda) \ + { \ + CLIO_REFLECT_FOREACH_MEMBER_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + } \ + template \ + inline static void get(const std::string_view& m, L&& lambda) \ + { \ + CLIO_REFLECT_MEMBER_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + return false; \ + } \ + template \ + inline static void get(int64_t m, L&& lambda) \ + { \ + switch (m) \ + { \ + CLIO_REFLECT_MEMBER_INDEX_HELPER(QUERY_CLASS, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__)) \ + } \ + return false; \ + } \ + using tuple_type = \ + std::tuple \ + }; \ + template \ + reflect_impl_##QUERY_CLASS get_reflect_impl(const QUERY_CLASS&); \ + constexpr const char* get_type_name(QUERY_CLASS*) \ + { \ + return reflect_impl_##QUERY_CLASS ::name(); \ + } + + template + struct reflect_undefined + { static constexpr bool is_defined = false; - static constexpr bool is_struct = false; - template - static void get( const std::string_view& m, L&& lambda ); + static constexpr bool is_struct = false; + template + static void get(const std::string_view& m, L&& lambda); }; - - - /* + /* using std::string; CLIO_REFLECT_TYPENAME( int32_t ) CLIO_REFLECT_TYPENAME( int64_t ) @@ -310,79 +349,95 @@ namespace clio { CLIO_REFLECT_TYPENAME( string ) */ - - template + template reflect_undefined get_reflect_impl(const QueryClass&); - - template + template using reflect = std::decay_t()))>; - template - struct is_std_vector : std::false_type {}; - - template - struct is_std_vector> : std::true_type { using value_type = T; }; - - template - struct is_std_optional : std::false_type {}; - - template - struct is_std_optional> : std::true_type { using value_type = T; }; - - template - struct is_std_variant : std::false_type {}; - - template - struct is_std_variant> : std::true_type { - static std::string name() { - return get_variant_typename(); - } - template - static std::string get_variant_typename() { - if constexpr ( sizeof...(Rest) > 0 ) - return std::string( get_type_name() ) + "|" + get_variant_typename(); - else - return std::string( get_type_name() ) + "|"; - } - using alts_as_tuple = std::tuple; + template + struct is_std_vector : std::false_type + { }; - template - struct is_std_tuple : std::false_type {}; - - template - struct is_std_tuple> : std::true_type { - static std::string name() { - return get_tuple_typename(); - } - template - static std::string get_tuple_typename() { - if constexpr ( sizeof...(Rest) > 0 ) - return std::string( get_type_name() ) + "&" + get_tuple_typename(); - else - return std::string( get_type_name() ) + "&"; - } + template + struct is_std_vector> : std::true_type + { + using value_type = T; }; + template + struct is_std_optional : std::false_type + { + }; - template - struct is_std_map : std::false_type {}; + template + struct is_std_optional> : std::true_type + { + using value_type = T; + }; - template - struct is_std_map> : std::true_type { - static const std::string& name() { - static std::string n = std::string("map<") + get_type_name() +"," + get_type_name() + ">"; - return n; - } + template + struct is_std_variant : std::false_type + { }; + template + struct is_std_variant> : std::true_type + { + static std::string name() { return get_variant_typename(); } + template + static std::string get_variant_typename() + { + if constexpr (sizeof...(Rest) > 0) + return std::string(get_type_name()) + "|" + get_variant_typename(); + else + return std::string(get_type_name()) + "|"; + } + using alts_as_tuple = std::tuple; + }; -} /// namespace clio + template + struct is_std_tuple : std::false_type + { + }; + + template + struct is_std_tuple> : std::true_type + { + static std::string name() { return get_tuple_typename(); } + template + static std::string get_tuple_typename() + { + if constexpr (sizeof...(Rest) > 0) + return std::string(get_type_name()) + "&" + get_tuple_typename(); + else + return std::string(get_type_name()) + "&"; + } + }; + + template + struct is_std_map : std::false_type + { + }; + + template + struct is_std_map> : std::true_type + { + static const std::string& name() + { + static std::string n = + std::string("map<") + get_type_name() + "," + get_type_name() + ">"; + return n; + } + }; -namespace std { - namespace { - CLIO_REFLECT_TYPENAME( string ) - } -} +} // namespace clio +namespace std +{ + namespace + { + CLIO_REFLECT_TYPENAME(string) + } +} // namespace std diff --git a/libraries/clio/include/clio/schema.hpp b/libraries/clio/include/clio/schema.hpp index e1acc6236..73e892e9d 100644 --- a/libraries/clio/include/clio/schema.hpp +++ b/libraries/clio/include/clio/schema.hpp @@ -1,63 +1,69 @@ #pragma once -#include -#include -#include -#include #include #include +#include +#include +#include +#include -namespace clio { - using std::string; - using std::vector; - - enum scalar_type { - int64_type, - int32_type, - int16_type, - int8_type, - uint64_type, - uint32_type, - uint16_type, - uint8_type, - varuint32_type, - double_type, - float_type, - string_type, - bool_type - }; - - using type_name = std::string; - - struct variant_type { - vector types; - }; - - CLIO_REFLECT( variant_type, types ) - - struct tuple_type { - vector types; - }; - CLIO_REFLECT( tuple_type, types ) - - struct vector_type { - type_name type; - }; - CLIO_REFLECT( vector_type, type ) - - - struct enum_type { - scalar_type value_type; - struct enum_value { - string name; - int64_t value; - }; - vector values; - }; - using enum_value = enum_type::enum_value; - CLIO_REFLECT( enum_value, name, value ) - CLIO_REFLECT( enum_type, values ) - - /* +namespace clio +{ + using std::string; + using std::vector; + + enum scalar_type + { + int64_type, + int32_type, + int16_type, + int8_type, + uint64_type, + uint32_type, + uint16_type, + uint8_type, + varuint32_type, + double_type, + float_type, + string_type, + bool_type + }; + + using type_name = std::string; + + struct variant_type + { + vector types; + }; + + CLIO_REFLECT(variant_type, types) + + struct tuple_type + { + vector types; + }; + CLIO_REFLECT(tuple_type, types) + + struct vector_type + { + type_name type; + }; + CLIO_REFLECT(vector_type, type) + + struct enum_type + { + scalar_type value_type; + struct enum_value + { + string name; + int64_t value; + }; + vector values; + }; + using enum_value = enum_type::enum_value; + CLIO_REFLECT(enum_value, name, value) + CLIO_REFLECT(enum_type, values) + + /* struct function_type { struct parameter { type_name type; @@ -71,218 +77,251 @@ namespace clio { CLIO_REFLECT( function_type, args, result ) */ - struct object_type { - - - struct member { - string name; - type_name type; - int32_t number; - - template - void set_params( std::initializer_list names, R (T::*method)(Args...) ) { - if constexpr ( sizeof...(Args) > 0 ) - push_param( names.begin(), names.end() ); - } - template - void set_params( std::initializer_list names, R (T::*method)(Args...)const ) { - if constexpr ( sizeof...(Args) > 0 ) - push_param(names.begin(), names.end()); - } - - struct param { - string name; - type_name type; - }; - std::vector params; - - private: - template - void push_param( std::initializer_list::iterator begin, - std::initializer_list::iterator end ) { - params.push_back( { - .name = begin != end ? *begin : "", - .type = get_type_name>() - } ); - if constexpr ( sizeof...(Args) > 0 ) - push_param( begin != end ? ++begin : end, end ); - } - }; - - const member* get_member_by_name( const std::string_view& n )const { - for( const auto& m : members ) - if( m.name == n ) return &m; - return nullptr; - - } - - const member* get_member_by_number( uint32_t n )const { - for( const auto& m : members ) - if( m.number == n ) return &m; - return nullptr; - } - - vector members; - }; - - using object_member = object_type::member; - using object_method_param = object_type::member::param; - CLIO_REFLECT( object_method_param, name, type ) - CLIO_REFLECT( object_member, name, type, number, params ) - CLIO_REFLECT( object_type, members ) - - struct typedef_type { - type_name type; - }; - CLIO_REFLECT( typedef_type, type ) - - using schema_type = std::variant< - object_type, - variant_type, - tuple_type, - vector_type, - enum_type, - typedef_type>; - CLIO_REFLECT_TYPENAME( schema_type ) - - - - struct schema { - std::map types; - - std::optional get_type( const std::string& name )const { - auto itr = types.find(name); - if( itr == types.end() ) return {}; - return itr->second; - } - std::optional get_object( const string& name )const { - auto itr = types.find(name); - if( itr == types.end() ) return {}; - auto objptr = std::get_if(&itr->second); - if( objptr ) - return *objptr; + struct object_type + { + struct member + { + string name; + type_name type; + int32_t number; + + template + void set_params(std::initializer_list names, R (T::*method)(Args...)) + { + if constexpr (sizeof...(Args) > 0) + push_param(names.begin(), names.end()); + } + template + void set_params(std::initializer_list names, R (T::*method)(Args...) const) + { + if constexpr (sizeof...(Args) > 0) + push_param(names.begin(), names.end()); + } + + struct param + { + string name; + type_name type; + }; + std::vector params; + + private: + template + void push_param(std::initializer_list::iterator begin, + std::initializer_list::iterator end) + { + params.push_back( + {.name = begin != end ? *begin : "", .type = get_type_name>()}); + if constexpr (sizeof...(Args) > 0) + push_param(begin != end ? ++begin : end, end); + } + }; + + const member* get_member_by_name(const std::string_view& n) const + { + for (const auto& m : members) + if (m.name == n) + return &m; + return nullptr; + } + + const member* get_member_by_number(uint32_t n) const + { + for (const auto& m : members) + if (m.number == n) + return &m; + return nullptr; + } + + vector members; + }; + + using object_member = object_type::member; + using object_method_param = object_type::member::param; + CLIO_REFLECT(object_method_param, name, type) + CLIO_REFLECT(object_member, name, type, number, params) + CLIO_REFLECT(object_type, members) + + struct typedef_type + { + type_name type; + }; + CLIO_REFLECT(typedef_type, type) + + using schema_type = + std::variant; + CLIO_REFLECT_TYPENAME(schema_type) + + struct schema + { + std::map types; + + std::optional get_type(const std::string& name) const + { + auto itr = types.find(name); + if (itr == types.end()) return {}; - } - std::optional get_vector( const string& name )const { - auto itr = types.find(name); - if( itr == types.end() ) return {}; - auto objptr = std::get_if(&itr->second); - if( objptr ) - return *objptr; + return itr->second; + } + std::optional get_object(const string& name) const + { + auto itr = types.find(name); + if (itr == types.end()) return {}; - } - template - bool visit_type( const std::string& type, Visitor&& v ) { - auto itr = types.find( type ); - if( itr == types.end() ) return false; - std::visit( std::forward(v), itr->second ); - return true; - } - - template - bool add_type( schema_type t, L&& on_generate ) { - auto tn = get_type_name(); - if( types.find( tn ) == types.end() ) { - on_generate( static_cast(nullptr) ); - types[tn] = t; - return true; - } + auto objptr = std::get_if(&itr->second); + if (objptr) + return *objptr; + return {}; + } + std::optional get_vector(const string& name) const + { + auto itr = types.find(name); + if (itr == types.end()) + return {}; + auto objptr = std::get_if(&itr->second); + if (objptr) + return *objptr; + return {}; + } + template + bool visit_type(const std::string& type, Visitor&& v) + { + auto itr = types.find(type); + if (itr == types.end()) return false; - } - - template - void generate_map( const std::map*, L&& on_generate ) { - generate( on_generate ); - } - - template - void generate_variant( const std::tuple* v, L&& on_generate ) { - if( add_type>( tuple_type(), on_generate ) ) { - tuple_type vt; - generate_helper(vt, on_generate ); - types[get_type_name>()] = vt; - } - } - - template - void generate_variant( const std::variant* v, L&& on_generate ) { - auto tn = get_type_name>(); - if( add_type>( variant_type(), on_generate ) ) + std::visit(std::forward(v), itr->second); + return true; + } + + template + bool add_type(schema_type t, L&& on_generate) + { + auto tn = get_type_name(); + if (types.find(tn) == types.end()) + { + on_generate(static_cast(nullptr)); + types[tn] = t; + return true; + } + return false; + } + + template + void generate_map(const std::map*, L&& on_generate) + { + generate(on_generate); + } + + template + void generate_variant(const std::tuple* v, L&& on_generate) + { + if (add_type>(tuple_type(), on_generate)) + { + tuple_type vt; + generate_helper(vt, on_generate); + types[get_type_name>()] = vt; + } + } + + template + void generate_variant(const std::variant* v, L&& on_generate) + { + auto tn = get_type_name>(); + if (add_type>(variant_type(), on_generate)) + { + variant_type vt; + generate_helper(vt, on_generate); + types[tn] = vt; + } + } + + template + void generate_helper(OT& vt, L&& on_generate) + { + generate(on_generate); + vt.types.push_back(get_type_name()); + if constexpr (sizeof...(Args) > 0) + { + generate_helper(vt, on_generate); + } + } + + template + void generate() + { + generate([](auto) {}); + } + + template + void generate(L on_generate) + { + if constexpr (std::is_same_v>) + { + } + else if constexpr (is_std_map::value) + { + generate_map((const T*)nullptr, on_generate); + } + else if constexpr (is_std_tuple::value) + { + generate_variant((const T*)nullptr, on_generate); + } + else if constexpr (is_std_variant::value) + { + generate_variant((const T*)nullptr, on_generate); + } + else if constexpr (is_std_optional::value) + { + generate::value_type>(on_generate); + } + else if constexpr (is_std_vector::value) + { + using value_type = typename is_std_vector::value_type; + if (add_type(vector_type{get_type_name()}, on_generate)) { - variant_type vt; - generate_helper(vt, on_generate); - types[tn] = vt; - } - } - - template - void generate_helper( OT& vt, L&& on_generate ) { - generate( on_generate ); - vt.types.push_back( get_type_name() ); - if constexpr ( sizeof...(Args) > 0 ) { - generate_helper( vt, on_generate ); - } - } - - template - void generate() { - generate( [](auto){} ); - } - - template - void generate( L on_generate ) { - if constexpr ( std::is_same_v> ){} - else if constexpr ( is_std_map::value ) { - generate_map( (const T*)nullptr, on_generate ); - } - else if constexpr ( is_std_tuple::value ) { - generate_variant( (const T*)nullptr, on_generate ); - } - else if constexpr ( is_std_variant::value ) { - generate_variant( (const T*)nullptr, on_generate ); - } - else if constexpr ( is_std_optional::value ) { - generate::value_type>( on_generate ); - } - else if constexpr ( is_std_vector::value ) { - using value_type = typename is_std_vector::value_type; - if( add_type( vector_type{ get_type_name() }, on_generate ) ) { - generate( on_generate ); - } + generate(on_generate); } - else if constexpr ( reflect::is_struct ) { - auto n = get_type_name(); - if( add_type( object_type(), on_generate ) ) { - object_type ot; - reflect::for_each( [&]( const meta& r, auto m ) { - if constexpr( not std::is_member_function_pointer_v ) { - using member_type = std::decay_t*>(nullptr)->*m )>; - generate( on_generate ); - - auto tn = get_type_name(); - ot.members.push_back( { .name = r.name, .type = tn, .number = r.number } ); - } else { - using member_type = decltype(result_of( m )); - generate( on_generate ); - auto tn = get_type_name(); - ot.members.push_back( { .name = r.name, .type = tn, .number = r.number } ); - ot.members.back().set_params( r.param_names, m ); - } - }); - types[n] = ot; - } + } + else if constexpr (reflect::is_struct) + { + auto n = get_type_name(); + if (add_type(object_type(), on_generate)) + { + object_type ot; + reflect::for_each([&](const meta& r, auto m) { + if constexpr (not std::is_member_function_pointer_v) + { + using member_type = + std::decay_t*>(nullptr)->*m)>; + generate(on_generate); + + auto tn = get_type_name(); + ot.members.push_back({.name = r.name, .type = tn, .number = r.number}); + } + else + { + using member_type = decltype(result_of(m)); + generate(on_generate); + auto tn = get_type_name(); + ot.members.push_back({.name = r.name, .type = tn, .number = r.number}); + ot.members.back().set_params(r.param_names, m); + } + }); + types[n] = ot; } - } /// generate - }; - - CLIO_REFLECT( schema, types ) - - -} /// clio - -namespace std { - namespace { - using clio::schema_type; - CLIO_REFLECT_TYPENAME( schema_type ) - } -} + } + } /// generate + }; + + CLIO_REFLECT(schema, types) + +} // namespace clio + +namespace std +{ + namespace + { + using clio::schema_type; + CLIO_REFLECT_TYPENAME(schema_type) + } // namespace +} // namespace std diff --git a/libraries/clio/include/clio/stream.hpp b/libraries/clio/include/clio/stream.hpp index 726591d10..b76eedfe8 100644 --- a/libraries/clio/include/clio/stream.hpp +++ b/libraries/clio/include/clio/stream.hpp @@ -1,46 +1,54 @@ #pragma once +#include +#include #include #include #include -#include -#include -#include -#include #include +#include +#include -namespace clio { -enum class stream_error { - no_error, - overrun, - underrun, - float_error, - varuint_too_big, - invalid_varuint_encoding, - bad_variant_index, - invalid_asset_format, - array_size_mismatch, - invalid_name_char, - invalid_name_char13, - name_too_long, - json_writer_error, // !!! -}; // stream_error -} // namespace clio - -namespace std { -template <> -struct is_error_code_enum : true_type {}; -} // namespace std - -namespace clio { - -class stream_error_category_type : public std::error_category { - public: - const char* name() const noexcept override final { return "ConversionError"; } - - std::string message(int c) const override final { - switch (static_cast(c)) { - // clang-format off +namespace clio +{ + enum class stream_error + { + no_error, + overrun, + underrun, + float_error, + varuint_too_big, + invalid_varuint_encoding, + bad_variant_index, + invalid_asset_format, + array_size_mismatch, + invalid_name_char, + invalid_name_char13, + name_too_long, + json_writer_error, // !!! + }; // stream_error +} // namespace clio + +namespace std +{ + template <> + struct is_error_code_enum : true_type + { + }; +} // namespace std + +namespace clio +{ + class stream_error_category_type : public std::error_category + { + public: + const char* name() const noexcept override final { return "ConversionError"; } + + std::string message(int c) const override final + { + switch (static_cast(c)) + { + // clang-format off case stream_error::no_error: return "No error"; case stream_error::overrun: return "Stream overrun"; case stream_error::underrun: return "Stream underrun"; @@ -54,270 +62,299 @@ class stream_error_category_type : public std::error_category { case stream_error::invalid_name_char13: return "thirteenth character in name cannot be a letter that comes after j"; case stream_error::name_too_long: return "string is too long to be a valid name"; case stream_error::json_writer_error: return "Error writing json"; - // clang-format on + // clang-format on - default: return "unknown"; + default: + return "unknown"; + } } + }; // stream_error_category_type + + inline const stream_error_category_type& stream_error_category() + { + static stream_error_category_type c; + return c; } -}; // stream_error_category_type -inline const stream_error_category_type& stream_error_category() { - static stream_error_category_type c; - return c; -} + inline std::error_code make_error_code(stream_error e) + { + return {static_cast(e), stream_error_category()}; + } -inline std::error_code make_error_code(stream_error e) { return { static_cast(e), stream_error_category() }; } + template + constexpr bool has_bitwise_serialization() + { + if constexpr (std::is_arithmetic_v) + { + return true; + } + else if constexpr (std::is_enum_v) + { + static_assert(!std::is_convertible_v>, + "Serializing unscoped enum"); + return true; + } + else + { + return false; + } + } + template + struct small_buffer + { + char data[max_size]; + char* pos{data}; -template -constexpr bool has_bitwise_serialization() { - if constexpr (std::is_arithmetic_v) { - return true; - } else if constexpr (std::is_enum_v) { - static_assert(!std::is_convertible_v>, "Serializing unscoped enum"); - return true; - } else { - return false; - } -} + void reverse() { std::reverse(data, pos); } + }; -template -struct small_buffer { - char data[max_size]; - char* pos{ data }; + struct string_stream + { + std::string& data; + string_stream(std::string& data) : data(data) {} - void reverse() { std::reverse(data, pos); } -}; + void write(char ch) { data += ch; } -struct string_stream { - std::string& data; - string_stream(std::string& data) : data(data) {} + void write(const void* src, size_t size) + { + auto s = reinterpret_cast(src); + data.insert(data.end(), s, s + size); + } - void write(char ch) { - data += ch; - } + template + void write(const char (&src)[size]) + { + write(src, size - 1); + } - void write(const void* src, size_t size) { - auto s = reinterpret_cast(src); - data.insert(data.end(), s, s + size); - } + template + void write_raw(const T& v) + { + write(&v, sizeof(v)); + } - template - void write(const char (&src)[size]) { - write(src, size-1); - } + void write(const std::string& v) { write(v.c_str(), v.size()); } + }; - template - void write_raw(const T& v) { - write(&v, sizeof(v)); - } + struct vector_stream + { + std::vector& data; + vector_stream(std::vector& data) : data(data) {} - void write(const std::string& v) { - write(v.c_str(), v.size()); - } -}; + void write(char ch) { data.push_back(ch); } + void write(const void* src, size_t size) + { + auto s = reinterpret_cast(src); + data.insert(data.end(), s, s + size); + } -struct vector_stream { - std::vector& data; - vector_stream(std::vector& data) : data(data) {} + template + void write(const char (&src)[size]) + { + write(src, size - 1); + } - void write(char ch) { - data.push_back(ch); - } + template + void write_raw(const T& v) + { + write(&v, sizeof(v)); + } - void write(const void* src, size_t size) { - auto s = reinterpret_cast(src); - data.insert(data.end(), s, s + size); - } + void write(const std::string& v) { write(v.c_str(), v.size()); } + }; - template - void write(const char (&src)[size]) { - write(src, size-1); - } + struct fixed_buf_stream + { + char* begin; + char* pos; + char* end; - template - void write_raw(const T& v) { - write(&v, sizeof(v)); - } + fixed_buf_stream(char* pos, size_t size) : begin(pos), pos{pos}, end{pos + size} {} - void write(const std::string& v) { - write(v.c_str(), v.size()); - } -}; + void write(char ch) + { + if (pos >= end) + throw_error(stream_error::overrun); + *pos++ = ch; + } + + void write(const void* src, size_t size) + { + if (pos + size > end) + throw_error(stream_error::overrun); + memcpy(pos, src, size); + pos += size; + } -struct fixed_buf_stream { - char* begin; - char* pos; - char* end; + template + void write(const char (&src)[size]) + { + write(src, size - 1); + } - fixed_buf_stream(char* pos, size_t size) : begin(pos), pos{ pos }, end{ pos + size } {} + template + void write_raw(const T& v) + { + write(&v, sizeof(v)); + } - void write(char ch) { - if (pos >= end) - throw_error(stream_error::overrun); - *pos++ = ch; - } + void write(const std::string& v) { write(v.c_str(), v.size()); } - void write(const void* src, size_t size) { - if (pos + size > end) - throw_error(stream_error::overrun); - memcpy(pos, src, size); - pos += size; - } + void skip(int32_t s) + { + if ((pos + s > end) or (pos + s < begin)) + throw_error(stream_error::overrun); + pos += s; + } + auto get_pos() const { return pos; } - template - void write(const char (&src)[size]) { - write(src, size-1); - } + size_t remaining() { return end - pos; } + }; - template - void write_raw(const T& v) { - write(&v, sizeof(v)); - } + struct size_stream + { + size_t size = 0; - void write(const std::string& v) { - write(v.c_str(), v.size()); - } + size_t get_pos() const { return size; } + void write(char ch) { ++size; } - void skip( int32_t s ) { - if( (pos + s > end) or (pos + s < begin) ) - throw_error( stream_error::overrun ); - pos += s; - } - auto get_pos()const { return pos; } + void write(const void* src, size_t size) { this->size += size; } - size_t remaining() { return end - pos; } -}; + template + void write(const char (&src)[size]) + { + this->size += size - 1; + } + + template + void write_raw(const T& v) + { + size += sizeof(v); + } -struct size_stream { - size_t size = 0; + void write(const std::string& v) { write(v.c_str(), v.size()); } - size_t get_pos()const { return size; } - void write(char ch) { - ++size; + void skip(int32_t s) { size += s; } + }; + + template + void increase_indent(S&) + { } - void write(const void* src, size_t size) { - this->size += size; + template + void decrease_indent(S&) + { } - template - void write(const char (&src)[size]) { - this->size += size-1; + template + void write_colon(S& s) + { + s.write(':'); } - template - void write_raw(const T& v) { - size += sizeof(v); + template + void write_newline(S&) + { } - void write(const std::string& v) { - write(v.c_str(), v.size()); + template + struct pretty_stream : Base + { + using Base::Base; + int indent_size = 4; + std::vector current_indent; + }; + + template + void increase_indent(pretty_stream& s) + { + s.current_indent.resize(s.current_indent.size() + s.indent_size, ' '); } - void skip( int32_t s ) { - size += s; + template + void decrease_indent(pretty_stream& s) + { + if (s.current_indent.size() < s.indent_size) + throw_error(stream_error::overrun); + s.current_indent.resize(s.current_indent.size() - s.indent_size); } -}; - -template -void increase_indent(S&) { } - -template -void decrease_indent(S&) { } - -template -void write_colon(S& s) { - s.write(':'); -} - -template -void write_newline(S&) { -} - -template -struct pretty_stream : Base { - using Base::Base; - int indent_size = 4; - std::vector current_indent; -}; - -template -void increase_indent(pretty_stream& s) { - s.current_indent.resize(s.current_indent.size() + s.indent_size, ' '); -} - -template -void decrease_indent(pretty_stream& s) { - if (s.current_indent.size() < s.indent_size) - throw_error(stream_error::overrun); - s.current_indent.resize(s.current_indent.size() - s.indent_size); -} - -template -void write_colon(pretty_stream& s) { - s.write(": ", 2); -} - -template -void write_newline(pretty_stream& s) { - s.write('\n'); - s.write(s.current_indent.data(), s.current_indent.size()); -} - -struct input_stream { - const char* pos; - const char* end; - - input_stream() : pos{ nullptr }, end{ nullptr } {} - input_stream(const char* pos, size_t size) : pos{ pos }, end{ pos + size } { - if ( size < 0 ) - throw_error( stream_error::overrun ); + + template + void write_colon(pretty_stream& s) + { + s.write(": ", 2); } - input_stream(const char* pos, const char* end) : pos{ pos }, end{ end } { - if ( end < pos ) - throw_error( stream_error::overrun ); + + template + void write_newline(pretty_stream& s) + { + s.write('\n'); + s.write(s.current_indent.data(), s.current_indent.size()); } - input_stream(const std::vector& v) : pos{ v.data() }, end{ v.data() + v.size() } {} - input_stream(std::string_view v) : pos{ v.data() }, end{ v.data() + v.size() } {} - input_stream(const input_stream&) = default; - input_stream& operator=(const input_stream&) = default; + struct input_stream + { + const char* pos; + const char* end; - size_t remaining() { return end - pos; } + input_stream() : pos{nullptr}, end{nullptr} {} + input_stream(const char* pos, size_t size) : pos{pos}, end{pos + size} + { + if (size < 0) + throw_error(stream_error::overrun); + } + input_stream(const char* pos, const char* end) : pos{pos}, end{end} + { + if (end < pos) + throw_error(stream_error::overrun); + } + input_stream(const std::vector& v) : pos{v.data()}, end{v.data() + v.size()} {} + input_stream(std::string_view v) : pos{v.data()}, end{v.data() + v.size()} {} + input_stream(const input_stream&) = default; - void check_available(size_t size) { - if (size > size_t(end - pos)) - throw_error(stream_error::overrun); - } + input_stream& operator=(const input_stream&) = default; - auto get_pos()const { return pos; } + size_t remaining() { return end - pos; } - void read(void* dest, size_t size) { - if (size > size_t(end - pos)) - throw_error( stream_error::overrun ); - memcpy(dest, pos, size); - pos += size; - } + void check_available(size_t size) + { + if (size > size_t(end - pos)) + throw_error(stream_error::overrun); + } - template - void read_raw(T& dest) { - read(&dest, sizeof(dest)); - } + auto get_pos() const { return pos; } - void skip(size_t size) { - if (size > size_t(end - pos)) - throw_error(stream_error::overrun); - pos += size; - } + void read(void* dest, size_t size) + { + if (size > size_t(end - pos)) + throw_error(stream_error::overrun); + memcpy(dest, pos, size); + pos += size; + } - void read_reuse_storage(const char*& result, size_t size) { - if (size > size_t(end - pos)) - throw_error( stream_error::overrun ); - result = pos; - pos += size; - } -}; + template + void read_raw(T& dest) + { + read(&dest, sizeof(dest)); + } + + void skip(size_t size) + { + if (size > size_t(end - pos)) + throw_error(stream_error::overrun); + pos += size; + } + + void read_reuse_storage(const char*& result, size_t size) + { + if (size > size_t(end - pos)) + throw_error(stream_error::overrun); + result = pos; + pos += size; + } + }; -} // namespace clio +} // namespace clio diff --git a/libraries/clio/include/clio/to_bin.hpp b/libraries/clio/include/clio/to_bin.hpp index c28ee6f38..75989feec 100644 --- a/libraries/clio/include/clio/to_bin.hpp +++ b/libraries/clio/include/clio/to_bin.hpp @@ -1,172 +1,211 @@ #pragma once -#include #include -#include +#include #include +#include + +namespace clio +{ + template + std::vector to_bin(const T& t); + + template + void to_bin(std::string_view sv, S& stream); + + template + void to_bin(const std::string& s, S& stream); + + template + void to_bin(const std::vector& obj, S& stream); + + template + void to_bin(const std::optional& obj, S& stream); + + template + void to_bin(const std::variant& obj, S& stream); + + template + void to_bin(const std::tuple& obj, S& stream); + + template + void to_bin(const T& obj, S& stream); + + template + void varuint32_to_bin(uint64_t val, S& stream) + { + if (val >> 32) + { + /// TODO throw error return stream_error::varuint_too_big; + return; + } + do + { + uint8_t b = val & 0x7f; + val >>= 7; + b |= ((val > 0) << 7); + stream.write(b); + } while (val); + } + + // !!! temp + inline void push_varuint32(std::vector& bin, uint32_t v) + { + vector_stream st{bin}; + varuint32_to_bin(v, st); + } + + template + void to_bin(std::string_view sv, S& stream) + { + varuint32_to_bin(sv.size(), stream); + stream.write(sv.data(), sv.size()); + } + + template + void to_bin(const std::string& s, S& stream) + { + to_bin(std::string_view{s}, stream); + } + + template + void to_bin_range(const T& obj, S& stream) + { + varuint32_to_bin(obj.size(), stream); + for (auto& x : obj) + { + to_bin(x, stream); + } + } + + template + void to_bin(const T (&obj)[N], S& stream) + { + varuint32_to_bin(N, stream); + if constexpr (has_bitwise_serialization()) + { + stream.write(reinterpret_cast(&obj), N * sizeof(T)); + } + else + { + for (auto& x : obj) + { + to_bin(x, stream); + } + } + } + + template + void to_bin(const std::vector& obj, S& stream) + { + varuint32_to_bin(obj.size(), stream); + if constexpr (has_bitwise_serialization()) + { + stream.write(reinterpret_cast(obj.data()), obj.size() * sizeof(T)); + } + else + { + for (auto& x : obj) + { + to_bin(x, stream); + } + } + } + + template + void to_bin(const std::variant& obj, S& stream) + { + varuint32_to_bin(obj.index(), stream); + std::visit([&](auto& x) { to_bin(x, stream); }, obj); + } + + template + void to_bin(const input_stream& obj, S& stream) + { + varuint32_to_bin(obj.end - obj.pos, stream); + stream.write(obj.pos, obj.end - obj.pos); + } + + template + void to_bin(const std::pair& obj, S& stream) + { + to_bin(obj.first, stream); + return to_bin(obj.second, stream); + } + + template + void to_bin(const std::optional& obj, S& stream) + { + to_bin(obj.has_value(), stream); + if (obj) + to_bin(*obj, stream); + } + + template + void to_bin_tuple(const T& obj, S& stream) + { + if constexpr (i < std::tuple_size_v) + { + to_bin(std::get(obj), stream); + to_bin_tuple(obj, stream); + } + } + + template + void to_bin(const std::tuple& obj, S& stream) + { + return to_bin_tuple<0>(obj, stream); + } + + template + void to_bin(const std::array& obj, S& stream) + { + for (const T& elem : obj) + { + to_bin(elem, stream); + } + } + + template + void to_bin(const T& obj, S& stream) + { + if constexpr (has_bitwise_serialization()) + { + stream.write(reinterpret_cast(&obj), sizeof(obj)); + } + else + { + reflect::for_each([&](const clio::meta&, auto m) { + if constexpr (not std::is_member_function_pointer_v) + { + to_bin(obj.*m, stream); + } + }); + } + } + + template + void convert_to_bin(const T& t, std::vector& bin) + { + size_stream ss; + to_bin(t, ss); + + auto orig_size = bin.size(); + bin.resize(orig_size + ss.size); + fixed_buf_stream fbs(bin.data() + orig_size, ss.size); + to_bin(t, fbs); + + /** TODO maybe throw + if (fbs.pos != fbs.end) + return stream_error::underrun; + */ + } + + template + std::vector to_bin(const T& t) + { + std::vector result; + convert_to_bin(t, result); + return result; + } -namespace clio { - -template -std::vector to_bin(const T& t); - -template -void to_bin(std::string_view sv, S& stream); - -template -void to_bin(const std::string& s, S& stream); - -template -void to_bin(const std::vector& obj, S& stream); - -template -void to_bin(const std::optional& obj, S& stream); - -template -void to_bin(const std::variant& obj, S& stream); - -template -void to_bin(const std::tuple& obj, S& stream); - -template -void to_bin(const T& obj, S& stream); - -template -void varuint32_to_bin(uint64_t val, S& stream) { - if (val >> 32) { - /// TODO throw error return stream_error::varuint_too_big; - return; - } - do { - uint8_t b = val & 0x7f; - val >>= 7; - b |= ((val > 0) << 7); - stream.write(b); - } while (val); -} - -// !!! temp -inline void push_varuint32(std::vector& bin, uint32_t v) { - vector_stream st{ bin }; - varuint32_to_bin(v, st); -} - -template -void to_bin(std::string_view sv, S& stream) { - varuint32_to_bin(sv.size(), stream); - stream.write(sv.data(), sv.size()); -} - -template -void to_bin(const std::string& s, S& stream) { - to_bin(std::string_view{ s }, stream); -} - -template -void to_bin_range(const T& obj, S& stream) { - varuint32_to_bin(obj.size(), stream); - for (auto& x : obj) { to_bin(x, stream); } -} - -template -void to_bin(const T (&obj)[N], S& stream) { - varuint32_to_bin(N, stream); - if constexpr (has_bitwise_serialization()) { - stream.write(reinterpret_cast(&obj), N * sizeof(T)); - } else { - for (auto& x : obj) { to_bin(x, stream); } - } -} - -template -void to_bin(const std::vector& obj, S& stream) { - varuint32_to_bin(obj.size(), stream); - if constexpr (has_bitwise_serialization()) { - stream.write(reinterpret_cast(obj.data()), obj.size() * sizeof(T)); - } else { - for (auto& x : obj) { to_bin(x, stream); } - } -} - -template -void to_bin(const std::variant& obj, S& stream) { - varuint32_to_bin(obj.index(), stream); - std::visit([&](auto& x) { to_bin(x, stream); }, obj); -} - - -template -void to_bin(const input_stream& obj, S& stream) { - varuint32_to_bin(obj.end - obj.pos, stream); - stream.write(obj.pos, obj.end - obj.pos); -} - -template -void to_bin(const std::pair& obj, S& stream) { - to_bin(obj.first, stream); - return to_bin(obj.second, stream); -} - -template -void to_bin(const std::optional& obj, S& stream) { - to_bin(obj.has_value(), stream); - if (obj) to_bin(*obj, stream); -} - - -template -void to_bin_tuple(const T& obj, S& stream) { - if constexpr (i < std::tuple_size_v) { - to_bin(std::get(obj), stream); - to_bin_tuple(obj, stream); - } -} - -template -void to_bin(const std::tuple& obj, S& stream) { - return to_bin_tuple<0>(obj, stream); -} - -template -void to_bin(const std::array& obj, S& stream) { - for (const T& elem : obj) { to_bin(elem, stream); } -} - -template -void to_bin(const T& obj, S& stream) { - if constexpr (has_bitwise_serialization()) { - stream.write(reinterpret_cast(&obj), sizeof(obj)); - } else { - - reflect::for_each( [&]( const clio::meta&, auto m ) { - if constexpr( not std::is_member_function_pointer_v ) { - to_bin( obj.*m, stream ); - } - }); - } -} - -template -void convert_to_bin(const T& t, std::vector& bin) { - size_stream ss; - to_bin(t, ss); - - auto orig_size = bin.size(); - bin.resize(orig_size + ss.size); - fixed_buf_stream fbs(bin.data() + orig_size, ss.size); - to_bin(t, fbs); - - /** TODO maybe throw - if (fbs.pos != fbs.end) - return stream_error::underrun; - */ -} - -template -std::vector to_bin(const T& t) { - std::vector result; - convert_to_bin(t, result); - return result; -} - - -} // clio +} // namespace clio diff --git a/libraries/clio/include/clio/to_bin/deque.hpp b/libraries/clio/include/clio/to_bin/deque.hpp index e20952671..df66b0337 100644 --- a/libraries/clio/include/clio/to_bin/deque.hpp +++ b/libraries/clio/include/clio/to_bin/deque.hpp @@ -2,11 +2,12 @@ #include #include -namespace clio { +namespace clio +{ + template + result to_bin(const std::deque& obj, S& stream) + { + return to_bin_range(obj, stream); + } - template - result to_bin(const std::deque& obj, S& stream) { - return to_bin_range(obj, stream); - } - -} // clio +} // namespace clio diff --git a/libraries/clio/include/clio/to_bin/list.hpp b/libraries/clio/include/clio/to_bin/list.hpp index 367a124ba..a8dcde1e4 100644 --- a/libraries/clio/include/clio/to_bin/list.hpp +++ b/libraries/clio/include/clio/to_bin/list.hpp @@ -2,11 +2,12 @@ #include #include -namespace clio { +namespace clio +{ + template + result to_bin(const std::list& obj, S& stream) + { + return to_bin_range(obj, stream); + } - template - result to_bin(const std::list& obj, S& stream) { - return to_bin_range(obj, stream); - } - -} // clio +} // namespace clio diff --git a/libraries/clio/include/clio/to_bin/map.hpp b/libraries/clio/include/clio/to_bin/map.hpp index ef436d382..2c00f934b 100644 --- a/libraries/clio/include/clio/to_bin/map.hpp +++ b/libraries/clio/include/clio/to_bin/map.hpp @@ -2,11 +2,12 @@ #include #include -namespace clio { +namespace clio +{ + template + result to_bin(const std::map& obj, S& stream) + { + return to_bin_range(obj, stream); + } - template - result to_bin(const std::map& obj, S& stream) { - return to_bin_range(obj, stream); - } - -} // clio +} // namespace clio diff --git a/libraries/clio/include/clio/to_bin/set.hpp b/libraries/clio/include/clio/to_bin/set.hpp index 4afbe76ee..b1d04f239 100644 --- a/libraries/clio/include/clio/to_bin/set.hpp +++ b/libraries/clio/include/clio/to_bin/set.hpp @@ -2,11 +2,12 @@ #include #include -namespace clio { +namespace clio +{ + template + result to_bin(const std::set& obj, S& stream) + { + return to_bin_range(obj, stream); + } - template - result to_bin(const std::set& obj, S& stream) { - return to_bin_range(obj, stream); - } - -} // clio +} // namespace clio diff --git a/libraries/clio/include/clio/to_bin/varint.hpp b/libraries/clio/include/clio/to_bin/varint.hpp index 733e14f16..beefbd2bb 100644 --- a/libraries/clio/include/clio/to_bin/varint.hpp +++ b/libraries/clio/include/clio/to_bin/varint.hpp @@ -2,16 +2,18 @@ #include #include -namespace clio { +namespace clio +{ + template + void to_bin(const varuint32& obj, S& stream) + { + varuint32_to_bin(obj.value, stream); + } -template -void to_bin(const varuint32& obj, S& stream) { - varuint32_to_bin(obj.value, stream); -} + template + void to_bin(const varint32& obj, S& stream) + { + varuint32_to_bin((uint32_t(obj.value) << 1) ^ uint32_t(obj.value >> 31), stream); + } -template -void to_bin(const varint32& obj, S& stream) { - varuint32_to_bin((uint32_t(obj.value) << 1) ^ uint32_t(obj.value >> 31), stream); -} - -} /// namespace clio +} // namespace clio diff --git a/libraries/clio/include/clio/to_json.hpp b/libraries/clio/include/clio/to_json.hpp index 0d172d174..bbf5e09a2 100644 --- a/libraries/clio/include/clio/to_json.hpp +++ b/libraries/clio/include/clio/to_json.hpp @@ -1,126 +1,152 @@ #pragma once -#include #include -#include #include +#include +#include #include #include #include #include -namespace clio { - -inline constexpr char hex_digits[] = "0123456789ABCDEF"; - -// Adaptors for rapidjson -struct stream_adaptor { - stream_adaptor(const char* src, int sz) { - int chars = std::min(sz, 4); - memcpy(buf, src, chars); - memset(buf + chars, 0, 4 - chars); - } - void Put(char ch) {} - char Take() { return buf[idx++]; } - char buf[4]; - int idx = 0; -}; +namespace clio +{ + inline constexpr char hex_digits[] = "0123456789ABCDEF"; + // Adaptors for rapidjson + struct stream_adaptor + { + stream_adaptor(const char* src, int sz) + { + int chars = std::min(sz, 4); + memcpy(buf, src, chars); + memset(buf + chars, 0, 4 - chars); + } + void Put(char ch) {} + char Take() { return buf[idx++]; } + char buf[4]; + int idx = 0; + }; -// Replaces any invalid utf-8 bytes with ? -template -void to_json(std::string_view sv, S& stream) { - stream.write('"'); - auto begin = sv.begin(); - auto end = sv.end(); - while (begin != end) { - auto pos = begin; - while (pos != end && *pos != '"' && *pos != '\\' && (unsigned char)(*pos) >= 32 && *pos != 127) ++pos; - while (begin != pos) { - stream_adaptor s2(begin, static_cast(pos - begin)); - if (rapidjson::UTF8<>::Validate(s2, s2)) { - stream.write(begin, s2.idx); - begin += s2.idx; - } else { - ++begin; - stream.write('?'); + // Replaces any invalid utf-8 bytes with ? + template + void to_json(std::string_view sv, S& stream) + { + stream.write('"'); + auto begin = sv.begin(); + auto end = sv.end(); + while (begin != end) + { + auto pos = begin; + while (pos != end && *pos != '"' && *pos != '\\' && (unsigned char)(*pos) >= 32 && + *pos != 127) + ++pos; + while (begin != pos) + { + stream_adaptor s2(begin, static_cast(pos - begin)); + if (rapidjson::UTF8<>::Validate(s2, s2)) + { + stream.write(begin, s2.idx); + begin += s2.idx; + } + else + { + ++begin; + stream.write('?'); + } } - } - if (begin != end) { - if (*begin == '"') { - stream.write("\\\"", 2); - } else if (*begin == '\\') { - stream.write("\\\\", 2); - } else { - stream.write("\\u00", 4); - stream.write(hex_digits[(unsigned char)(*begin) >> 4]); - stream.write(hex_digits[(unsigned char)(*begin) & 15]); + if (begin != end) + { + if (*begin == '"') + { + stream.write("\\\"", 2); + } + else if (*begin == '\\') + { + stream.write("\\\\", 2); + } + else + { + stream.write("\\u00", 4); + stream.write(hex_digits[(unsigned char)(*begin) >> 4]); + stream.write(hex_digits[(unsigned char)(*begin) & 15]); + } + ++begin; } - ++begin; } + stream.write('"'); } - stream.write('"'); -} -template -void to_json(const std::string& s, S& stream) { - return to_json(std::string_view{ s }, stream); -} + template + void to_json(const std::string& s, S& stream) + { + return to_json(std::string_view{s}, stream); + } -template -void to_json(const char* s, S& stream) { - return to_json(std::string_view{ s }, stream); -} + template + void to_json(const char* s, S& stream) + { + return to_json(std::string_view{s}, stream); + } -template -void to_json(bool value, S& stream) { - if (value) - return stream.write("true", 4); - else - return stream.write("false", 5); -} + template + void to_json(bool value, S& stream) + { + if (value) + return stream.write("true", 4); + else + return stream.write("false", 5); + } -template -void int_to_json(T value, S& stream) { - auto uvalue = std::make_unsigned_t(value); - small_buffer::digits10 + 4> b; - bool neg = value < 0; - if (neg) - uvalue = -uvalue; - if (sizeof(T) > 4) - *b.pos++ = '"'; - do { - *b.pos++ = '0' + (uvalue % 10); - uvalue /= 10; - } while (uvalue); - if (neg) - *b.pos++ = '-'; - if (sizeof(T) > 4) - *b.pos++ = '"'; - b.reverse(); - return stream.write(b.data, b.pos - b.data); -} + template + void int_to_json(T value, S& stream) + { + auto uvalue = std::make_unsigned_t(value); + small_buffer::digits10 + 4> b; + bool neg = value < 0; + if (neg) + uvalue = -uvalue; + if (sizeof(T) > 4) + *b.pos++ = '"'; + do + { + *b.pos++ = '0' + (uvalue % 10); + uvalue /= 10; + } while (uvalue); + if (neg) + *b.pos++ = '-'; + if (sizeof(T) > 4) + *b.pos++ = '"'; + b.reverse(); + return stream.write(b.data, b.pos - b.data); + } -template -void fp_to_json(double value, S& stream) { - // fpconv is not quite consistent with javascript for nans and infinities - if (value == std::numeric_limits::infinity()) { - return stream.write("\"Infinity\"", 10); - } else if (value == -std::numeric_limits::infinity()) { - return stream.write("\"-Infinity\"", 11); - } else if (std::isnan(value)) { - return stream.write("\"NaN\"", 5); + template + void fp_to_json(double value, S& stream) + { + // fpconv is not quite consistent with javascript for nans and infinities + if (value == std::numeric_limits::infinity()) + { + return stream.write("\"Infinity\"", 10); + } + else if (value == -std::numeric_limits::infinity()) + { + return stream.write("\"-Infinity\"", 11); + } + else if (std::isnan(value)) + { + return stream.write("\"NaN\"", 5); + } + small_buffer<24> b; // fpconv_dtoa generates at most 24 characters + int n = fpconv_dtoa(value, b.pos); + if (n <= 0) + throw_error(stream_error::float_error); + b.pos += n; + stream.write(b.data, b.pos - b.data); } - small_buffer<24> b; // fpconv_dtoa generates at most 24 characters - int n = fpconv_dtoa(value, b.pos); - if (n <= 0) - throw_error(stream_error::float_error); - b.pos += n; - stream.write(b.data, b.pos - b.data); -} -// clang-format off + // clang-format off template void to_json(uint8_t value, S& stream) { return int_to_json(value, stream); } template void to_json(uint16_t value, S& stream) { return int_to_json(value, stream); } template void to_json(uint32_t value, S& stream) { return int_to_json(value, stream); } @@ -133,151 +159,180 @@ template void to_json(int64_t value, S& stream) { return int_to_j template void to_json(__int128 value, S& stream) { return int_to_json(value, stream); } template void to_json(double value, S& stream) { return fp_to_json(value, stream); } template void to_json(float value, S& stream) { return fp_to_json(value, stream); } -// clang-format on + // clang-format on -template -void to_json(const std::vector& obj, S& stream) { - stream.write('['); - bool first = true; - for (auto& v : obj) { - if (first) { - increase_indent(stream); - } else { - stream.write(','); + template + void to_json(const std::vector& obj, S& stream) + { + stream.write('['); + bool first = true; + for (auto& v : obj) + { + if (first) + { + increase_indent(stream); + } + else + { + stream.write(','); + } + write_newline(stream); + first = false; + to_json(v, stream); } - write_newline(stream); - first = false; - to_json(v, stream); + if (!first) + { + decrease_indent(stream); + write_newline(stream); + } + stream.write(']'); } - if (!first) { - decrease_indent(stream); - write_newline(stream); + + template + void to_json(const std::optional& obj, S& stream) + { + if (obj) + { + return to_json(*obj, stream); + } + else + { + return stream.write("null", 4); + } } - stream.write(']'); -} -template -void to_json(const std::optional& obj, S& stream) { - if (obj) { - return to_json(*obj, stream); - } else { - return stream.write("null", 4); + template + void to_json(const std::variant& obj, S& stream) + { + stream.write('['); + std::visit( + [&](const auto& t) { to_json(get_type_name>(), stream); }, obj); + stream.write(','); + std::visit([&](auto& x) { return to_json(x, stream); }, obj); + stream.write(']'); } -} + template + void to_json(const std::tuple& obj, S& stream) + { + stream.write('['); + if (sizeof...(T) == 0) + return stream.write(']'); + increase_indent(stream); + tuple_for_each(obj, [&](int idx, auto& item) { + write_newline(stream); + if (idx) + { + stream.write(','); + } + to_json(item, stream); + }); + decrease_indent(stream); + write_newline(stream); + return stream.write(']'); + } -template -void to_json(const std::variant& obj, S& stream) { - stream.write('['); - std::visit( - [&](const auto& t) { to_json( get_type_name>(), stream); }, obj); - stream.write(','); - std::visit([&](auto& x) { return to_json(x, stream); }, obj); - stream.write(']'); -} + template + void to_json(const T& t, S& stream) + { + if constexpr (not reflect::is_defined) + { + stream.write('"'); + std::string str(t); + stream.write(str.data(), str.size()); + stream.write('"'); + } + else + { + bool first = true; + stream.write('{'); + reflect::for_each([&](const clio::meta& ref, auto&& member) { + if constexpr (not std::is_member_function_pointer_v>) + { + auto addfield = [&]() { + if (first) + { + increase_indent(stream); + first = false; + } + else + { + stream.write(','); + } + write_newline(stream); + to_json(ref.name, stream); + write_colon(stream); + to_json(t.*member, stream); + }; -template -void to_json(const std::tuple& obj, S& stream) { - stream.write('['); - if( sizeof...(T) == 0 ) - return stream.write(']'); - increase_indent(stream); + using member_type = std::decay_t; + if constexpr (not is_std_optional::value) + { + addfield(); + } + else + { + if (!!(t.*member)) + addfield(); + } + } + }); + if (!first) + { + decrease_indent(stream); + write_newline(stream); + } + stream.write('}'); + } + } - tuple_for_each( obj, [&]( int idx, auto& item ) { - write_newline(stream); - if( idx ) { stream.write(',');} - to_json( item, stream ); - }); - decrease_indent(stream); - write_newline(stream); - return stream.write(']'); -} + template + void to_json_hex(const char* data, size_t size, S& stream) + { + stream.write('"'); + for (size_t i = 0; i < size; ++i) + { + unsigned char byte = data[i]; + stream.write(hex_digits[byte >> 4]); + stream.write(hex_digits[byte & 15]); + } + stream.write('"'); + } + template + std::string convert_to_json(const T& t) + { + size_stream ss; + to_json(t, ss); -template -void to_json(const T& t, S& stream) { - if constexpr ( not reflect::is_defined ) { - stream.write('"'); - std::string str(t); - stream.write( str.data(), str.size() ); - stream.write('"'); - } else { - bool first = true; - stream.write('{'); - reflect::for_each([&]( const clio::meta& ref, auto&& member) { - if constexpr ( not std::is_member_function_pointer_v> ) { - auto addfield = [&]() { - if (first) { - increase_indent(stream); - first = false; - } else { - stream.write(','); - } - write_newline(stream); - to_json(ref.name, stream); - write_colon(stream); - to_json( t.*member, stream); - }; + std::string result(ss.size, 0); + fixed_buf_stream fbs(result.data(), result.size()); + to_json(t, fbs); - using member_type = std::decay_t; - if constexpr ( not is_std_optional::value ) { - addfield(); - } else { - if( !!(t.*member) ) - addfield(); - } - } - }); - if (!first) { - decrease_indent(stream); - write_newline(stream); - } - stream.write('}'); + if (fbs.pos == fbs.end) + return result; + else + throw_error(stream_error::underrun); } -} -template -void to_json_hex(const char* data, size_t size, S& stream) { - stream.write('"'); - for (size_t i = 0; i < size; ++i) { - unsigned char byte = data[i]; - stream.write(hex_digits[byte >> 4]); - stream.write(hex_digits[byte & 15]); + template + std::string to_json(const T& t) + { + return convert_to_json(t); } - stream.write('"'); -} -template -std::string convert_to_json(const T& t) { - size_stream ss; - to_json(t, ss); - - std::string result(ss.size, 0); - fixed_buf_stream fbs(result.data(), result.size()); - to_json(t, fbs); - - if (fbs.pos == fbs.end) + template + std::string format_json(const T& t) + { + pretty_stream ss; + to_json(t, ss); + std::string result(ss.size, 0); + pretty_stream fbs(result.data(), result.size()); + to_json(t, fbs); + if (fbs.pos != fbs.end) + throw_error(stream_error::underrun); return result; - else - throw_error(stream_error::underrun); -} - -template -std::string to_json(const T& t) { - return convert_to_json(t); -} - -template -std::string format_json(const T& t) { - pretty_stream ss; - to_json(t, ss); - std::string result(ss.size, 0); - pretty_stream fbs(result.data(), result.size()); - to_json(t, fbs); - if (fbs.pos != fbs.end) - throw_error(stream_error::underrun); - return result; -} + } -} // namespace eosio +} // namespace clio diff --git a/libraries/clio/include/clio/to_json/map.hpp b/libraries/clio/include/clio/to_json/map.hpp index ebbb48499..3e965de9c 100644 --- a/libraries/clio/include/clio/to_json/map.hpp +++ b/libraries/clio/include/clio/to_json/map.hpp @@ -1,28 +1,33 @@ #pragma once -#include #include +#include -namespace clio { -template -void to_json( const std::map& m, S& stream) { - stream.write('{'); - if( m.size() == 0 ) { - return stream.write('}'); - } - increase_indent(stream); - bool not_first = false; - for( const auto& p : m ) { - if( not_first ) { - stream.write(','); - } - write_newline(stream); - to_json( p.first, stream ); - write_colon(stream); - to_json( p.second, stream ); - not_first = true; +namespace clio +{ + template + void to_json(const std::map& m, S& stream) + { + stream.write('{'); + if (m.size() == 0) + { + return stream.write('}'); + } + increase_indent(stream); + bool not_first = false; + for (const auto& p : m) + { + if (not_first) + { + stream.write(','); + } + write_newline(stream); + to_json(p.first, stream); + write_colon(stream); + to_json(p.second, stream); + not_first = true; + } + decrease_indent(stream); + write_newline(stream); + return stream.write('}'); } - decrease_indent(stream); - write_newline(stream); - return stream.write('}'); -} -} /// namespace clio +} // namespace clio diff --git a/libraries/clio/include/clio/to_json/varint.hpp b/libraries/clio/include/clio/to_json/varint.hpp index f679897e5..5066bf9f1 100644 --- a/libraries/clio/include/clio/to_json/varint.hpp +++ b/libraries/clio/include/clio/to_json/varint.hpp @@ -2,16 +2,18 @@ #include #include -namespace clio { +namespace clio +{ + template + void to_json(const varuint32& obj, S& stream) + { + to_json(obj.value, stream); + } -template -void to_json(const varuint32& obj, S& stream) { - to_json(obj.value, stream); -} + template + void to_json(const varint32& obj, S& stream) + { + to_json(obj.value, stream); + } -template -void to_json(const varint32& obj, S& stream) { - to_json(obj.value, stream); -} - -} // namespace clio +} // namespace clio diff --git a/libraries/clio/include/clio/to_key.hpp b/libraries/clio/include/clio/to_key.hpp index e91c09b8a..7a01f92b4 100644 --- a/libraries/clio/include/clio/to_key.hpp +++ b/libraries/clio/include/clio/to_key.hpp @@ -4,114 +4,139 @@ //#include //#include //#include +#include #include #include +#include #include #include -#include -#include #include -namespace clio { - -template -void to_key(const std::tuple& obj, S& stream); +namespace clio +{ + template + void to_key(const std::tuple& obj, S& stream); -// to_key defines a conversion from a type to a sequence of bytes whose lexicograpical -// ordering is the same as the ordering of the original type. -// -// For any two objects of type T, a and b: -// -// - key(a) < key(b) iff a < b -// - key(a) is not a prefix of key(b) -// -// Overloads of to_key for user-defined types can be found by Koenig lookup. -// -// Abieos provides specializations of to_key for the following types -// - std::string and std::string_view -// - std::vector, std::list, std::deque -// - std::tuple -// - std::array -// - std::optional -// - std::variant -// - Arithmetic types -// - Scoped enumeration types -// - Reflected structs -// - All smart-contract related types defined by abieos -template -void to_key(const T& obj, S& stream); + // to_key defines a conversion from a type to a sequence of bytes whose lexicograpical + // ordering is the same as the ordering of the original type. + // + // For any two objects of type T, a and b: + // + // - key(a) < key(b) iff a < b + // - key(a) is not a prefix of key(b) + // + // Overloads of to_key for user-defined types can be found by Koenig lookup. + // + // Abieos provides specializations of to_key for the following types + // - std::string and std::string_view + // - std::vector, std::list, std::deque + // - std::tuple + // - std::array + // - std::optional + // - std::variant + // - Arithmetic types + // - Scoped enumeration types + // - Reflected structs + // - All smart-contract related types defined by abieos + template + void to_key(const T& obj, S& stream); -template -void to_key_tuple(const T& obj, S& stream) { - if constexpr (i < std::tuple_size_v) { - to_key(std::get(obj), stream); - to_key_tuple(obj, stream); + template + void to_key_tuple(const T& obj, S& stream) + { + if constexpr (i < std::tuple_size_v) + { + to_key(std::get(obj), stream); + to_key_tuple(obj, stream); + } } -} -template -void to_key(const std::tuple& obj, S& stream) { - return to_key_tuple<0>(obj, stream); -} - -template -void to_key(const std::array& obj, S& stream) { - for (const T& elem : obj) { to_key(elem, stream); } -} + template + void to_key(const std::tuple& obj, S& stream) + { + return to_key_tuple<0>(obj, stream); + } -template -void to_key_optional(const bool* obj, S& stream) { - if (obj == nullptr) - return stream.write('\0'); - else if (!*obj) - return stream.write('\1'); - else - return stream.write('\2'); -} + template + void to_key(const std::array& obj, S& stream) + { + for (const T& elem : obj) + { + to_key(elem, stream); + } + } -template -void to_key_optional(const T* obj, S& stream) { - if constexpr (has_bitwise_serialization() && sizeof(T) == 1) { + template + void to_key_optional(const bool* obj, S& stream) + { if (obj == nullptr) - return stream.write("\0", 2); - else { - char buf[1]; - fixed_buf_stream tmp_stream(buf, 1); - to_key(*obj, tmp_stream); - stream.write(buf[0]); - if (buf[0] == '\0') - stream.write('\1'); - } - } else { - if (obj) { - stream.write('\1'); - return to_key(*obj, stream); - } else { return stream.write('\0'); + else if (!*obj) + return stream.write('\1'); + else + return stream.write('\2'); + } + + template + void to_key_optional(const T* obj, S& stream) + { + if constexpr (has_bitwise_serialization() && sizeof(T) == 1) + { + if (obj == nullptr) + return stream.write("\0", 2); + else + { + char buf[1]; + fixed_buf_stream tmp_stream(buf, 1); + to_key(*obj, tmp_stream); + stream.write(buf[0]); + if (buf[0] == '\0') + stream.write('\1'); + } + } + else + { + if (obj) + { + stream.write('\1'); + return to_key(*obj, stream); + } + else + { + return stream.write('\0'); + } } } -} -template -void to_key(const std::pair& obj, S& stream) { - to_key(obj.first, stream); - return to_key(obj.second, stream); -} + template + void to_key(const std::pair& obj, S& stream) + { + to_key(obj.first, stream); + return to_key(obj.second, stream); + } -template -void to_key_range(const T& obj, S& stream) { - for (const auto& elem : obj) { to_key_optional(&elem, stream); } - return to_key_optional((decltype(&*std::begin(obj))) nullptr, stream); -} + template + void to_key_range(const T& obj, S& stream) + { + for (const auto& elem : obj) + { + to_key_optional(&elem, stream); + } + return to_key_optional((decltype(&*std::begin(obj))) nullptr, stream); + } -template -void to_key(const std::vector& obj, S& stream) { - for (const T& elem : obj) { to_key_optional(&elem, stream); } - return to_key_optional((const T*)nullptr, stream); -} + template + void to_key(const std::vector& obj, S& stream) + { + for (const T& elem : obj) + { + to_key_optional(&elem, stream); + } + return to_key_optional((const T*)nullptr, stream); + } -/* + /* template void to_key(const std::list& obj, S& stream) { return to_key_range(obj, stream); @@ -133,171 +158,224 @@ void to_key(const std::map& obj, S& stream) { } */ -template -void to_key(const std::optional& obj, S& stream) { - return to_key_optional(obj ? &*obj : nullptr, stream); -} - -// The first byte holds: -// 0-4 1's (number of additional bytes) 0 (terminator) bits -// -// The number is represented as big-endian using the low order -// bits of the first byte and all of the remaining bytes. -// -// Notes: -// - values must be encoded using the minimum number of bytes, -// as non-canonical representations will break the sort order. -template -void to_key_varuint32(std::uint32_t obj, S& stream) { - int num_bytes; - if (obj < 0x80u) { - num_bytes = 1; - } else if (obj < 0x4000u) { - num_bytes = 2; - } else if (obj < 0x200000u) { - num_bytes = 3; - } else if (obj < 0x10000000u) { - num_bytes = 4; - } else { - num_bytes = 5; + template + void to_key(const std::optional& obj, S& stream) + { + return to_key_optional(obj ? &*obj : nullptr, stream); } - stream.write( - static_cast(~(0xFFu >> (num_bytes - 1)) | (num_bytes == 5 ? 0 : (obj >> ((num_bytes - 1) * 8))))); - for (int i = num_bytes - 2; i >= 0; --i) { stream.write(static_cast((obj >> i * 8) & 0xFFu)); } -} + // The first byte holds: + // 0-4 1's (number of additional bytes) 0 (terminator) bits + // + // The number is represented as big-endian using the low order + // bits of the first byte and all of the remaining bytes. + // + // Notes: + // - values must be encoded using the minimum number of bytes, + // as non-canonical representations will break the sort order. + template + void to_key_varuint32(std::uint32_t obj, S& stream) + { + int num_bytes; + if (obj < 0x80u) + { + num_bytes = 1; + } + else if (obj < 0x4000u) + { + num_bytes = 2; + } + else if (obj < 0x200000u) + { + num_bytes = 3; + } + else if (obj < 0x10000000u) + { + num_bytes = 4; + } + else + { + num_bytes = 5; + } -// for non-negative values -// The first byte holds: -// 1 (signbit) 0-4 1's (number of additional bytes) 0 (terminator) bits -// The value is represented as big endian -// for negative values -// The first byte holds: -// 0 (signbit) 0-4 0's (number of additional bytes) 1 (terminator) bits -// The value is adjusted to be positive based on the range that can -// be represented with this number of bytes and then encoded as big endian. -// -// Notes: -// - negative values must sort before positive values -// - For negative value, numbers that need more bytes are smaller, hence -// the encoding of the width must be opposite the encoding used for -// non-negative values. -// - A 5-byte varint can represent values in $[-2^34, 2^34)$. In this case, -// the argument will be sign-extended. -template -void to_key_varint32(std::int32_t obj, S& stream) { - static_assert(std::is_same_v, "to_key for varint32 has been temporarily disabled"); - int num_bytes; - bool sign = (obj < 0); - if (obj < 0x40 && obj >= -0x40) { - num_bytes = 1; - } else if (obj < 0x2000 && obj >= -0x2000) { - num_bytes = 2; - } else if (obj < 0x100000 && obj >= -0x100000) { - num_bytes = 3; - } else if (obj < 0x08000000 && obj >= -0x08000000) { - num_bytes = 4; - } else { - num_bytes = 5; + stream.write(static_cast(~(0xFFu >> (num_bytes - 1)) | + (num_bytes == 5 ? 0 : (obj >> ((num_bytes - 1) * 8))))); + for (int i = num_bytes - 2; i >= 0; --i) + { + stream.write(static_cast((obj >> i * 8) & 0xFFu)); + } } - unsigned char width_field; - if (sign) { - width_field = 0x80u >> num_bytes; - } else { - width_field = 0x80u | ~(0xFFu >> num_bytes); + // for non-negative values + // The first byte holds: + // 1 (signbit) 0-4 1's (number of additional bytes) 0 (terminator) bits + // The value is represented as big endian + // for negative values + // The first byte holds: + // 0 (signbit) 0-4 0's (number of additional bytes) 1 (terminator) bits + // The value is adjusted to be positive based on the range that can + // be represented with this number of bytes and then encoded as big endian. + // + // Notes: + // - negative values must sort before positive values + // - For negative value, numbers that need more bytes are smaller, hence + // the encoding of the width must be opposite the encoding used for + // non-negative values. + // - A 5-byte varint can represent values in $[-2^34, 2^34)$. In this case, + // the argument will be sign-extended. + template + void to_key_varint32(std::int32_t obj, S& stream) + { + static_assert(std::is_same_v, "to_key for varint32 has been temporarily disabled"); + int num_bytes; + bool sign = (obj < 0); + if (obj < 0x40 && obj >= -0x40) + { + num_bytes = 1; + } + else if (obj < 0x2000 && obj >= -0x2000) + { + num_bytes = 2; + } + else if (obj < 0x100000 && obj >= -0x100000) + { + num_bytes = 3; + } + else if (obj < 0x08000000 && obj >= -0x08000000) + { + num_bytes = 4; + } + else + { + num_bytes = 5; + } + + unsigned char width_field; + if (sign) + { + width_field = 0x80u >> num_bytes; + } + else + { + width_field = 0x80u | ~(0xFFu >> num_bytes); + } + auto uobj = static_cast(obj); + unsigned char value_mask = (0xFFu >> (num_bytes + 1)); + unsigned char high_byte = + (num_bytes == 5 ? (sign ? 0xFF : 0) : (uobj >> ((num_bytes - 1) * 8))); + stream.write(width_field | (high_byte & value_mask)); + for (int i = num_bytes - 2; i >= 0; --i) + { + stream.write(static_cast((uobj >> i * 8) & 0xFFu)); + } } - auto uobj = static_cast(obj); - unsigned char value_mask = (0xFFu >> (num_bytes + 1)); - unsigned char high_byte = (num_bytes == 5 ? (sign ? 0xFF : 0) : (uobj >> ((num_bytes - 1) * 8))); - stream.write(width_field | (high_byte & value_mask)); - for (int i = num_bytes - 2; i >= 0; --i) { stream.write(static_cast((uobj >> i * 8) & 0xFFu)); } -} -template -void to_key(const std::variant& obj, S& stream) { - to_key_varuint32(static_cast(obj.index()), stream); - std::visit([&](const auto& item) { return to_key(item, stream); }, obj); -} + template + void to_key(const std::variant& obj, S& stream) + { + to_key_varuint32(static_cast(obj.index()), stream); + std::visit([&](const auto& item) { return to_key(item, stream); }, obj); + } -template -void to_key(std::string_view obj, S& stream) { - for (char ch : obj) { - stream.write(ch); - if (ch == '\0') { - stream.write('\1'); + template + void to_key(std::string_view obj, S& stream) + { + for (char ch : obj) + { + stream.write(ch); + if (ch == '\0') + { + stream.write('\1'); + } } + return stream.write("\0", 2); } - return stream.write("\0", 2); -} -template -void to_key(const std::string& obj, S& stream) { - return to_key(std::string_view(obj), stream); -} + template + void to_key(const std::string& obj, S& stream) + { + return to_key(std::string_view(obj), stream); + } -template -void to_key(bool obj, S& stream) { - return stream.write(static_cast(obj ? 1 : 0)); -} + template + void to_key(bool obj, S& stream) + { + return stream.write(static_cast(obj ? 1 : 0)); + } -template -UInt float_to_key(T value) { - static_assert(sizeof(T) == sizeof(UInt), "Expected unsigned int of the same size"); - UInt result; - std::memcpy(&result, &value, sizeof(T)); - UInt signbit = (static_cast(1) << (std::numeric_limits::digits - 1)); - UInt mask = 0; - if (result == signbit) - result = 0; - if (result & signbit) - mask = ~mask; - return result ^ (mask | signbit); -} + template + UInt float_to_key(T value) + { + static_assert(sizeof(T) == sizeof(UInt), "Expected unsigned int of the same size"); + UInt result; + std::memcpy(&result, &value, sizeof(T)); + UInt signbit = (static_cast(1) << (std::numeric_limits::digits - 1)); + UInt mask = 0; + if (result == signbit) + result = 0; + if (result & signbit) + mask = ~mask; + return result ^ (mask | signbit); + } -template -void to_key(const T& obj, S& stream) { - if constexpr (std::is_floating_point_v) { - if constexpr (sizeof(T) == 4) { - return to_key(float_to_key(obj), stream); - } else { - static_assert(sizeof(T) == 8, "Unknown floating point type"); - return to_key(float_to_key(obj), stream); - } - } else if constexpr (std::is_integral_v) { - auto v = static_cast>(obj); - v -= static_cast>(std::numeric_limits::min()); - std::reverse(reinterpret_cast(&v), reinterpret_cast(&v + 1)); - return stream.write_raw(v); - } else if constexpr (std::is_enum_v) { - static_assert(!std::is_convertible_v>, "Serializing unscoped enum"); - return to_key(static_cast>(obj), stream); - } else { - clio::reflect::for_each( [&](const clio::meta&, auto member) { - if constexpr( std::is_member_pointer_v ) { - to_key( obj.*member, stream); + template + void to_key(const T& obj, S& stream) + { + if constexpr (std::is_floating_point_v) + { + if constexpr (sizeof(T) == 4) + { + return to_key(float_to_key(obj), stream); } - }); + else + { + static_assert(sizeof(T) == 8, "Unknown floating point type"); + return to_key(float_to_key(obj), stream); + } + } + else if constexpr (std::is_integral_v) + { + auto v = static_cast>(obj); + v -= static_cast>(std::numeric_limits::min()); + std::reverse(reinterpret_cast(&v), reinterpret_cast(&v + 1)); + return stream.write_raw(v); + } + else if constexpr (std::is_enum_v) + { + static_assert(!std::is_convertible_v>, + "Serializing unscoped enum"); + return to_key(static_cast>(obj), stream); + } + else + { + clio::reflect::for_each([&](const clio::meta&, auto member) { + if constexpr (std::is_member_pointer_v) + { + to_key(obj.*member, stream); + } + }); + } } -} -template -void convert_to_key(const T& t, std::vector& bin) { - size_stream ss; - to_key(t, ss); - auto orig_size = bin.size(); - bin.resize(orig_size + ss.size); - fixed_buf_stream fbs(bin.data() + orig_size, ss.size); - to_key(t, fbs); - if (fbs.pos != fbs.end) - throw_error(stream_error::underrun); -} + template + void convert_to_key(const T& t, std::vector& bin) + { + size_stream ss; + to_key(t, ss); + auto orig_size = bin.size(); + bin.resize(orig_size + ss.size); + fixed_buf_stream fbs(bin.data() + orig_size, ss.size); + to_key(t, fbs); + if (fbs.pos != fbs.end) + throw_error(stream_error::underrun); + } -template -std::vector convert_to_key(const T& t) { - std::vector result; - convert_to_key(t, result); - return result; -} + template + std::vector convert_to_key(const T& t) + { + std::vector result; + convert_to_key(t, result); + return result; + } -} // namespace clio +} // namespace clio diff --git a/libraries/clio/include/clio/to_protobuf.hpp b/libraries/clio/include/clio/to_protobuf.hpp index fa71080ee..301f2292e 100644 --- a/libraries/clio/include/clio/to_protobuf.hpp +++ b/libraries/clio/include/clio/to_protobuf.hpp @@ -1,152 +1,177 @@ #pragma once +#include #include #include -#include -namespace clio { - -template -void to_protobuf_object(const std::tuple& obj, S& stream); - -template -void to_protobuf_object(const std::variant& obj, S& stream); - -template -void to_protobuf_object(const std::vector& obj, S& stream); - -template -void to_protobuf_object(const T& obj, S& stream); - -template -struct wire_type { static constexpr const auto value = 2; }; - -#define WIRE_TYPE(X,T) \ -template<> struct wire_type { static constexpr const auto value = T; }; - - -template struct wire_type< std::variant> { static constexpr const auto value = 2; }; - -WIRE_TYPE( uint16_t, 0 ) -WIRE_TYPE( uint8_t, 0 ) -WIRE_TYPE( int16_t, 0 ) -WIRE_TYPE( int8_t, 0 ) -WIRE_TYPE( char, 0 ) -WIRE_TYPE( bool, 0 ) -WIRE_TYPE( varuint32, 0 ) -WIRE_TYPE( uint64_t, 1 ) -WIRE_TYPE( int64_t, 1 ) -WIRE_TYPE( double, 1 ) -WIRE_TYPE( uint32_t, 5 ) -WIRE_TYPE( int32_t, 5 ) -WIRE_TYPE( float, 5 ) -WIRE_TYPE( std::string, 2 ) -WIRE_TYPE( bytes, 2 ) - -/** - * Assumes key has been writen, then writes the rest... - */ -template -void to_protobuf_member( const T& obj, S& stream ) { - if constexpr( std::is_same_v< T, std::string> ) { - varuint32_to_bin( obj.size(), stream ); - stream.write( obj.data(), obj.size() ); - } else if constexpr( std::is_same_v< T, bytes> ) { - varuint32_to_bin( obj.data.size(), stream ); - stream.write( obj.data.data(), obj.data.size()); - } else if constexpr ( 5 == wire_type::value or 1 == wire_type::value ) { - to_bin( obj, stream ); - } else if constexpr ( 0 == wire_type::value ) { - varuint32_to_bin( obj, stream ); - } else if constexpr( reflect::is_struct || - is_std_vector::value || - is_std_tuple::value || - is_std_variant::value ) - { - size_stream ss; - to_protobuf_object(obj, ss); - varuint32_to_bin( ss.size, stream ); - to_protobuf_object(obj, stream); - } else { - T::to_protobuf_is_not_defined; /// used to generate useful compile error - } -} - -template -void write_protobuf_field( int field, const Member& member, Stream& stream ) { - if constexpr( is_std_vector< std::decay_t >::value ) { - if( member.size() == 0 ) return; - } - uint32_t key = (field << 3) | wire_type::value; - varuint32_to_bin( key, stream ); - to_protobuf_member( member, stream ); -} - -template -void to_protobuf_object(const std::variant& obj, S& stream) { - std::visit( [&](auto m){ - write_protobuf_field( obj.index()+1, m, stream ); - }, obj ); -} - - -/** - * A vector is protobuf object that is either packed in a single field or - * listed as N fields of the same type. - */ -template -void to_protobuf_object(const std::vector& vec, S& stream) { - uint32_t key = (1 << 3) | wire_type>::value; - if constexpr ( std::is_arithmetic_v< T > ) { /// [packed=true] - varuint32_to_bin( key, stream ); - auto size = uint32_t(vec.size() * sizeof( T )); - varuint32_to_bin( size, stream ); - stream.write( vec.data(), vec.size()*sizeof(T) ); - } - else { - for( const auto& item : vec ) { - varuint32_to_bin( key, stream ); - to_protobuf_member( item, stream ); - } - } -} - - -template -void to_protobuf_object(const T& obj, S& stream) { - reflect::for_each( [&]( const clio::meta& ref, auto m ) { - if constexpr( not std::is_member_function_pointer_v ) { - write_protobuf_field( ref.number, obj.*m, stream ); - } - }); -} - -template -void to_protobuf_object(const std::tuple& obj, S& stream) { - tuple_for_each( obj, [&]( int idx, const auto& m ) { - write_protobuf_field( idx+1, m, stream ); - }); -} - -template -void to_protobuf(const T& obj, S& stream) { - to_protobuf_object( obj, stream ); -} - -template -std::vector to_protobuf(const T& t) { - size_stream ss; - to_protobuf(t, ss); - - std::vector result(ss.size, 0); - fixed_buf_stream fbs(result.data(), result.size()); - - to_protobuf(t, fbs); - - if (fbs.pos != fbs.end) - throw_error( stream_error::underrun ); - return result; -} - - - -} // clio +namespace clio +{ + template + void to_protobuf_object(const std::tuple& obj, S& stream); + + template + void to_protobuf_object(const std::variant& obj, S& stream); + + template + void to_protobuf_object(const std::vector& obj, S& stream); + + template + void to_protobuf_object(const T& obj, S& stream); + + template + struct wire_type + { + static constexpr const auto value = 2; + }; + +#define WIRE_TYPE(X, T) \ + template <> \ + struct wire_type \ + { \ + static constexpr const auto value = T; \ + }; + + template + struct wire_type> + { + static constexpr const auto value = 2; + }; + + WIRE_TYPE(uint16_t, 0) + WIRE_TYPE(uint8_t, 0) + WIRE_TYPE(int16_t, 0) + WIRE_TYPE(int8_t, 0) + WIRE_TYPE(char, 0) + WIRE_TYPE(bool, 0) + WIRE_TYPE(varuint32, 0) + WIRE_TYPE(uint64_t, 1) + WIRE_TYPE(int64_t, 1) + WIRE_TYPE(double, 1) + WIRE_TYPE(uint32_t, 5) + WIRE_TYPE(int32_t, 5) + WIRE_TYPE(float, 5) + WIRE_TYPE(std::string, 2) + WIRE_TYPE(bytes, 2) + + /** + * Assumes key has been writen, then writes the rest... + */ + template + void to_protobuf_member(const T& obj, S& stream) + { + if constexpr (std::is_same_v) + { + varuint32_to_bin(obj.size(), stream); + stream.write(obj.data(), obj.size()); + } + else if constexpr (std::is_same_v) + { + varuint32_to_bin(obj.data.size(), stream); + stream.write(obj.data.data(), obj.data.size()); + } + else if constexpr (5 == wire_type::value or 1 == wire_type::value) + { + to_bin(obj, stream); + } + else if constexpr (0 == wire_type::value) + { + varuint32_to_bin(obj, stream); + } + else if constexpr (reflect::is_struct || is_std_vector::value || + is_std_tuple::value || is_std_variant::value) + { + size_stream ss; + to_protobuf_object(obj, ss); + varuint32_to_bin(ss.size, stream); + to_protobuf_object(obj, stream); + } + else + { + T::to_protobuf_is_not_defined; /// used to generate useful compile error + } + } + + template + void write_protobuf_field(int field, const Member& member, Stream& stream) + { + if constexpr (is_std_vector>::value) + { + if (member.size() == 0) + return; + } + uint32_t key = (field << 3) | wire_type::value; + varuint32_to_bin(key, stream); + to_protobuf_member(member, stream); + } + + template + void to_protobuf_object(const std::variant& obj, S& stream) + { + std::visit([&](auto m) { write_protobuf_field(obj.index() + 1, m, stream); }, obj); + } + + /** + * A vector is protobuf object that is either packed in a single field or + * listed as N fields of the same type. + */ + template + void to_protobuf_object(const std::vector& vec, S& stream) + { + uint32_t key = (1 << 3) | wire_type>::value; + if constexpr (std::is_arithmetic_v) + { /// [packed=true] + varuint32_to_bin(key, stream); + auto size = uint32_t(vec.size() * sizeof(T)); + varuint32_to_bin(size, stream); + stream.write(vec.data(), vec.size() * sizeof(T)); + } + else + { + for (const auto& item : vec) + { + varuint32_to_bin(key, stream); + to_protobuf_member(item, stream); + } + } + } + + template + void to_protobuf_object(const T& obj, S& stream) + { + reflect::for_each([&](const clio::meta& ref, auto m) { + if constexpr (not std::is_member_function_pointer_v) + { + write_protobuf_field(ref.number, obj.*m, stream); + } + }); + } + + template + void to_protobuf_object(const std::tuple& obj, S& stream) + { + tuple_for_each(obj, + [&](int idx, const auto& m) { write_protobuf_field(idx + 1, m, stream); }); + } + + template + void to_protobuf(const T& obj, S& stream) + { + to_protobuf_object(obj, stream); + } + + template + std::vector to_protobuf(const T& t) + { + size_stream ss; + to_protobuf(t, ss); + + std::vector result(ss.size, 0); + fixed_buf_stream fbs(result.data(), result.size()); + + to_protobuf(t, fbs); + + if (fbs.pos != fbs.end) + throw_error(stream_error::underrun); + return result; + } + +} // namespace clio diff --git a/libraries/clio/include/clio/translator.hpp b/libraries/clio/include/clio/translator.hpp index 98388574b..5ca6d12d4 100644 --- a/libraries/clio/include/clio/translator.hpp +++ b/libraries/clio/include/clio/translator.hpp @@ -1,237 +1,270 @@ #pragma once -#include +#include #include -#include #include -#include -#include #include -#include #include +#include +#include +#include +#include -namespace clio { - - class meta_type_base { - public: - meta_type_base( uint32_t n ):_number(n){} - - virtual ~meta_type_base(){} - virtual const char* name()const = 0; - - virtual std::vector json_to_protobuf( std::string json )const = 0; - virtual std::vector json_to_bin( std::string json )const = 0; - - virtual std::string protobuf_to_json( const std::vector& b )const = 0; - virtual std::vector protobuf_to_bin( const std::vector& b )const = 0; - - virtual std::vector bin_to_protobuf( const std::vector& b )const = 0; - virtual std::string bin_to_json( const std::vector& b )const = 0; - - uint32_t number()const { return _number; } - private: - uint32_t _number; - }; - - template - class meta_type : public meta_type_base - { - public: - meta_type( uint32_t n ):meta_type_base(n){} - virtual ~meta_type() {}; - - virtual const char* name()const override { - return get_type_name();//reflect::name(); - } - - virtual std::vector json_to_protobuf( std::string json )const override { - auto t = from_json( json ); - return to_protobuf( t ); - } - - virtual std::vector json_to_bin( std::string json )const override { - auto t = from_json( json ); - return to_bin( t ); - } - - virtual std::string protobuf_to_json( const std::vector& b )const override { - auto t = from_protobuf( b ); - return to_json(t); - } - - virtual std::vector protobuf_to_bin( const std::vector& b )const override { - auto t = from_protobuf( b ); - return to_bin(t); - } - - virtual std::vector bin_to_protobuf( const std::vector& b )const override { - auto t = from_bin( b ); - return to_protobuf( t ); - } - - virtual std::string bin_to_json( const std::vector& b )const override { - auto t = from_bin( b ); - return to_json( t ); - } - }; - - - template - class translator { - public: - translator() { - _schema.generate( [&]( auto* p ) { - _types.push_back( new meta_type>( _types.size() ) ); - }); - } - string get_json_schema()const { - return format_json( _schema ); - } - string get_protobuf_schema() { - return to_protobuf_schema( _schema ); - } - // string get_gql_schema(); - - uint32_t get_type_num( const string& type_name )const { - for( const auto& t : _types ) - if( t->name() == type_name ) return t->number(); - return -1; - } - - string get_type_name( uint32_t type_num )const { - if( type_num < _types.size() ) - return _types[type_num]->name(); - return string(); - } - - std::vector json_to_bin( uint32_t type_num, std::string json )const { - if( type_num < _types.size() ) - return _types[type_num]->json_to_bin( json ); - return std::vector(); - } - - std::vector json_to_protobuf( int type_num, std::string json )const { - if( type_num < _types.size() ) - return _types[type_num]->json_to_protobuf( json ); - return std::vector(); - } - - string bin_to_json( int type_num, const std::vector& bin )const { - if( type_num < _types.size() ) - return _types[type_num]->bin_to_json( bin ); - return string(); - } - std::vector bin_to_protobuf( int type_num, const std::vector& bin )const { - if( type_num < _types.size() ) - return _types[type_num]->bin_to_protobuf( bin ); - return std::vector(); - } - - string protobuf_to_json( int type_num, const std::vector& pbuf )const { - if( type_num < _types.size() ) - return _types[type_num]->protobuf_to_json( pbuf ); - return string(); - } - std::vector protobuf_to_bin( int type_num, const std::vector& pbuf )const { - if( type_num < _types.size() ) - return _types[type_num]->protobuf_to_bin( pbuf); - return std::vector(); - } - - string query_protobuf_to_json( const std::vector& pbuf_query )const { - clio::input_stream in( pbuf_query.data(), pbuf_query.size() ); - auto pbquery = clio::from_protobuf(in); - auto jquery = protobuf::to_json_query( pbquery ); - return to_json( jquery ); - } - - std::vector query_json_to_protobuf( string json_query )const { - auto jq = from_json( std::move(json_query) ); - auto pbq = clio::protobuf::from_json_query( jq ); - return to_protobuf(pbq); - } - - // string query_protobuf_to_gql( const byte_view& pbuf_query ); - // string query_json_to_gql( const string_view& json_query ); +namespace clio +{ + class meta_type_base + { + public: + meta_type_base(uint32_t n) : _number(n) {} + + virtual ~meta_type_base() {} + virtual const char* name() const = 0; + + virtual std::vector json_to_protobuf(std::string json) const = 0; + virtual std::vector json_to_bin(std::string json) const = 0; + + virtual std::string protobuf_to_json(const std::vector& b) const = 0; + virtual std::vector protobuf_to_bin(const std::vector& b) const = 0; + + virtual std::vector bin_to_protobuf(const std::vector& b) const = 0; + virtual std::string bin_to_json(const std::vector& b) const = 0; + + uint32_t number() const { return _number; } + + private: + uint32_t _number; + }; + + template + class meta_type : public meta_type_base + { + public: + meta_type(uint32_t n) : meta_type_base(n) {} + virtual ~meta_type(){}; + + virtual const char* name() const override + { + return get_type_name(); // reflect::name(); + } + + virtual std::vector json_to_protobuf(std::string json) const override + { + auto t = from_json(json); + return to_protobuf(t); + } + + virtual std::vector json_to_bin(std::string json) const override + { + auto t = from_json(json); + return to_bin(t); + } + + virtual std::string protobuf_to_json(const std::vector& b) const override + { + auto t = from_protobuf(b); + return to_json(t); + } + + virtual std::vector protobuf_to_bin(const std::vector& b) const override + { + auto t = from_protobuf(b); + return to_bin(t); + } + + virtual std::vector bin_to_protobuf(const std::vector& b) const override + { + auto t = from_bin(b); + return to_protobuf(t); + } + + virtual std::string bin_to_json(const std::vector& b) const override + { + auto t = from_bin(b); + return to_json(t); + } + }; + + template + class translator + { + public: + translator() + { + _schema.generate([&](auto* p) { + _types.push_back(new meta_type>(_types.size())); + }); + } + string get_json_schema() const { return format_json(_schema); } + string get_protobuf_schema() { return to_protobuf_schema(_schema); } + // string get_gql_schema(); + + uint32_t get_type_num(const string& type_name) const + { + for (const auto& t : _types) + if (t->name() == type_name) + return t->number(); + return -1; + } + + string get_type_name(uint32_t type_num) const + { + if (type_num < _types.size()) + return _types[type_num]->name(); + return string(); + } + + std::vector json_to_bin(uint32_t type_num, std::string json) const + { + if (type_num < _types.size()) + return _types[type_num]->json_to_bin(json); + return std::vector(); + } + + std::vector json_to_protobuf(int type_num, std::string json) const + { + if (type_num < _types.size()) + return _types[type_num]->json_to_protobuf(json); + return std::vector(); + } + + string bin_to_json(int type_num, const std::vector& bin) const + { + if (type_num < _types.size()) + return _types[type_num]->bin_to_json(bin); + return string(); + } + std::vector bin_to_protobuf(int type_num, const std::vector& bin) const + { + if (type_num < _types.size()) + return _types[type_num]->bin_to_protobuf(bin); + return std::vector(); + } + + string protobuf_to_json(int type_num, const std::vector& pbuf) const + { + if (type_num < _types.size()) + return _types[type_num]->protobuf_to_json(pbuf); + return string(); + } + std::vector protobuf_to_bin(int type_num, const std::vector& pbuf) const + { + if (type_num < _types.size()) + return _types[type_num]->protobuf_to_bin(pbuf); + return std::vector(); + } + + string query_protobuf_to_json(const std::vector& pbuf_query) const + { + clio::input_stream in(pbuf_query.data(), pbuf_query.size()); + auto pbquery = clio::from_protobuf(in); + auto jquery = protobuf::to_json_query(pbquery); + return to_json(jquery); + } + + std::vector query_json_to_protobuf(string json_query) const + { + auto jq = from_json(std::move(json_query)); + auto pbq = clio::protobuf::from_json_query(jq); + return to_protobuf(pbq); + } + + // string query_protobuf_to_gql( const byte_view& pbuf_query ); + // string query_json_to_gql( const string_view& json_query ); // string query_gql_to_protobuf( const string_view& gql_query ); // string query_gql_to_josn( const string_view& gql_query ); - private: - schema _schema; - std::vector _types; - }; - - template - void translate_json_to_protobuf( uint32_t number, InStream& in, OutStream& out ) { - varuint32 key; - key.value = number << 3; - if constexpr( std::is_same_v ) { - // wiretype 0 - } else if constexpr ( sizeof(T) == 8 ) - key.value |= uint8_t(protobuf::fixed64); - else if constexpr ( sizeof(T) == 4 ) - key.value |= uint8_t(protobuf::fixed32); - T r; - from_json( r, in ); - to_bin( key, out ); - to_bin( r, out ); - } - - template - bool translate_json_to_protobuf( const schema& sch, const std::string& pbuf_type, - InStream& in, OutStream& out ) { - auto otype = sch.get_object( pbuf_type ); - if( not otype ) { - auto vtype = sch.get_vector( pbuf_type ); - if( vtype ) { + private: + schema _schema; + std::vector _types; + }; + + template + void translate_json_to_protobuf(uint32_t number, InStream& in, OutStream& out) + { + varuint32 key; + key.value = number << 3; + if constexpr (std::is_same_v) + { + // wiretype 0 + } + else if constexpr (sizeof(T) == 8) + key.value |= uint8_t(protobuf::fixed64); + else if constexpr (sizeof(T) == 4) + key.value |= uint8_t(protobuf::fixed32); + T r; + from_json(r, in); + to_bin(key, out); + to_bin(r, out); + } + + template + bool translate_json_to_protobuf(const schema& sch, + const std::string& pbuf_type, + InStream& in, + OutStream& out) + { + auto otype = sch.get_object(pbuf_type); + if (not otype) + { + auto vtype = sch.get_vector(pbuf_type); + if (vtype) + { + } + return false; + } + in.get_start_object(); + auto t = in.peek_token(); + while (t.type != json_token_type::type_end_object) + { + std::string_view key = in.get_key(); + auto member = otype->get_member_by_name(key); + if (member) + { + if (member->type == "double") + translate_json_to_protobuf(member->number, in, out); + else if (member->type == "int64_t") + translate_json_to_protobuf(member->number, in, out); + else if (member->type == "uint64_t") + translate_json_to_protobuf(member->number, in, out); + else if (member->type == "int32_t") + translate_json_to_protobuf(member->number, in, out); + else if (member->type == "uint32_t") + translate_json_to_protobuf(member->number, in, out); + else if (member->type == "int16_t") + translate_json_to_protobuf(member->number, in, out); + else if (member->type == "uint16_t") + translate_json_to_protobuf(member->number, in, out); + else if (member->type == "int8_t") + translate_json_to_protobuf(member->number, in, out); + else if (member->type == "uint8_t") + translate_json_to_protobuf(member->number, in, out); + else if (member->type == "bool") + translate_json_to_protobuf(member->number, in, out); + else if (member->type == "char") + translate_json_to_protobuf(member->number, in, out); + else if (member->type == "varuint32") + translate_json_to_protobuf(member->number, in, out); + else + { + auto st = sch.get_type(member->type); + if (not st) + from_json_skip_value(in); + else if (object_type* obj = std::get_if(&*st)) + { + /// wire_type == wire_type_enum::buffer + } + else if (vector_type* obj = std::get_if(&*st)) + { + /// wire_type == wire_type_enum::buffer + } + else if (variant_type* obj = std::get_if(&*st)) + { + /// wire_type == wire_type_enum::buffer + } + else if (tuple_type* obj = std::get_if(&*st)) + { + /// wire_type == wire_type_enum::buffer + } } - return false; - } - in.get_start_object(); - auto t = in.peek_token(); - while( t.type != json_token_type::type_end_object ) { - std::string_view key = in.get_key(); - auto member = otype->get_member_by_name( key ); - if( member ) { - if( member->type == "double" ) - translate_json_to_protobuf( member->number, in, out ); - else if( member->type == "int64_t" ) - translate_json_to_protobuf( member->number, in, out ); - else if( member->type == "uint64_t" ) - translate_json_to_protobuf( member->number, in, out ); - else if( member->type == "int32_t" ) - translate_json_to_protobuf( member->number, in, out ); - else if( member->type == "uint32_t" ) - translate_json_to_protobuf( member->number, in, out ); - else if( member->type == "int16_t" ) - translate_json_to_protobuf( member->number, in, out ); - else if( member->type == "uint16_t" ) - translate_json_to_protobuf( member->number, in, out ); - else if( member->type == "int8_t" ) - translate_json_to_protobuf( member->number, in, out ); - else if( member->type == "uint8_t" ) - translate_json_to_protobuf( member->number, in, out ); - else if( member->type == "bool" ) - translate_json_to_protobuf( member->number, in, out ); - else if( member->type == "char" ) - translate_json_to_protobuf( member->number, in, out ); - else if( member->type == "varuint32" ) - translate_json_to_protobuf( member->number, in, out ); - else { - auto st = sch.get_type( member->type ); - if( not st ) from_json_skip_value( in ); - else if( object_type* obj = std::get_if(&*st) ) { - /// wire_type == wire_type_enum::buffer - } - else if( vector_type* obj = std::get_if(&*st) ) { - /// wire_type == wire_type_enum::buffer - } - else if( variant_type* obj = std::get_if(&*st) ) { - /// wire_type == wire_type_enum::buffer - } - else if( tuple_type* obj = std::get_if(&*st) ) { - /// wire_type == wire_type_enum::buffer - } - } - /* + /* translate_json_to_protobuf( sch, member->type, in, out ); auto t = in.peek_token(); switch( t.type ) { @@ -242,161 +275,210 @@ namespace clio { case json_token_type::type_start_array: } */ - } else { - from_json_skip_value( in ); + } + else + { + from_json_skip_value(in); + } + } + } + + template + bool translate_protobuf_to_json(const schema& sch, + const std::string& pbuf_type, + InStream& in, + OutStream& out) + { + using namespace clio::protobuf; + auto otype = sch.get_object(pbuf_type); + if (not otype) + { + auto vtype = sch.get_vector(pbuf_type); + if (vtype) + { + out.write('['); + bool first = true; + + while (in.remaining()) + { + uint32_t key = 0; + varuint32_from_bin(key, in); + wire_type_enum type = wire_type_enum(uint8_t(key) & 0x07); + // uint32_t number = key >> 3; + + auto contained_type = vtype->type; + + if (first) + { + first = false; + } + else + { + out.write(','); + } + + switch (type) + { + case wire_type_enum::varint: + { + varuint32 val; + from_bin(val, in); + if (contained_type == "bool") + to_json(bool(val.value), out); + else + to_json(val.value, out); + } + break; + case wire_type_enum::fixed64: + { + int64_t val; + from_bin(val, in); + if (contained_type == "double") + to_json(*((double*)&val), out); + else + to_json(val, out); + } + break; + case wire_type_enum::buffer: + { + if (contained_type == "string") + { + std::string val; + from_bin(val, in); + to_json(val, out); + } + else if (contained_type == "bytes") + { + bytes val; + from_bin(val, in); + to_json(val, out); + } + else + { + varuint32 size; + from_bin(size, in); + if (size.value > in.remaining()) + return false; + input_stream obj_in(in.pos, in.pos + size); + translate_protobuf_to_json(sch, contained_type, obj_in, out); + in.skip(size); + } + } + break; + case wire_type_enum::fixed32: + { + int32_t val; + from_bin(val, in); + if (contained_type == "float") + to_json(*((float*)&val), out); + else + to_json(val, out); + } + break; + }; } - } - } - - template - bool translate_protobuf_to_json( const schema& sch, const std::string& pbuf_type, - InStream& in, OutStream& out ) { - using namespace clio::protobuf; - auto otype = sch.get_object( pbuf_type ); - if( not otype ) { - auto vtype = sch.get_vector( pbuf_type ); - if( vtype ) { - out.write( '[' ); - bool first = true; - - while( in.remaining() ) { - uint32_t key = 0; - varuint32_from_bin( key, in ); - wire_type_enum type = wire_type_enum(uint8_t(key) & 0x07); - //uint32_t number = key >> 3; - - auto contained_type = vtype->type; - - if( first ) { - first = false; - } else { - out.write( ',' ); - } - - switch( type ) { - case wire_type_enum::varint: { - varuint32 val; - from_bin( val, in ); - if( contained_type == "bool" ) - to_json( bool(val.value), out ); - else - to_json( val.value, out ); - } break; - case wire_type_enum::fixed64: { - int64_t val; - from_bin( val, in ); - if( contained_type == "double" ) - to_json( *((double*)&val), out ); - else - to_json( val, out ); - } break; - case wire_type_enum::buffer: { - if( contained_type == "string" ) { - std::string val; - from_bin( val, in ); - to_json( val, out ); - } else if( contained_type == "bytes" ) { - bytes val; - from_bin( val, in ); - to_json( val, out ); - } else { - varuint32 size; - from_bin( size, in ); - if( size.value > in.remaining() ) - return false; - input_stream obj_in( in.pos, in.pos + size ); - translate_protobuf_to_json( sch, contained_type, obj_in, out ); - in.skip( size ); - } - } break; - case wire_type_enum::fixed32: { - int32_t val; - from_bin( val, in ); - if( contained_type == "float" ) - to_json( *((float*)&val), out ); - else - to_json( val, out ); - } break; - }; - } - out.write( ']' ); - return true; + out.write(']'); + return true; + } + return false; + } + + out.write('{'); + bool first = true; + while (in.remaining()) + { + uint32_t key = 0; + varuint32_from_bin(key, in); + wire_type_enum type = wire_type_enum(uint8_t(key) & 0x07); + uint32_t number = key >> 3; + + // std::cout <<"field number: " << number <<" type: " << int64_t(type)<<" "; + auto member_meta = otype->get_member_by_number(number); + + if (first) + { + first = false; + } + else + { + out.write(','); + } + + if (member_meta) + { + to_json(member_meta->name, out); + } + else + { + to_json(std::to_string(number), out); + } + + out.write(':'); + + switch (type) + { + case wire_type_enum::varint: + { + varuint32 val; + from_bin(val, in); + if (member_meta->type == "bool") + to_json(bool(val.value), out); + else + to_json(val.value, out); } - return false; - } - - out.write( '{' ); - bool first = true; - while( in.remaining() ) { - uint32_t key = 0; - varuint32_from_bin( key, in ); - wire_type_enum type = wire_type_enum(uint8_t(key) & 0x07); - uint32_t number = key >> 3; - - //std::cout <<"field number: " << number <<" type: " << int64_t(type)<<" "; - auto member_meta = otype->get_member_by_number( number ); - - if( first ) { - first = false; - } else { - out.write( ',' ); - } - - if( member_meta ) { to_json( member_meta->name, out ); } - else { to_json( std::to_string(number), out ); } - - out.write( ':' ); - - switch( type ) { - case wire_type_enum::varint: { - varuint32 val; - from_bin( val, in ); - if( member_meta->type == "bool" ) - to_json( bool(val.value), out ); - else - to_json( val.value, out ); - } break; - case wire_type_enum::fixed64: { - int64_t val; - from_bin( val, in ); - if( member_meta->type == "double" ) - to_json( *((double*)&val), out ); - else - to_json( val, out ); - } break; - case wire_type_enum::buffer: { - if( member_meta->type == "string" ) { - std::string val; - from_bin( val, in ); - to_json( val, out ); - } else if( member_meta->type == "bytes" ) { - bytes val; - from_bin( val, in ); - to_json( val, out ); - } else { - varuint32 size; - from_bin( size, in ); - if( size.value > in.remaining() ) - return false; - input_stream obj_in( in.pos, in.pos + size ); - translate_protobuf_to_json( sch, member_meta->type, obj_in, out ); - in.skip( size ); - } + break; + case wire_type_enum::fixed64: + { + int64_t val; + from_bin(val, in); + if (member_meta->type == "double") + to_json(*((double*)&val), out); + else + to_json(val, out); + } + break; + case wire_type_enum::buffer: + { + if (member_meta->type == "string") + { + std::string val; + from_bin(val, in); + to_json(val, out); + } + else if (member_meta->type == "bytes") + { + bytes val; + from_bin(val, in); + to_json(val, out); + } + else + { + varuint32 size; + from_bin(size, in); + if (size.value > in.remaining()) + return false; + input_stream obj_in(in.pos, in.pos + size); + translate_protobuf_to_json(sch, member_meta->type, obj_in, out); + in.skip(size); + } // a.members.emplace_back( number, std::move(val) ); - } break; - case wire_type_enum::fixed32: { - int32_t val; - from_bin( val, in ); - if( member_meta->type == "float" ) - to_json( *((float*)&val), out ); - else - to_json( val, out ); - } break; - }; - } - out.write( '}' ); - return true; - } - -}; + } + break; + case wire_type_enum::fixed32: + { + int32_t val; + from_bin(val, in); + if (member_meta->type == "float") + to_json(*((float*)&val), out); + else + to_json(val, out); + } + break; + }; + } + out.write('}'); + return true; + } + +}; // namespace clio diff --git a/libraries/clio/include/clio/tuple.hpp b/libraries/clio/include/clio/tuple.hpp index fac00854e..ff1316d6c 100644 --- a/libraries/clio/include/clio/tuple.hpp +++ b/libraries/clio/include/clio/tuple.hpp @@ -1,28 +1,35 @@ #pragma once #include -namespace clio { -enum class tuple_error { - no_error, - invalid_tuple_index -}; // tuple_error -} // namespace clio - -namespace std { - template <> - struct is_error_code_enum : true_type {}; -} // namespace std - - -namespace clio { - -class tuple_error_category_type : public std::error_category { - public: - const char* name() const noexcept override final { return "ConversionError"; } - - std::string message(int c) const override final { - switch (static_cast(c)) { - // clang-format off +namespace clio +{ + enum class tuple_error + { + no_error, + invalid_tuple_index + }; // tuple_error +} // namespace clio + +namespace std +{ + template <> + struct is_error_code_enum : true_type + { + }; +} // namespace std + +namespace clio +{ + class tuple_error_category_type : public std::error_category + { + public: + const char* name() const noexcept override final { return "ConversionError"; } + + std::string message(int c) const override final + { + switch (static_cast(c)) + { + // clang-format off case tuple_error::no_error: return "No error"; case tuple_error::invalid_tuple_index: return "invalid tuple index"; default: return "unknown"; diff --git a/libraries/clio/include/clio/varint.hpp b/libraries/clio/include/clio/varint.hpp index ceed4f1ae..53ee369bb 100644 --- a/libraries/clio/include/clio/varint.hpp +++ b/libraries/clio/include/clio/varint.hpp @@ -5,428 +5,466 @@ #pragma once #include -namespace clio { -/** - * @defgroup varint Variable Length Integer Type - * @ingroup core - * @ingroup types - * @brief Defines variable length integer type which provides more efficient serialization - */ - -/** - * Variable Length Unsigned Integer. This provides more efficient serialization of 32-bit unsigned int. - * It serialuzes a 32-bit unsigned integer in as few bytes as possible - * `varuint32` is unsigned and uses [VLQ or Base-128 encoding](https://en.wikipedia.org/wiki/Variable-length_quantity) - * - * @ingroup varint - */ -struct unsigned_int { - /** - * Construct a new unsigned int object - * - * @param v - Source - */ - unsigned_int(uint32_t v = 0) : value(v) {} - - /** - * Construct a new unsigned int object from a type that is convertible to uint32_t - * - * @tparam T - Type of the source - * @param v - Source - * @pre T must be convertible to uint32_t - */ - template - unsigned_int(T v) : value(v) {} - - // operator uint32_t()const { return value; } - // operator uint64_t()const { return value; } - - /** - * Convert unsigned_int as T - * - * @tparam T - Target type of conversion - * @return T - Converted target - */ - template - operator T() const { - return static_cast(value); - } - - /// @cond OPERATORS - - /** - * Assign 32-bit unsigned integer - * - * @param v - Soruce - * @return unsigned_int& - Reference to this object - */ - unsigned_int& operator=(uint32_t v) { - value = v; - return *this; +namespace clio +{ + /** + * @defgroup varint Variable Length Integer Type + * @ingroup core + * @ingroup types + * @brief Defines variable length integer type which provides more efficient serialization + */ + + /** + * Variable Length Unsigned Integer. This provides more efficient serialization of 32-bit + * unsigned int. It serialuzes a 32-bit unsigned integer in as few bytes as possible `varuint32` + * is unsigned and uses [VLQ or Base-128 + * encoding](https://en.wikipedia.org/wiki/Variable-length_quantity) + * + * @ingroup varint + */ + struct unsigned_int + { + /** + * Construct a new unsigned int object + * + * @param v - Source + */ + unsigned_int(uint32_t v = 0) : value(v) {} + + /** + * Construct a new unsigned int object from a type that is convertible to uint32_t + * + * @tparam T - Type of the source + * @param v - Source + * @pre T must be convertible to uint32_t + */ + template + unsigned_int(T v) : value(v) + { + } + + // operator uint32_t()const { return value; } + // operator uint64_t()const { return value; } + + /** + * Convert unsigned_int as T + * + * @tparam T - Target type of conversion + * @return T - Converted target + */ + template + operator T() const + { + return static_cast(value); + } + + /// @cond OPERATORS + + /** + * Assign 32-bit unsigned integer + * + * @param v - Soruce + * @return unsigned_int& - Reference to this object + */ + unsigned_int& operator=(uint32_t v) + { + value = v; + return *this; + } + + /// @endcond + + /** + * Contained value + */ + uint32_t value; + + /// @cond OPERATORS + + /** + * Check equality between a unsigned_int object and 32-bit unsigned integer + * + * @param i - unsigned_int object to compare + * @param v - 32-bit unsigned integer to compare + * @return true - if equal + * @return false - otherwise + */ + friend bool operator==(const unsigned_int& i, const uint32_t& v) { return i.value == v; } + + /** + * Check equality between 32-bit unsigned integer and a unsigned_int object + * + * @param i - 32-bit unsigned integer to compare + * @param v - unsigned_int object to compare + * @return true - if equal + * @return false - otherwise + */ + friend bool operator==(const uint32_t& i, const unsigned_int& v) { return i == v.value; } + + /** + * Check equality between two unsigned_int objects + * + * @param i - First unsigned_int object to compare + * @param v - Second unsigned_int object to compare + * @return true - if equal + * @return false - otherwise + */ + friend bool operator==(const unsigned_int& i, const unsigned_int& v) + { + return i.value == v.value; + } + + /** + * Check inequality between a unsigned_int object and 32-bit unsigned integer + * + * @param i - unsigned_int object to compare + * @param v - 32-bit unsigned integer to compare + * @return true - if inequal + * @return false - otherwise + */ + friend bool operator!=(const unsigned_int& i, const uint32_t& v) { return i.value != v; } + + /** + * Check inequality between 32-bit unsigned integer and a unsigned_int object + * + * @param i - 32-bit unsigned integer to compare + * @param v - unsigned_int object to compare + * @return true - if unequal + * @return false - otherwise + */ + friend bool operator!=(const uint32_t& i, const unsigned_int& v) { return i != v.value; } + + /** + * Check inequality between two unsigned_int objects + * + * @param i - First unsigned_int object to compare + * @param v - Second unsigned_int object to compare + * @return true - if inequal + * @return false - otherwise + */ + friend bool operator!=(const unsigned_int& i, const unsigned_int& v) + { + return i.value != v.value; + } + + /** + * Check if the given unsigned_int object is less than the given 32-bit unsigned integer + * + * @param i - unsigned_int object to compare + * @param v - 32-bit unsigned integer to compare + * @return true - if i less than v + * @return false - otherwise + */ + friend bool operator<(const unsigned_int& i, const uint32_t& v) { return i.value < v; } + + /** + * Check if the given 32-bit unsigned integer is less than the given unsigned_int object + * + * @param i - 32-bit unsigned integer to compare + * @param v - unsigned_int object to compare + * @return true - if i less than v + * @return false - otherwise + */ + friend bool operator<(const uint32_t& i, const unsigned_int& v) { return i < v.value; } + + /** + * Check if the first given unsigned_int is less than the second given unsigned_int object + * + * @param i - First unsigned_int object to compare + * @param v - Second unsigned_int object to compare + * @return true - if i less than v + * @return false - otherwise + */ + friend bool operator<(const unsigned_int& i, const unsigned_int& v) + { + return i.value < v.value; + } + + /** + * Check if the given unsigned_int object is greater or equal to the given 32-bit unsigned + * integer + * + * @param i - unsigned_int object to compare + * @param v - 32-bit unsigned integer to compare + * @return true - if i is greater or equal to v + * @return false - otherwise + */ + friend bool operator>=(const unsigned_int& i, const uint32_t& v) { return i.value >= v; } + + /** + * Check if the given 32-bit unsigned integer is greater or equal to the given unsigned_int + * object + * + * @param i - 32-bit unsigned integer to compare + * @param v - unsigned_int object to compare + * @return true - if i is greater or equal to v + * @return false - otherwise + */ + friend bool operator>=(const uint32_t& i, const unsigned_int& v) { return i >= v.value; } + + /** + * Check if the first given unsigned_int is greater or equal to the second given unsigned_int + * object + * + * @param i - First unsigned_int object to compare + * @param v - Second unsigned_int object to compare + * @return true - if i is greater or equal to v + * @return false - otherwise + */ + friend bool operator>=(const unsigned_int& i, const unsigned_int& v) + { + return i.value >= v.value; + } + + /// @endcond + }; + + using varuint32 = unsigned_int; + CLIO_REFLECT(varuint32, value); + + template + void convert(const varuint32& src, uint32_t& dst, F&& chooser) + { + dst = src.value; } - /// @endcond - - /** - * Contained value - */ - uint32_t value; - - /// @cond OPERATORS - - /** - * Check equality between a unsigned_int object and 32-bit unsigned integer - * - * @param i - unsigned_int object to compare - * @param v - 32-bit unsigned integer to compare - * @return true - if equal - * @return false - otherwise - */ - friend bool operator==(const unsigned_int& i, const uint32_t& v) { return i.value == v; } - - /** - * Check equality between 32-bit unsigned integer and a unsigned_int object - * - * @param i - 32-bit unsigned integer to compare - * @param v - unsigned_int object to compare - * @return true - if equal - * @return false - otherwise - */ - friend bool operator==(const uint32_t& i, const unsigned_int& v) { return i == v.value; } - - /** - * Check equality between two unsigned_int objects - * - * @param i - First unsigned_int object to compare - * @param v - Second unsigned_int object to compare - * @return true - if equal - * @return false - otherwise - */ - friend bool operator==(const unsigned_int& i, const unsigned_int& v) { return i.value == v.value; } - - /** - * Check inequality between a unsigned_int object and 32-bit unsigned integer - * - * @param i - unsigned_int object to compare - * @param v - 32-bit unsigned integer to compare - * @return true - if inequal - * @return false - otherwise - */ - friend bool operator!=(const unsigned_int& i, const uint32_t& v) { return i.value != v; } - - /** - * Check inequality between 32-bit unsigned integer and a unsigned_int object - * - * @param i - 32-bit unsigned integer to compare - * @param v - unsigned_int object to compare - * @return true - if unequal - * @return false - otherwise - */ - friend bool operator!=(const uint32_t& i, const unsigned_int& v) { return i != v.value; } - - /** - * Check inequality between two unsigned_int objects - * - * @param i - First unsigned_int object to compare - * @param v - Second unsigned_int object to compare - * @return true - if inequal - * @return false - otherwise - */ - friend bool operator!=(const unsigned_int& i, const unsigned_int& v) { return i.value != v.value; } - - /** - * Check if the given unsigned_int object is less than the given 32-bit unsigned integer - * - * @param i - unsigned_int object to compare - * @param v - 32-bit unsigned integer to compare - * @return true - if i less than v - * @return false - otherwise - */ - friend bool operator<(const unsigned_int& i, const uint32_t& v) { return i.value < v; } - - /** - * Check if the given 32-bit unsigned integer is less than the given unsigned_int object - * - * @param i - 32-bit unsigned integer to compare - * @param v - unsigned_int object to compare - * @return true - if i less than v - * @return false - otherwise - */ - friend bool operator<(const uint32_t& i, const unsigned_int& v) { return i < v.value; } - - /** - * Check if the first given unsigned_int is less than the second given unsigned_int object - * - * @param i - First unsigned_int object to compare - * @param v - Second unsigned_int object to compare - * @return true - if i less than v - * @return false - otherwise - */ - friend bool operator<(const unsigned_int& i, const unsigned_int& v) { return i.value < v.value; } - - /** - * Check if the given unsigned_int object is greater or equal to the given 32-bit unsigned integer - * - * @param i - unsigned_int object to compare - * @param v - 32-bit unsigned integer to compare - * @return true - if i is greater or equal to v - * @return false - otherwise - */ - friend bool operator>=(const unsigned_int& i, const uint32_t& v) { return i.value >= v; } - - /** - * Check if the given 32-bit unsigned integer is greater or equal to the given unsigned_int object - * - * @param i - 32-bit unsigned integer to compare - * @param v - unsigned_int object to compare - * @return true - if i is greater or equal to v - * @return false - otherwise - */ - friend bool operator>=(const uint32_t& i, const unsigned_int& v) { return i >= v.value; } - - /** - * Check if the first given unsigned_int is greater or equal to the second given unsigned_int object - * - * @param i - First unsigned_int object to compare - * @param v - Second unsigned_int object to compare - * @return true - if i is greater or equal to v - * @return false - otherwise - */ - friend bool operator>=(const unsigned_int& i, const unsigned_int& v) { return i.value >= v.value; } - - /// @endcond -}; - -using varuint32 = unsigned_int; -CLIO_REFLECT(varuint32, value); - -template -void convert(const varuint32& src, uint32_t& dst, F&& chooser) { - dst = src.value; -} - -template -void from_bin(varuint32& obj, S& stream); - -template -void to_bin(const varuint32& obj, S& stream); + template + void from_bin(varuint32& obj, S& stream); -template -void from_json(varuint32& obj, S& stream); + template + void to_bin(const varuint32& obj, S& stream); -template -void to_json(const varuint32& obj, S& stream); + template + void from_json(varuint32& obj, S& stream); -template -void to_key(const varuint32& obj, S& stream) { - return to_key_varuint32(obj.value, stream); -} + template + void to_json(const varuint32& obj, S& stream); -/** - * Variable Length Signed Integer. This provides more efficient serialization of 32-bit signed int. - * It serializes a 32-bit signed integer in as few bytes as possible. - * - * @ingroup varint - * @note `varint32' is signed and uses [Zig-Zag - * encoding](https://developers.google.com/protocol-buffers/docs/encoding#signed-integers) - */ -struct signed_int { - /** - * Construct a new signed int object - * - * @param v - Source - */ - signed_int(int32_t v = 0) : value(v) {} - - /// @cond OPERATORS - - /** - * Convert signed_int to primitive 32-bit signed integer - * - * @return int32_t - The converted result - */ - operator int32_t() const { return value; } - - /** - * Assign an object that is convertible to int32_t - * - * @tparam T - Type of the assignment object - * @param v - Source - * @return unsigned_int& - Reference to this object - */ - template - signed_int& operator=(const T& v) { - value = v; - return *this; + template + void to_key(const varuint32& obj, S& stream) + { + return to_key_varuint32(obj.value, stream); } /** - * Increment operator - * - * @return signed_int - New signed_int with value incremented from the current object's value - */ - signed_int operator++(int) { return value++; } - - /** - * Increment operator - * - * @return signed_int - Reference to current object - */ - signed_int& operator++() { - ++value; - return *this; + * Variable Length Signed Integer. This provides more efficient serialization of 32-bit signed + * int. It serializes a 32-bit signed integer in as few bytes as possible. + * + * @ingroup varint + * @note `varint32' is signed and uses [Zig-Zag + * encoding](https://developers.google.com/protocol-buffers/docs/encoding#signed-integers) + */ + struct signed_int + { + /** + * Construct a new signed int object + * + * @param v - Source + */ + signed_int(int32_t v = 0) : value(v) {} + + /// @cond OPERATORS + + /** + * Convert signed_int to primitive 32-bit signed integer + * + * @return int32_t - The converted result + */ + operator int32_t() const { return value; } + + /** + * Assign an object that is convertible to int32_t + * + * @tparam T - Type of the assignment object + * @param v - Source + * @return unsigned_int& - Reference to this object + */ + template + signed_int& operator=(const T& v) + { + value = v; + return *this; + } + + /** + * Increment operator + * + * @return signed_int - New signed_int with value incremented from the current object's value + */ + signed_int operator++(int) { return value++; } + + /** + * Increment operator + * + * @return signed_int - Reference to current object + */ + signed_int& operator++() + { + ++value; + return *this; + } + + /// @endcond + + /** + * Contained value + */ + int32_t value; + + /// @cond OPERATORS + + /** + * Check equality between a signed_int object and 32-bit integer + * + * @param i - signed_int object to compare + * @param v - 32-bit integer to compare + * @return true - if equal + * @return false - otherwise + */ + friend bool operator==(const signed_int& i, const int32_t& v) { return i.value == v; } + + /** + * Check equality between 32-bit integer and a signed_int object + * + * @param i - 32-bit integer to compare + * @param v - signed_int object to compare + * @return true - if equal + * @return false - otherwise + */ + friend bool operator==(const int32_t& i, const signed_int& v) { return i == v.value; } + + /** + * Check equality between two signed_int objects + * + * @param i - First signed_int object to compare + * @param v - Second signed_int object to compare + * @return true - if equal + * @return false - otherwise + */ + friend bool operator==(const signed_int& i, const signed_int& v) + { + return i.value == v.value; + } + + /** + * Check inequality between a signed_int object and 32-bit integer + * + * @param i - signed_int object to compare + * @param v - 32-bit integer to compare + * @return true - if inequal + * @return false - otherwise + */ + friend bool operator!=(const signed_int& i, const int32_t& v) { return i.value != v; } + + /** + * Check inequality between 32-bit integer and a signed_int object + * + * @param i - 32-bit integer to compare + * @param v - signed_int object to compare + * @return true - if unequal + * @return false - otherwise + */ + friend bool operator!=(const int32_t& i, const signed_int& v) { return i != v.value; } + + /** + * Check inequality between two signed_int objects + * + * @param i - First signed_int object to compare + * @param v - Second signed_int object to compare + * @return true - if inequal + * @return false - otherwise + */ + friend bool operator!=(const signed_int& i, const signed_int& v) + { + return i.value != v.value; + } + + /** + * Check if the given signed_int object is less than the given 32-bit integer + * + * @param i - signed_int object to compare + * @param v - 32-bit integer to compare + * @return true - if i less than v + * @return false - otherwise + */ + friend bool operator<(const signed_int& i, const int32_t& v) { return i.value < v; } + + /** + * Check if the given 32-bit integer is less than the given signed_int object + * + * @param i - 32-bit integer to compare + * @param v - signed_int object to compare + * @return true - if i less than v + * @return false - otherwise + */ + friend bool operator<(const int32_t& i, const signed_int& v) { return i < v.value; } + + /** + * Check if the first given signed_int is less than the second given signed_int object + * + * @param i - First signed_int object to compare + * @param v - Second signed_int object to compare + * @return true - if i less than v + * @return false - otherwise + */ + friend bool operator<(const signed_int& i, const signed_int& v) { return i.value < v.value; } + + /** + * Check if the given signed_int object is greater or equal to the given 32-bit integer + * + * @param i - signed_int object to compare + * @param v - 32-bit integer to compare + * @return true - if i is greater or equal to v + * @return false - otherwise + */ + friend bool operator>=(const signed_int& i, const int32_t& v) { return i.value >= v; } + + /** + * Check if the given 32-bit integer is greater or equal to the given signed_int object + * + * @param i - 32-bit integer to compare + * @param v - signed_int object to compare + * @return true - if i is greater or equal to v + * @return false - otherwise + */ + friend bool operator>=(const int32_t& i, const signed_int& v) { return i >= v.value; } + + /** + * Check if the first given signed_int is greater or equal to the second given signed_int + * object + * + * @param i - First signed_int object to compare + * @param v - Second signed_int object to compare + * @return true - if i is greater or equal to v + * @return false - otherwise + */ + friend bool operator>=(const signed_int& i, const signed_int& v) + { + return i.value >= v.value; + } + + /// @endcond + }; + + using varint32 = signed_int; + CLIO_REFLECT(varint32, value); + + template + void from_bin(varint32& obj, S& stream); + + template + void to_bin(const varint32& obj, S& stream); + + template + void from_json(varint32& obj, S& stream); + + template + void to_json(const varint32& obj, S& stream); + + template + void to_key(const varint32& obj, S& stream) + { + to_key_varint32(obj.value, stream); } - /// @endcond - - /** - * Contained value - */ - int32_t value; - - /// @cond OPERATORS - - /** - * Check equality between a signed_int object and 32-bit integer - * - * @param i - signed_int object to compare - * @param v - 32-bit integer to compare - * @return true - if equal - * @return false - otherwise - */ - friend bool operator==(const signed_int& i, const int32_t& v) { return i.value == v; } - - /** - * Check equality between 32-bit integer and a signed_int object - * - * @param i - 32-bit integer to compare - * @param v - signed_int object to compare - * @return true - if equal - * @return false - otherwise - */ - friend bool operator==(const int32_t& i, const signed_int& v) { return i == v.value; } - - /** - * Check equality between two signed_int objects - * - * @param i - First signed_int object to compare - * @param v - Second signed_int object to compare - * @return true - if equal - * @return false - otherwise - */ - friend bool operator==(const signed_int& i, const signed_int& v) { return i.value == v.value; } - - /** - * Check inequality between a signed_int object and 32-bit integer - * - * @param i - signed_int object to compare - * @param v - 32-bit integer to compare - * @return true - if inequal - * @return false - otherwise - */ - friend bool operator!=(const signed_int& i, const int32_t& v) { return i.value != v; } - - /** - * Check inequality between 32-bit integer and a signed_int object - * - * @param i - 32-bit integer to compare - * @param v - signed_int object to compare - * @return true - if unequal - * @return false - otherwise - */ - friend bool operator!=(const int32_t& i, const signed_int& v) { return i != v.value; } - - /** - * Check inequality between two signed_int objects - * - * @param i - First signed_int object to compare - * @param v - Second signed_int object to compare - * @return true - if inequal - * @return false - otherwise - */ - friend bool operator!=(const signed_int& i, const signed_int& v) { return i.value != v.value; } - - /** - * Check if the given signed_int object is less than the given 32-bit integer - * - * @param i - signed_int object to compare - * @param v - 32-bit integer to compare - * @return true - if i less than v - * @return false - otherwise - */ - friend bool operator<(const signed_int& i, const int32_t& v) { return i.value < v; } - - /** - * Check if the given 32-bit integer is less than the given signed_int object - * - * @param i - 32-bit integer to compare - * @param v - signed_int object to compare - * @return true - if i less than v - * @return false - otherwise - */ - friend bool operator<(const int32_t& i, const signed_int& v) { return i < v.value; } - - /** - * Check if the first given signed_int is less than the second given signed_int object - * - * @param i - First signed_int object to compare - * @param v - Second signed_int object to compare - * @return true - if i less than v - * @return false - otherwise - */ - friend bool operator<(const signed_int& i, const signed_int& v) { return i.value < v.value; } - - /** - * Check if the given signed_int object is greater or equal to the given 32-bit integer - * - * @param i - signed_int object to compare - * @param v - 32-bit integer to compare - * @return true - if i is greater or equal to v - * @return false - otherwise - */ - friend bool operator>=(const signed_int& i, const int32_t& v) { return i.value >= v; } - - /** - * Check if the given 32-bit integer is greater or equal to the given signed_int object - * - * @param i - 32-bit integer to compare - * @param v - signed_int object to compare - * @return true - if i is greater or equal to v - * @return false - otherwise - */ - friend bool operator>=(const int32_t& i, const signed_int& v) { return i >= v.value; } - - /** - * Check if the first given signed_int is greater or equal to the second given signed_int object - * - * @param i - First signed_int object to compare - * @param v - Second signed_int object to compare - * @return true - if i is greater or equal to v - * @return false - otherwise - */ - friend bool operator>=(const signed_int& i, const signed_int& v) { return i.value >= v.value; } - - /// @endcond -}; - -using varint32 = signed_int; -CLIO_REFLECT(varint32, value); - -template -void from_bin(varint32& obj, S& stream); - -template -void to_bin(const varint32& obj, S& stream); - -template -void from_json(varint32& obj, S& stream); - -template -void to_json(const varint32& obj, S& stream); - -template -void to_key(const varint32& obj, S& stream) { - to_key_varint32(obj.value, stream); -} - -} // namespace clio +} // namespace clio From 33bb5a82706ba6ed8d015f84a62a9046d080b0ef Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Mon, 26 Apr 2021 22:01:59 -0400 Subject: [PATCH 3/3] clio tests --- CMakeLists.txt | 1 + libraries/CMakeLists.txt | 3 + libraries/clio/CMakeLists.txt | 57 ++----------------- libraries/clio/include/clio/from_protobuf.hpp | 1 + libraries/clio/tests/CMakeLists.txt | 30 ++++------ libraries/eosiolib/CMakeLists.txt | 23 ++++++++ .../eosiolib/tester/wasi_polyfill/prints.cpp | 8 +++ native/CMakeLists.txt | 4 ++ wasm/CMakeLists.txt | 5 ++ 9 files changed, 61 insertions(+), 71 deletions(-) create mode 100644 libraries/eosiolib/tester/wasi_polyfill/prints.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c7105bf1..ecd572372 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,6 +15,7 @@ file(WRITE ${CMAKE_BINARY_DIR}/CTestTestfile.cmake) option(BUILD_NATIVE "Build native code" ON) if(BUILD_NATIVE) add_subdirectory(native) + file(APPEND ${CMAKE_BINARY_DIR}/CTestTestfile.cmake "subdirs(\"native\")\n") endif() if(DEFINED WASI_SDK_PREFIX) diff --git a/libraries/CMakeLists.txt b/libraries/CMakeLists.txt index 465521e09..dc310a509 100644 --- a/libraries/CMakeLists.txt +++ b/libraries/CMakeLists.txt @@ -1,4 +1,7 @@ +enable_testing() + add_subdirectory(abieos) +add_subdirectory(clio) if(DEFINED IS_WASM) add_subdirectory(eosiolib) diff --git a/libraries/clio/CMakeLists.txt b/libraries/clio/CMakeLists.txt index 45e45a802..0e5d4f2b3 100644 --- a/libraries/clio/CMakeLists.txt +++ b/libraries/clio/CMakeLists.txt @@ -1,54 +1,7 @@ -add_library( clio INTERFACE ) -target_link_libraries( clio INTERFACE rapidjson ) -target_include_directories( clio INTERFACE include ${Boost_INCLUDE_DIRS} ) +enable_testing() -target_sources( clio INTERFACE - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/fpconv.h - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/powers.h - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/compress.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/error.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/reflect.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/schema.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/translator.hpp +add_library(clio INTERFACE) +target_link_libraries(clio INTERFACE rapidjson) +target_include_directories(clio INTERFACE include ${Boost_INCLUDE_DIRS}) - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/stream.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/to_key.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/tuple.hpp - - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/flatbuf.hpp - - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/to_json.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/to_json/map.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/to_json/varint.hpp - - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/json/any.hpp - - - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/to_bin.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/to_bin/map.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/to_bin/set.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/to_bin/list.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/to_bin/deque.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/to_bin/varint.hpp - - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/from_bin.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/from_json.hpp - - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/varint.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/from_bin/varint.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/from_json/varint.hpp - - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/bytes.hpp - - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/bytes/from_json.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/bytes/to_json.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/bytes/to_bin.hpp - - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/from_protobuf.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/to_protobuf.hpp - - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/protobuf/any.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/protobuf/json.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/protobuf/query.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/include/clio/protobuf/schema.hpp -) +add_subdirectory(tests) diff --git a/libraries/clio/include/clio/from_protobuf.hpp b/libraries/clio/include/clio/from_protobuf.hpp index f651ec32a..94d0b27b2 100644 --- a/libraries/clio/include/clio/from_protobuf.hpp +++ b/libraries/clio/include/clio/from_protobuf.hpp @@ -1,5 +1,6 @@ #pragma once #include +#include #include namespace clio diff --git a/libraries/clio/tests/CMakeLists.txt b/libraries/clio/tests/CMakeLists.txt index 54e0dde62..b7265bb7f 100644 --- a/libraries/clio/tests/CMakeLists.txt +++ b/libraries/clio/tests/CMakeLists.txt @@ -1,23 +1,15 @@ -if(DEFINED IS_NATIVE) - find_package(Threads REQUIRED) - add_executable(clio-tests - clio_tests.cpp - flat_views.cpp - benchmark.cpp - crypto.cpp - ) - target_link_libraries(clio-tests clio clcrypto fc catch2-portable Threads::Threads) -endif() +enable_testing() + +add_executable(test-clio + clio_tests.cpp + flat_views.cpp + benchmark.cpp +) +target_link_libraries(test-clio clio catch2) +set_target_properties(test-clio PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${ROOT_BINARY_DIR}) if(DEFINED IS_WASM) - add_executable(clio-tests - clio_tests.cpp - flat_views.cpp - benchmark.cpp - ) - target_link_libraries(clio-tests clio catch2-portable boost) + target_link_libraries(test-clio basic-polyfill boost) endif() -set_target_properties(clio-tests PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${ROOT_BINARY_DIR}) - -native_and_wasm_test(clio-tests) +native_and_wasm_test(test-clio) diff --git a/libraries/eosiolib/CMakeLists.txt b/libraries/eosiolib/CMakeLists.txt index 9d253df29..c949581f9 100644 --- a/libraries/eosiolib/CMakeLists.txt +++ b/libraries/eosiolib/CMakeLists.txt @@ -87,3 +87,26 @@ target_link_libraries(testlib PUBLIC fmt boost eosio-core) target_include_directories(testlib PUBLIC contracts tester) target_link_options(testlib INTERFACE -Wl,--export-table) add_dependencies(testlib fmt) + +add_library(basic-polyfill) +target_compile_options(basic-polyfill PUBLIC -fno-exceptions) +target_sources(basic-polyfill PRIVATE + tester/wasi_polyfill/__wasi_args_get.cpp + tester/wasi_polyfill/__wasi_args_sizes_get.cpp + tester/wasi_polyfill/__wasi_clock_time_get.cpp + tester/wasi_polyfill/__wasi_environ_get.cpp + tester/wasi_polyfill/__wasi_environ_sizes_get.cpp + tester/wasi_polyfill/__wasi_fd_close.cpp + tester/wasi_polyfill/__wasi_fd_fdstat_get.cpp + tester/wasi_polyfill/__wasi_fd_fdstat_set_flags.cpp + tester/wasi_polyfill/__wasi_fd_prestat_dir_name.cpp + tester/wasi_polyfill/__wasi_fd_prestat_get.cpp + tester/wasi_polyfill/__wasi_fd_read.cpp + tester/wasi_polyfill/__wasi_fd_seek.cpp + tester/wasi_polyfill/__wasi_fd_write.cpp + tester/wasi_polyfill/__wasi_path_open.cpp + tester/wasi_polyfill/__wasi_proc_exit.cpp + tester/wasi_polyfill/prints.cpp +) +target_link_libraries(basic-polyfill PUBLIC) +target_link_options(basic-polyfill INTERFACE -Wl,--export-table) diff --git a/libraries/eosiolib/tester/wasi_polyfill/prints.cpp b/libraries/eosiolib/tester/wasi_polyfill/prints.cpp new file mode 100644 index 000000000..940e701e2 --- /dev/null +++ b/libraries/eosiolib/tester/wasi_polyfill/prints.cpp @@ -0,0 +1,8 @@ +#include +#include + +extern "C" void prints(const char* cstr) +{ + [[clang::import_name("prints_l")]] void prints_l(const char*, uint32_t); + prints_l(cstr, strlen(cstr)); +} diff --git a/native/CMakeLists.txt b/native/CMakeLists.txt index ed4719b7f..4422ee055 100644 --- a/native/CMakeLists.txt +++ b/native/CMakeLists.txt @@ -25,6 +25,10 @@ function(native_test N) ) endfunction() +function(native_and_wasm_test N) + native_test(${N}) +endfunction() + add_subdirectory(../external external) add_subdirectory(../libraries libraries) add_subdirectory(../programs programs) diff --git a/wasm/CMakeLists.txt b/wasm/CMakeLists.txt index 4eda40674..a1fa317c1 100644 --- a/wasm/CMakeLists.txt +++ b/wasm/CMakeLists.txt @@ -29,6 +29,11 @@ function(eden_tester_test N) endfunction() add_subdirectory(../contracts contracts) + +function(native_and_wasm_test N) + eden_tester_test(${N}) +endfunction() + add_subdirectory(../libraries libraries) add_subdirectory(../external external) add_subdirectory(boost)