Merge pull request #9 from APokorny/feat/parser_with_unrolled

Feat/parser with unrolled

CHANGELOG.md

@@ -1,6 +1,18 @@
 
 # Changelog
 
+## Version 0.3.0 9th of March 2023
+
+**Feature**: Bump to HSM 0.3.3 
+- This allows async_json to benefit from the new hsm backend
+
+**Feature**: Fast parser mode
+- Based on the new backend a fast parser mode is added - it can be used to trade ROM for performance:
+- Early tests shows that the new mode is twice as fast with gcc-12
+- The space efficient mode is still the default
+- The feature is enabled in basic_path, basic_json_parser and on_array_element via a tag type: `unrolled_tag` vs `table_tag`
+- Use the 'fast' variants of extractor, path, on_array_element.
+
 ## Version 0.2.0 - 2nd March 2023
 
 **Feature**: build system

CMakeLists.txt

@@ -2,21 +2,31 @@ cmake_policy(SET CMP0057 NEW)
 cmake_policy(SET CMP0048 NEW)
 cmake_policy(SET CMP0077 NEW)
 cmake_minimum_required(VERSION 3.14)
-project(async_json VERSION 0.2.0 LANGUAGES CXX)
+project(async_json VERSION 0.3.0 LANGUAGES CXX)
 
 include(cmake/CPM.cmake)
-CPMAddPackage("gh:APokorny/hsm@0.2.0") 
+CPMAddPackage("gh:APokorny/hsm@0.3.3")
 CPMAddPackage("gh:TheLartians/[email protected]")
 
+option(async_json_PARSER_UNROLLED_SM "Build parser with unrolled state machine interpreter" OFF)
+option(async_json_EXTRACTOR_UNROLLED_SM "Build secondary state machines with unrolled state machine interpreter" OFF)
 if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR)
   option(async_json_BUILD_TESTS "Build examples and tests" ON)
+  option(async_json_BUILD_BENCHMARK "Build benchmarks" ON)
 else()
   option(async_json_BUILD_TESTS "Build examples and tests" OFF)
+  option(async_json_BUILD_BENCHMARK "Build benchmarks" OFF)
 endif()
 add_library(async_json INTERFACE)
 add_library(async_json::async_json ALIAS async_json)
 target_link_libraries(async_json INTERFACE hsm)
 target_compile_features(async_json INTERFACE cxx_std_17)
+if(async_json_PARSER_UNROLLED_SM)
+  target_compile_definitions(async_json INTERFACE ASYNC_JSON_PARSER_UNROLLED_SM)
+endif()
+if(async_json_EXTRACTOR_UNROLLED_SM)
+  target_compile_definitions(async_json INTERFACE ASYNC_JSON_EXTRACTOR_UNROLLED_SM)
+endif()
 target_include_directories(async_json INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include/>
     $<INSTALL_INTERFACE:include>
     )
@@ -27,20 +37,23 @@ packageProject(
   BINARY_DIR ${PROJECT_BINARY_DIR}
   INCLUDE_DIR ${PROJECT_SOURCE_DIR}/include
   INCLUDE_DESTINATION include/${PROJECT_NAME}-${PROJECT_VERSION}
-  DEPENDENCIES "hsm 0.2.0"
+  DEPENDENCIES "hsm 0.3.3"
   VERSION_HEADER "${PROJECT_NAME}/version.h"
   NAMESPACE async_json
   COMPATIBILITY AnyNewerVersion
   DISABLE_VERSION_SUFFIX YES
   ARCH_INDEPENDENT YES
   )
 
+if(async_json_BUILD_BENCHMARK)
+  add_subdirectory(bench)
+endif()
 if(async_json_BUILD_TESTS)
   CPMAddPackage("gh:catchorg/[email protected]")
   enable_testing()
 
   find_program(MEMORYCHECK_COMMAND valgrind)
-  set(MEMORYCHECK_COMMAND_OPTIONS --error-exitcode=1 --verbose --tool=memcheck --leak-check=full --show-leak-kinds=all 
+  set(MEMORYCHECK_COMMAND_OPTIONS --error-exitcode=1 --verbose --tool=memcheck --leak-check=full --show-leak-kinds=all
                                   --track-origins=yes --show-reachable=yes --error-limit=no --gen-suppressions=all)
   include(CTest)
 

README.md

@@ -19,39 +19,16 @@ Boost Software License.
 
 ## Dependencies
 
-* optional C++ std features like string_view
+* In theory only c++17, but the dependencies of async_json already require c++20
 * hsm: https://github.com/APokorny/hsm (so transitively kvasir mpl and https://github.com/APokorny/tiny_tuple)
-* Upcoming setup with fetch content will allow you to just:
+* Everything is being setup with CPM:
 ```CMake
-project(async_json_env LANGUAGES CXX)
+project(your_project LANGUAGES CXX)
 cmake_minimum_required(VERSION 3.14)
 cmake_policy(SET CMP0057 NEW)
 cmake_policy(SET CMP0048 NEW)
-include(FetchContent)
-fetchcontent_declare(async_json 
-    GIT_REPOSITORY https://github.com/APokorny/async_json
-    GIT_TAG development)
-fetchcontent_makeavailable(async_json)
+include(cmake/CPM.cmake)
+cpmaddpacakge("gh:APokorny/[email protected]") 
   ...
 ```
-* previously this was the preferred setup:
 
-```CMake
-project(async_json_env LANGUAGES CXX)
-cmake_minimum_required(VERSION 3.10)
-cmake_policy(SET CMP0057 NEW)
-cmake_policy(SET CMP0048 NEW)
-set(as_subproject
-    kvasir_mpl hsm tiny_tuple)
-
-macro(find_package)
-    if(NOT ${ARGV0} IN_LIST as_subproject)
-        _find_package(${ARGV})
-    endif()
-endmacro()
-
-add_subdirectory(mpl)
-add_subdirectory(tiny-tuple)
-add_subdirectory(hsm)
-add_subdirectory(async_json)
-```

bench/CMakeLists.txt

@@ -0,0 +1,19 @@
+
+cpmaddpackage(NAME nonius
+  GITHUB_REPOSITORY libnonius/nonius
+  GIT_TAG devel
+  DOWNLOAD_ONLY TRUE)
+if(nonius_ADDED)
+  add_library(nonius INTERFACE)
+  target_include_directories(nonius INTERFACE ${nonius_SOURCE_DIR}/include)
+endif()
+
+add_executable(bench parser_variants.cpp)
+target_link_libraries(bench PRIVATE nonius async_json::async_json)
+target_compile_features(bench PRIVATE cxx_std_20)
+
+find_program(MOLD_FOUND mold) 
+if(MOLD_FOUND)
+  target_link_options(bench PRIVATE -fuse-ld=mold -Wl,--icf=all -Wl,--print-icf-sections)
+endif()
+

bench/parser_variants.cpp

@@ -0,0 +1,158 @@
+#define NONIUS_RUNNER
+#include <nonius/nonius_single.h++>
+#include <async_json/json_extractor.hpp>
+#include <ranges>
+
+char const model_data[] = R"(
+{
+  "kind": "youtube#searchListResponse",
+  "etag": "\"m2yskBQFythfE4irbTIeOgYYfBU/PaiEDiVxOyCWelLPuuwa9LKz3Gk\"",
+  "nextPageToken": "CAUQAA",
+  "regionCode": "KE",
+  "pageInfo": {
+    "totalResults": 4249,
+    "resultsPerPage": 5
+  },
+  "items": [
+    {
+      "kind": "youtube#searchResult",
+      "etag": "\"m2yskBQFythfE4irbTIeOgYYfBU/QpOIr3QKlV5EUlzfFcVvDiJT0hw\"",
+      "id": {
+        "kind": "youtube#channel",
+        "channelId": "UCJowOS1R0FnhipXVqEnYU1A"
+      }
+    },
+    {
+      "kind": "youtube#searchResult",
+      "etag": "\"m2yskBQFythfE4irbTIeOgYYfBU/sbCCQ5dWx1zCQwp5fKHlrtFTFJ4\"",
+      "id": {
+        "kind": "youtube#video",
+        "videoId": "IGYEtw94zMM"
+      }
+    },
+    {
+      "kind": "youtube#searchResult",
+      "etag": "\"m2yskBQFythfE4irbTIeOgYYfBU/dKYVYE72ZdmqgW9jTeTQOLtgUsI\"",
+      "id": {
+        "kind": "youtube#video",
+        "videoId": "nxa6TeeJQ1s"
+      }
+    },
+    {
+      "kind": "youtube#searchResult",
+      "etag": "\"m2yskBQFythfE4irbTIeOgYYfBU/AWutzVOt_5p1iLVifyBdfoSTf9E\"",
+      "id": {
+        "kind": "youtube#video",
+        "videoId": "Eqa2nAAhHN0"
+      }
+    },
+    {
+      "kind": "youtube#searchResult",
+      "etag": "\"m2yskBQFythfE4irbTIeOgYYfBU/2dIR9BTfr7QphpBuY3hPU-h5u-4\"",
+      "id": {
+        "kind": "youtube#video",
+        "videoId": "IirngItQuVs"
+      }
+    }
+  ],
+  "result": {
+    "kind": "youtube#searchListResponse",
+    "etag": "\"m2yskBQFythfE4irbTIeOgYYfBU/PaiEDiVxOyCWelLPuuwa9LKz3Gk\"",
+    "nextPageToken": "CAUQAA",
+    "regionCode": "KE",
+    "pageInfo": {
+      "totalResults": 4249,
+      "resultsPerPage": 5
+    },
+    "items": [
+      {
+        "kind": "youtube#searchResult",
+        "etag": "\"m2yskBQFythfE4irbTIeOgYYfBU/QpOIr3QKlV5EUlzfFcVvDiJT0hw\"",
+        "id": {
+          "kind": "youtube#channel",
+          "channelId": "UCJowOS1R0FnhipXVqEnYU1A"
+        }
+      },
+      {
+        "kind": "youtube#searchResult",
+        "etag": "\"m2yskBQFythfE4irbTIeOgYYfBU/sbCCQ5dWx1zCQwp5fKHlrtFTFJ4\"",
+        "id": {
+          "kind": "youtube#video",
+          "videoId": "IGYEtw94zMM"
+        }
+      },
+      {
+        "kind": "youtube#searchResult",
+        "etag": "\"m2yskBQFythfE4irbTIeOgYYfBU/dKYVYE72ZdmqgW9jTeTQOLtgUsI\"",
+        "id": {
+          "kind": "youtube#video",
+          "videoId": "nxa6TeeJQ1s"
+        }
+      },
+      {
+        "kind": "youtube#searchResult",
+        "etag": "\"m2yskBQFythfE4irbTIeOgYYfBU/AWutzVOt_5p1iLVifyBdfoSTf9E\"",
+        "id": {
+          "kind": "youtube#video",
+          "videoId": "Eqa2nAAhHN0"
+        }
+      },
+      {
+        "kind": "youtube#searchResult",
+        "etag": "\"m2yskBQFythfE4irbTIeOgYYfBU/2dIR9BTfr7QphpBuY3hPU-h5u-4\"",
+        "id": {
+          "kind": "youtube#video",
+          "videoId": "IirngItQuVs"
+        }
+      }
+    ]
+  }
+}
+)";
+
+NONIUS_BENCHMARK("parse_with_tables",
+                 []
+                 {
+                     int         kind_counter;
+                     int         items_counter;
+                     int         total_results;
+                     std::string next_page_token;
+                     std::string channel_id;
+                     std::string etag;
+                     using namespace async_json;
+                     auto parser = make_extractor(                            //
+                         [](auto error) { std::cerr << "It failed\n"; },      //
+                         path([](auto const&) {}, "nextPageToken"),           //
+                         path(                                                //
+                             all(                                             //
+                                 path([](auto const&) {}, "etag"),            //
+                                 path([](auto const&) {}, "id", "channelId")  //
+                                 ),                                           //
+                             "result", "items")                               //
+                     );
+                     parser.parse_bytes({model_data, sizeof(model_data)});
+                 })
+
+NONIUS_BENCHMARK("parse_with_unrolled",
+                 []
+                 {
+                     int         kind_counter;
+                     int         items_counter;
+                     int         total_results;
+                     std::string next_page_token;
+                     std::string channel_id;
+                     std::string etag;
+                     using namespace async_json;
+                     auto parser =
+                         make_fast_extractor([](auto error) { std::cerr << "It failed\n"; },
+                                             fast_path([](auto const&) {  }, "nextPageToken"),  //
+                                             fast_path(                                                                         //
+                                                 all(                                                                           //
+                                                     fast_path([](auto const&) {}, "etag"),                                     //
+                                                     fast_path([](auto const&) { }, "id", "channelId")  // a
+                                                     ),
+                                                 "result", "items")  //
+                         );
+                     parser.parse_bytes({model_data, sizeof(model_data)});
+                 })
+

include/async_json/basic_is_path.hpp

@@ -14,6 +14,8 @@
 
 namespace async_json
 {
+struct unrolled_tag;
+struct table_tag;
 
 namespace detail
 {
@@ -36,7 +38,7 @@ struct path_element
 };
 
 }  // namespace detail
-template <typename Traits>
+template <typename Traits, typename IT = table_tag>
 struct basic_is_path
 {
     using traits_type = Traits;
@@ -68,7 +70,7 @@ struct basic_is_path
 
 constexpr detail::path_element arbitrary{detail::path_type::arbitrary};
 #if defined(ASYNC_JSON_EXTERN) && !defined(ASYNC_JSON_EXPAND)
-extern template class basic_is_path<default_traits>;
+extern template class basic_is_path<default_traits, table_tag>;
 #endif
 }  // namespace async_json
 

include/async_json/basic_json_parser.hpp

@@ -8,12 +8,15 @@
 #define ASYNC_JSON_BASIC_JSON_PARSER_HPP_INCLUDED
 #include <cmath>
 #include <hsm/hsm.hpp>
+#include <hsm/unroll_sm.hpp>
 #include <async_json/default_traits.hpp>
 #include <async_json/saj_event_value.hpp>
 namespace async_json
 {
+struct unrolled_tag;
+struct table_tag;
 
-template <typename Handler = std::function<void(saj_event_value<default_traits> const&)>, typename Traits = default_traits>
+template <typename Handler = std::function<void(saj_event_value<default_traits> const&)>, typename Traits = default_traits, typename InterpreterTag = table_tag>
 struct basic_json_parser
 {
     using float_t   = typename Traits::float_t;
@@ -63,7 +66,7 @@ struct basic_json_parser
 };
 
 #if defined(ASYNC_JSON_EXTERN) && !defined(ASYNC_JSON_EXPAND)
-extern template class basic_json_parser<std::function<void(async_json::saj_event_value<async_json::default_traits> const& )>, default_traits>;
+extern template class basic_json_parser<std::function<void(async_json::saj_event_value<async_json::default_traits> const& )>, default_traits, table_tag>;
 #endif
 
 }  // namespace async_json

include/async_json/basic_on_array_element.hpp

@@ -12,7 +12,9 @@
 
 namespace async_json
 {
-template <typename Traits>
+struct table_tag;
+struct unrolled_tag;
+template <typename Traits, typename InterpreterTag = async_json::table_tag>
 struct basic_on_array_element
 {
     using event_value = async_json::saj_event_value<Traits>;