diff --git a/common/vars.mk b/common/vars.mk index 1eb4845a..3b1c241c 100644 --- a/common/vars.mk +++ b/common/vars.mk @@ -226,6 +226,8 @@ RAPIDYAML := $(ROOT)/rapidyaml RAPIDYAML_VERSION := 0.7.2 RAPIDYAML_TAG ?= v$(RAPIDYAML_VERSION) RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml +RAPIDYAML_BUILD_TYPE := Release +RAPIDYAML_DBG := 0 RAPIDYAML_TIMED := 0 RAPIDYAML_JAVA := \ $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java \ diff --git a/rapidyaml/Makefile b/rapidyaml/Makefile index c0d7649a..e29ba8c0 100644 --- a/rapidyaml/Makefile +++ b/rapidyaml/Makefile @@ -10,9 +10,7 @@ RAPIDYAML_JAR_DEPS := \ pom.xml \ ) -RAPIDYAML_CLASS := \ - src/main/java/org/rapidyaml/Rapidyaml.class \ - src/main/java/org/rapidyaml/YamlParseErrorException.class +RAPIDYAML_CLASS := $(RAPIDYAML_JAVA:.java=.class) #------------------------------------------------------------------------------ @@ -23,7 +21,7 @@ default:: x: YS -ce 'foo: var' -build:: $(RAPIDYAML_SO) $(RAPIDYAML_LIB) +build:: $(RAPIDYAML_JNI_H) $(RAPIDYAML_SO) $(RAPIDYAML_LIB) @: jar: $(RAPIDYAML_JAR) @@ -51,7 +49,7 @@ sysclean:: #------------------------------------------------------------------------------ -$(RAPIDYAML_SO): +$(RAPIDYAML_SO): $(RAPIDYAML_JNI_H) $(MAKE) -C native $@ $(RAPIDYAML_LIB): $(MAKE) -C native $@ @@ -63,9 +61,12 @@ $(RAPIDYAML_JAR): $(RAPIDYAML_CLASS) $(JAVA_INSTALLED) jar -cvf $@ $< $(RAPIDYAML_CLASS): $(RAPIDYAML_JAVA) $(JAVA_INSTALLED) - # this doesn't work: - #javac $< - # ... but this does: + @# this doesn't work: + @#javac $< + @# ... but this does: javac $(RAPIDYAML_JAVA) +$(RAPIDYAML_JNI_H): $(RAPIDYAML_JAVA) + $(MAKE) -C native $@ + crl: $(RAPIDYAML_CLASS) diff --git a/rapidyaml/native/.gitignore b/rapidyaml/native/.gitignore index 226243a9..8e53ad03 100644 --- a/rapidyaml/native/.gitignore +++ b/rapidyaml/native/.gitignore @@ -1,4 +1,5 @@ /rapidyaml/ -/rapidyaml-build/ -/rapidyaml_all.hpp +/_build/ /librapidyaml.* +/.cache +/compile_commands.json diff --git a/rapidyaml/native/CMakeLists.txt b/rapidyaml/native/CMakeLists.txt index 877eeb4f..61fd11e3 100644 --- a/rapidyaml/native/CMakeLists.txt +++ b/rapidyaml/native/CMakeLists.txt @@ -1,45 +1,69 @@ cmake_minimum_required(VERSION 3.12) -project(rapidyaml - DESCRIPTION "rapidyaml -> yamlscript" +project(ysparse + DESCRIPTION "ysparse: rapidyaml -> yamlscript" HOMEPAGE_URL "https://github.com/biojppm/rapidyaml -> https://github.com/yaml/yamlscript" LANGUAGES CXX) find_package(JNI REQUIRED) -option(YS2EDN_TIMED "add timings to sections" OFF) +option(YSPARSE_TIMED "add timings to sections" OFF) +option(YSPARSE_DBG "enable debug logs" OFF) if(UNIX) set(CMAKE_SHARED_LIBRARY_SUFFIX .so) endif() -add_library(rapidyaml +set(libname rapidyaml) # TODO rename to ysparse + +add_library(${libname} + # + # JNI bridge org_rapidyaml_Rapidyaml.h org_rapidyaml_Rapidyaml.cpp - rapidyaml_all.hpp - rapidyaml_edn_handler.hpp - rapidyaml_edn_handler.cpp - rapidyaml_edn.hpp - rapidyaml_edn.cpp + # + # ysparse files + ysparse_edn_handler.hpp + ysparse_edn_handler.cpp + ysparse_edn.hpp + ysparse_edn.cpp + ysparse_evt_handler.hpp + ysparse_evt_handler.cpp + ysparse_evt.hpp + ysparse_evt.cpp + # + # files required from rapidyaml + rapidyaml/src/c4/yml/common.cpp + rapidyaml/src/c4/yml/node_type.cpp + rapidyaml/src/c4/yml/parse.cpp + rapidyaml/src/c4/yml/tree.cpp + rapidyaml/src/c4/yml/tag.cpp + rapidyaml/src/c4/yml/reference_resolver.cpp + # + # files required from rapidyaml/ext/c4core + rapidyaml/ext/c4core/src/c4/base64.cpp + rapidyaml/ext/c4core/src/c4/error.cpp + rapidyaml/ext/c4core/src/c4/language.cpp + rapidyaml/ext/c4core/src/c4/utf.cpp ) -target_include_directories(rapidyaml PUBLIC +target_include_directories(${libname} PUBLIC ${CMAKE_CURRENT_LIST_DIR} - ${CMAKE_CURRENT_LIST_DIR}/rapidyaml/test + ${CMAKE_CURRENT_LIST_DIR}/rapidyaml/src + ${CMAKE_CURRENT_LIST_DIR}/rapidyaml/ext/c4core/src ) -target_compile_definitions(rapidyaml PUBLIC +target_compile_definitions(${libname} PUBLIC RYML_WITH_TAB_TOKENS RYML_DEFAULT_CALLBACK_USES_EXCEPTIONS - RYML_SINGLE_HEADER - $<$:YS2EDN_TIMED> + $<$:YSPARSE_TIMED> + $<$:RYML_DBG> ) -set_target_properties(rapidyaml PROPERTIES CXX_STANDARD 17) +set_target_properties(${libname} PROPERTIES CXX_STANDARD 17) -# target_link_libraries(rapidyaml PUBLIC JNI::JNI) -target_include_directories(rapidyaml PUBLIC ${JNI_INCLUDE_DIRS}) +target_include_directories(${libname} PUBLIC ${JNI_INCLUDE_DIRS}) -add_executable(rapidyaml-test rapidyaml_test.cpp) -target_link_libraries(rapidyaml-test rapidyaml) -add_custom_target(rapidyaml-test-run - DEPENDS rapidyaml-test - COMMAND $ +add_executable(${libname}-test ysparse_test.cpp) +target_link_libraries(${libname}-test ${libname}) +add_custom_target(${libname}-test-run + DEPENDS ${libname}-test + COMMAND $ COMMENT "running C++ tests" ) diff --git a/rapidyaml/native/Makefile b/rapidyaml/native/Makefile index a11f5ccd..91dd05ea 100644 --- a/rapidyaml/native/Makefile +++ b/rapidyaml/native/Makefile @@ -3,24 +3,27 @@ include $(COMMON)/clojure.mk include $(COMMON)/java.mk include $(COMMON)/python.mk -RAPIDYAML_H := org_rapidyaml_Rapidyaml.h -RAPIDYAML_HPP := rapidyaml_all.hpp -RAPIDYAML_AMALGAMATE_OPTS := --stl +# TODO change to static library! +# https://www.graalvm.org/latest/reference-manual/native-image/guides/build-static-executables/ +# https://www.blog.akhil.cc/static-jni +# https://stackoverflow.com/questions/24493337/linking-static-library-with-jni + +THIS_DIR := $(shell pwd) +BDIR := $(THIS_DIR)/_build/$(BUILD_TYPE)-shared$(SHARED)-timed$(TIMED)-dbg$(DBG) RAPIDYAML_DEPS := \ Makefile \ CMakeLists.txt \ $(JAVA_HOME) \ - $(RAPIDYAML_H) \ - $(RAPIDYAML_HPP) \ + $(RAPIDYAML_JNI_H) \ $(wildcard ./*pp) \ -CMK_FLAGS := -CMK_FLAGS += -D YS2EDN_TIMED=$(RAPIDYAML_TIMED) - -BDIR := rapidyaml-build - -CMK_BUILD := +CMK_ENV := +CMK_FLAGS := \ + -D CMAKE_BUILD_TYPE=$(RAPIDYAML_BUILD_TYPE) \ + -D YSPARSE_TIMED=$(RAPIDYAML_TIMED) \ + -D YSPARSE_DBG=$(RAPIDYAML_DBG) \ + -D CMAKE_EXPORT_COMPILE_COMMANDS=ON #------------------------------------------------------------------------------ @@ -39,7 +42,6 @@ rapidyaml: clean:: $(RM) librapidyaml.* - $(RM) $(RAPIDYAML_HPP) $(RM) -r $(BDIR) $(RM) -r rapidyaml-install @@ -67,6 +69,3 @@ $(RAPIDYAML_SO): $(RAPIDYAML_DEPS) $(RAPIDYAML_H): $(RAPIDYAML_JAVA) javac -h . $(RAPIDYAML_JAVA) # $^ doesn't work - -$(RAPIDYAML_HPP): rapidyaml - $(PYTHON) rapidyaml/tools/amalgamate.py $(RAPIDYAML_AMALGAMATE_OPTS) > $@ diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp index 4e57b87d..63fa9efd 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp @@ -1,5 +1,6 @@ #include -#include "./rapidyaml_edn.hpp" +#include "ysparse_edn.hpp" +#include "ysparse_evt.hpp" #include #ifndef _Included_org_rapidyaml_Rapidyaml @@ -50,6 +51,18 @@ Java_org_rapidyaml_Rapidyaml_ys2edn_1init(JNIEnv *, jobject) return (jlong)obj; } +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2evt_init + * Signature: ()J + */ +JNIEXPORT jlong JNICALL +Java_org_rapidyaml_Rapidyaml_ys2evt_1init(JNIEnv *env, jobject) +{ + Ryml2Evt *obj = ys2evt_init(); + return (jlong)obj; +} + /* * Class: org_rapidyaml_Rapidyaml * Method: ys2edn_destroy @@ -61,6 +74,17 @@ Java_org_rapidyaml_Rapidyaml_ys2edn_1destroy(JNIEnv *, jobject, jlong obj) ys2edn_destroy((Ryml2Edn*)obj); } +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2evt_destroy + * Signature: (Ljava/lang/Object;)V + */ +JNIEXPORT void JNICALL +Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy(JNIEnv *, jobject, jlong obj) +{ + ys2evt_destroy((Ryml2Evt*)obj); +} + /* * Class: org_rapidyaml_Rapidyaml * Method: ys2edn_parse @@ -93,6 +117,38 @@ Java_org_rapidyaml_Rapidyaml_ys2edn_1parse(JNIEnv *env, jobject, return rc; } +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2evt_parse + * Signature: (Ljava/lang/Object;Ljava/lang/String;[BI[BI)I + */ +JNIEXPORT jint JNICALL +Java_org_rapidyaml_Rapidyaml_ys2evt_1parse(JNIEnv *env, jobject, + jlong obj, jstring jfilename, + jbyteArray src, jint src_len, + jintArray dst, jint dst_len) +{ + jboolean src_is_copy, dst_is_copy; + jbyte* src_ = env->GetByteArrayElements(src, &src_is_copy); + int* dst_ = env->GetIntArrayElements(dst, &dst_is_copy); + const char *filename = env->GetStringUTFChars(jfilename, 0); + int rc = 0; + try + { + rc = ys2evt_parse((Ryml2Evt*)obj, filename, + (char*)src_, src_len, + dst_, dst_len); + } + catch (Ryml2EvtParseError const& exc) + { + throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str()); + } + env->ReleaseByteArrayElements(src, src_, 0); + env->ReleaseIntArrayElements(dst, dst_, 0); + env->ReleaseStringUTFChars(jfilename, filename); + return rc; +} + /* * Class: org_rapidyaml_Rapidyaml * Method: ys2edn_retry_get diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.h b/rapidyaml/native/org_rapidyaml_Rapidyaml.h index c7349763..daba52ef 100644 --- a/rapidyaml/native/org_rapidyaml_Rapidyaml.h +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.h @@ -39,6 +39,30 @@ JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1parse JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2edn_1retry_1get (JNIEnv *, jobject, jlong, jbyteArray, jint); +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2evt_init + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1init + (JNIEnv *, jobject); + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2evt_destroy + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy + (JNIEnv *, jobject, jlong); + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ys2evt_parse + * Signature: (JLjava/lang/String;[BI[II)I + */ +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ys2evt_1parse + (JNIEnv *, jobject, jlong, jstring, jbyteArray, jint, jintArray, jint); + #ifdef __cplusplus } #endif diff --git a/rapidyaml/native/rapidyaml_test.cpp b/rapidyaml/native/rapidyaml_test.cpp deleted file mode 100644 index 50ca878d..00000000 --- a/rapidyaml/native/rapidyaml_test.cpp +++ /dev/null @@ -1,282 +0,0 @@ -#include - -using c4::csubstr; -using c4::substr; - - -struct Ys2EdnScoped -{ - Ryml2Edn *ryml2edn; - Ys2EdnScoped() : ryml2edn(ys2edn_init()) {} - ~Ys2EdnScoped() { if(ryml2edn) ys2edn_destroy(ryml2edn); } -}; - - -struct TestResult -{ - uint32_t num_assertions; - uint32_t num_tests; - uint32_t num_failed_assertions; - uint32_t num_failed_tests; - operator bool() const { return num_failed_tests == 0; } - void add(TestResult const& that) - { - num_tests += 1 + that.num_tests; - num_assertions += that.num_assertions; - num_failed_tests += (that.num_failed_assertions > 0) + that.num_failed_tests; - num_failed_assertions += that.num_failed_assertions; - } -}; - - -struct TestCase -{ - csubstr ys; - csubstr edn; - bool testeq(csubstr actual) const - { - const bool status = (actual == edn); - if(!status) - printf("------\n" - "FAIL:\n" - "input:~~~%.*s~~~\n" - "expected:~~~%.*s~~~\n" - "actual:~~~%.*s~~~\n", - (int)ys.len, ys.str, - (int)edn.len, edn.str, - (int)actual.len, actual.str); - return status; - } - - #define _runtest(name, ...) \ - do { \ - printf("[ RUN ] %s ... \n", #name); \ - TestResult tr_ = name(__VA_ARGS__); \ - tr.add(tr_); \ - printf("[ %s ] %s\n", tr_?"OK ":"FAIL", #name); \ - } while(0) - #define CHECK(cond) \ - do { \ - bool pass = !!(cond); \ - ++tr.num_assertions; \ - if(!pass) { \ - printf("%s:%d: fail! %s", __FILE__, __LINE__, #cond); \ - ++tr.num_failed_assertions; \ - } \ - } while(0) - - TestResult test(Ryml2Edn *ryml2edn) const - { - TestResult tr = {}; - _runtest(test_large_enough, ); - _runtest(test_too_small, ); - _runtest(test_nullptr, ); - _runtest(test_large_enough_reuse, ryml2edn); - _runtest(test_too_small_reuse, ryml2edn); - _runtest(test_nullptr_reuse, ryml2edn); - return tr; - } - - // happy path: large-enough destination string - TestResult test_large_enough_reuse(Ryml2Edn *ryml2edn) const - { - TestResult tr = {}; - std::string input_(ys.begin(), ys.end()); - substr input = c4::to_substr(input_); - std::string output; - output.resize(2 * edn.len); - size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", - input.str, (size_type)input.len, - &output[0], (size_type)output.size()); - CHECK(reqsize == edn.len+1); - CHECK(reqsize != 0); - output.resize(reqsize - 1u); - CHECK(testeq(c4::to_csubstr(output))); - return tr; - } - TestResult test_large_enough() const - { - Ys2EdnScoped lib; - return test_large_enough_reuse(lib.ryml2edn); - } - - // less-happy path: destination string not large enough - TestResult test_too_small_reuse(Ryml2Edn *ryml2edn) const - { - TestResult tr = {}; - std::string input_(ys.begin(), ys.end()); - substr input = c4::to_substr(input_); - std::string output = "?"; - size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", - input.str, (size_type)input.len, - output.data(), (size_type)output.size()); - CHECK(reqsize == edn.len+1); - CHECK(reqsize != 0); - CHECK(output != "?"); - output.resize(reqsize); - size_type getsize = ys2edn_retry_get(ryml2edn, &output[0], (size_type)output.size()); - CHECK(getsize == reqsize); - output.resize(reqsize - 1u); - CHECK(testeq(c4::to_csubstr(output))); - return tr; - } - TestResult test_too_small() const - { - Ys2EdnScoped lib; - return test_too_small_reuse(lib.ryml2edn); - } - - // safe calling with nullptr - TestResult test_nullptr_reuse(Ryml2Edn *ryml2edn) const - { - TestResult tr = {}; - std::string input_(ys.begin(), ys.end()); - substr input = c4::to_substr(input_); - size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", - input.str, (size_type)input.len, - nullptr, 0); - CHECK(reqsize == edn.len+1); - CHECK(reqsize != 0); - return tr; - } - TestResult test_nullptr() const - { - Ys2EdnScoped lib; - return test_nullptr_reuse(lib.ryml2edn); - } -}; - - -//----------------------------------------------------------------------------- - -const TestCase test_cases[] = { - // case ------------------------------ - {"say: 2 + 2", - R"(( -{:+ "+MAP"} -{:+ "=VAL", := "say"} -{:+ "=VAL", := "2 + 2"} -{:+ "-MAP"} -{:+ "-DOC"} -) -)"}, - // case ------------------------------ - {"a: 1", - R"(( -{:+ "+MAP"} -{:+ "=VAL", := "a"} -{:+ "=VAL", := "1"} -{:+ "-MAP"} -{:+ "-DOC"} -) -)"}, - // case ------------------------------ - {"𝄞: ✅", - R"(( -{:+ "+MAP"} -{:+ "=VAL", := "𝄞"} -{:+ "=VAL", := "✅"} -{:+ "-MAP"} -{:+ "-DOC"} -) -)"}, - // case ------------------------------ - {"[a, b, c]", - R"(( -{:+ "+SEQ", :flow true} -{:+ "=VAL", := "a"} -{:+ "=VAL", := "b"} -{:+ "=VAL", := "c"} -{:+ "-SEQ"} -{:+ "-DOC"} -) -)"}, - // case ------------------------------ - {"[a: b]", - R"(( -{:+ "+SEQ", :flow true} -{:+ "+MAP", :flow true} -{:+ "=VAL", := "a"} -{:+ "=VAL", := "b"} -{:+ "-MAP"} -{:+ "-SEQ"} -{:+ "-DOC"} -) -)"}, - // case ------------------------------ - {R"(--- !yamlscript/v0 -foo: ! -- {x: y} -- [x, y] -- foo -- 'foo' -- "foo" -- | - foo -- > - foo -- [1, 2, true, false, null] -- &anchor-1 !tag-1 foobar ---- -another: doc -)", - R"(( -{:+ "+MAP", :! "yamlscript/v0"} -{:+ "=VAL", := "foo"} -{:+ "+SEQ", :! ""} -{:+ "+MAP", :flow true} -{:+ "=VAL", := "x"} -{:+ "=VAL", := "y"} -{:+ "-MAP"} -{:+ "+SEQ", :flow true} -{:+ "=VAL", := "x"} -{:+ "=VAL", := "y"} -{:+ "-SEQ"} -{:+ "=VAL", := "foo"} -{:+ "=VAL", :' "foo"} -{:+ "=VAL", :$ "foo"} -{:+ "=VAL", :| "foo\n"} -{:+ "=VAL", :> "foo\n"} -{:+ "+SEQ", :flow true} -{:+ "=VAL", := "1"} -{:+ "=VAL", := "2"} -{:+ "=VAL", := "true"} -{:+ "=VAL", := "false"} -{:+ "=VAL", := "null"} -{:+ "-SEQ"} -{:+ "=VAL", :& "anchor-1", :! "tag-1", := "foobar"} -{:+ "-SEQ"} -{:+ "-MAP"} -{:+ "-DOC"} -{:+ "+DOC"} -{:+ "+MAP"} -{:+ "=VAL", := "another"} -{:+ "=VAL", := "doc"} -{:+ "-MAP"} -{:+ "-DOC"} -) -)"} -}; - - -int main() -{ - Ys2EdnScoped ys2edn; - TestResult total = {}; - size_t failed_cases = {}; - size_t num_cases = C4_COUNTOF(test_cases); - for(size_t i = 0; i < C4_COUNTOF(test_cases); ++i) - { - printf("-----------------------------------------\n" - "case %zu/%zu ...\n" - "[%zu]~~~%.*s~~~\n", i, num_cases, test_cases[i].ys.len, (int)test_cases[i].ys.len, test_cases[i].ys.str); - const TestResult tr = test_cases[i].test(ys2edn.ryml2edn); - total.add(tr); - failed_cases += (!tr); - printf("case %zu/%zu: %s\n", i, C4_COUNTOF(test_cases), tr ? "ok!" : "failed"); - } - printf("%u/%u assertions failed\n", total.num_failed_assertions, total.num_assertions); - printf("%u/%u tests failed\n", total.num_failed_tests, total.num_tests); - printf("%zu/%zu cases failed\n", failed_cases, num_cases); - return total ? 0 : -1; -} diff --git a/rapidyaml/native/rapidyaml_edn.cpp b/rapidyaml/native/ysparse_edn.cpp similarity index 96% rename from rapidyaml/native/rapidyaml_edn.cpp rename to rapidyaml/native/ysparse_edn.cpp index 59082f16..b074c484 100644 --- a/rapidyaml/native/rapidyaml_edn.cpp +++ b/rapidyaml/native/ysparse_edn.cpp @@ -1,6 +1,4 @@ -#define RYML_SINGLE_HDR_DEFINE_NOW -#include -#include "rapidyaml_edn.hpp" +#include "ysparse_edn.hpp" #include namespace ryml { @@ -10,7 +8,7 @@ using namespace c4::yml; using namespace ryml; -#ifndef YS2EDN_TIMED +#ifndef YSPARSE_TIMED #define TIMED_SECTION(name) #else #include @@ -64,6 +62,7 @@ RYML_EXPORT void ys2edn_destroy(Ryml2Edn *ryml2edn) { TIMED_SECTION("ys2edn_destroy"); ryml2edn->~Ryml2Edn(); + _RYML_CB_FREE(get_callbacks(), ryml2edn, Ryml2Edn, 1); } RYML_EXPORT size_type ys2edn_parse(Ryml2Edn *ryml2edn, diff --git a/rapidyaml/native/rapidyaml_edn.hpp b/rapidyaml/native/ysparse_edn.hpp similarity index 83% rename from rapidyaml/native/rapidyaml_edn.hpp rename to rapidyaml/native/ysparse_edn.hpp index 28ab0d37..5a587896 100644 --- a/rapidyaml/native/rapidyaml_edn.hpp +++ b/rapidyaml/native/ysparse_edn.hpp @@ -1,10 +1,9 @@ #pragma once -#ifndef RAPIDYAML_EVENTS_H -#define RAPIDYAML_EVENTS_H +#ifndef YSPARSE_EDN_HPP_ +#define YSPARSE_EDN_HPP_ #include -#include -#include "rapidyaml_edn_handler.hpp" +#include "ysparse_edn_handler.hpp" namespace ryml { using namespace c4; @@ -19,9 +18,9 @@ using size_type = int; struct RYML_EXPORT Ryml2Edn { - c4::yml::EventHandlerEdn::EventSink m_sink; - c4::yml::EventHandlerEdn m_handler; - c4::yml::ParseEngine m_parser; + ys::EventHandlerEdn::EventSink m_sink; + ys::EventHandlerEdn m_handler; + c4::yml::ParseEngine m_parser; Ryml2Edn() : m_sink() , m_handler(&m_sink) @@ -68,4 +67,4 @@ RYML_EXPORT size_type ys2edn_retry_get(Ryml2Edn *ryml2edn, } #endif -#endif /* RAPIDYAML_EVENTS_H */ +#endif /* YSPARSE_EDN_HPP_ */ diff --git a/rapidyaml/native/rapidyaml_edn_handler.cpp b/rapidyaml/native/ysparse_edn_handler.cpp similarity index 94% rename from rapidyaml/native/rapidyaml_edn_handler.cpp rename to rapidyaml/native/ysparse_edn_handler.cpp index 635f4460..ab295234 100644 --- a/rapidyaml/native/rapidyaml_edn_handler.cpp +++ b/rapidyaml/native/ysparse_edn_handler.cpp @@ -1,18 +1,17 @@ -#ifndef RYML_SINGLE_HEADER +#include "ysparse_edn_handler.hpp" #include -#include #include -#endif -#include "./rapidyaml_edn_handler.hpp" +// instantiate the template namespace c4 { namespace yml { - -// instantiate the template -template class ParseEngine; +template class ParseEngine; +} // namespace yml +} // namespace c4 +namespace ys { void EventHandlerEdn::EventSink::append_escaped(csubstr val) { #define _c4flush_use_instead(repl, skip) \ @@ -78,6 +77,4 @@ void EventHandlerEdn::EventSink::append_escaped(csubstr val) this->append(val.sub(prev)); #undef _c4flush_use_instead } - -} // namespace yml -} // namespace c4 +} // namespace ys diff --git a/rapidyaml/native/rapidyaml_edn_handler.hpp b/rapidyaml/native/ysparse_edn_handler.hpp similarity index 85% rename from rapidyaml/native/rapidyaml_edn_handler.hpp rename to rapidyaml/native/ysparse_edn_handler.hpp index a50ee026..5497fc60 100644 --- a/rapidyaml/native/rapidyaml_edn_handler.hpp +++ b/rapidyaml/native/ysparse_edn_handler.hpp @@ -1,19 +1,12 @@ -#ifndef _C4_YML_EVENT_HANDLER_YAMLSTD_HPP_ -#define _C4_YML_EVENT_HANDLER_YAMLSTD_HPP_ - -#ifdef RYML_SINGLE_HEADER -#include -#else -#ifndef _C4_YML_EVENT_HANDLER_STACK_HPP_ -#include "c4/yml/event_handler_stack.hpp" -#endif -#ifndef _C4_YML_STD_STRING_HPP_ -#include "c4/yml/std/string.hpp" -#endif -#ifndef _C4_YML_DETAIL_PRINT_HPP_ -#include "c4/yml/detail/print.hpp" -#endif -#endif +#ifndef _YSPARSE_EDN_HANDLER_HPP_ +#define _YSPARSE_EDN_HANDLER_HPP_ + +#include +#include +#include +#include +#include +#include #include @@ -22,21 +15,26 @@ C4_SUPPRESS_WARNING_GCC_CLANG("-Wold-style-cast") C4_SUPPRESS_WARNING_GCC("-Wuseless-cast") -namespace c4 { -namespace yml { - - -/** @addtogroup doc_event_handlers - * @{ */ +namespace ys { +using c4::csubstr; +using c4::substr; +using c4::to_substr; +using c4::to_csubstr; +using c4::yml::id_type; +using c4::yml::NodeType_e; +using c4::yml::type_bits; +#ifdef RYML_DBG +using c4::_dbg_printf; +#endif -struct EventHandlerEdnState : public ParserState +struct EventHandlerEdnState : public c4::yml::ParserState { - NodeData ev_data; + c4::yml::NodeData ev_data; }; -struct EventHandlerEdn : public EventHandlerStack +struct EventHandlerEdn : public c4::yml::EventHandlerStack { /** @name types @@ -82,20 +80,20 @@ struct EventHandlerEdn : public EventHandlerStackflags |= RUNK|RTOP; + m_curr->flags |= c4::yml::RUNK|c4::yml::RTOP; m_val_buffers.resize((size_t)m_stack.size()); m_arena.clear(); m_first_doc = true; @@ -123,7 +121,7 @@ struct EventHandlerEdn : public EventHandlerStack_stack_start_parse(filename, relocate_arena, relocate_arena_data); } @@ -173,7 +171,7 @@ struct EventHandlerEdn : public EventHandlerStack'); - _enable_(KEY|KEY_FOLDED); + _enable_(c4::yml::KEY|c4::yml::KEY_FOLDED); } C4_ALWAYS_INLINE void set_val_scalar_folded(csubstr scalar) { _send_val_scalar_(scalar, '>'); - _enable_(VAL|VAL_FOLDED); + _enable_(c4::yml::VAL|c4::yml::VAL_FOLDED); } @@ -423,26 +421,26 @@ struct EventHandlerEdn : public EventHandlerStackev_data.m_key.anchor = anchor; } void set_val_anchor(csubstr anchor) { - _enable_(VALANCH); + _enable_(c4::yml::VALANCH); m_curr->ev_data.m_val.anchor = anchor; } void set_key_ref(csubstr ref) { _RYML_CB_ASSERT(m_stack.m_callbacks, ref.begins_with('*')); - _enable_(KEY|KEYREF); + _enable_(c4::yml::KEY|c4::yml::KEYREF); _send_("{:+ \"=ALI\" :* \""); _send_(ref.sub(1)); _send_("\"}\n"); } void set_val_ref(csubstr ref) { - _enable_(VAL|VALREF); + _enable_(c4::yml::VAL|c4::yml::VALREF); _send_("{:+ \"=ALI\" :* \""); _send_(ref.sub(1)); _send_("\"}\n"); @@ -457,12 +455,12 @@ struct EventHandlerEdn : public EventHandlerStackev_data.m_key.tag = _transform_directive(tag, m_key_tag_buf); } void set_val_tag(csubstr tag) { - _enable_(VALTAG); + _enable_(c4::yml::VALTAG); m_curr->ev_data.m_val.tag = _transform_directive(tag, m_val_tag_buf); } @@ -513,8 +511,6 @@ struct EventHandlerEdn : public EventHandlerStackev_data.m_key.anchor); _send_('\"'); } - if(_has_any_(KEYTAG)) + if(_has_any_(c4::yml::KEYTAG)) { _send_(", :! \""); _send_tag_(m_curr->ev_data.m_key.tag); _send_('\"'); } m_curr->ev_data.m_key = {}; - _disable_(KEYANCH|KEYREF|KEYTAG); + _disable_(c4::yml::KEYANCH|c4::yml::KEYREF|c4::yml::KEYTAG); } void _send_val_props_() { - if(_has_any_(VALANCH|VALREF)) + if(_has_any_(c4::yml::VALANCH|c4::yml::VALREF)) { _send_(", :& \""); _send_(m_curr->ev_data.m_val.anchor); _send_('\"'); } - if(m_curr->ev_data.m_type.type & VALTAG) + if(m_curr->ev_data.m_type.type & c4::yml::VALTAG) { _send_(", :! \""); _send_tag_(m_curr->ev_data.m_val.tag); _send_('\"'); } m_curr->ev_data.m_val = {}; - _disable_(VALANCH|VALREF|VALTAG); + _disable_(c4::yml::VALANCH|c4::yml::VALREF|c4::yml::VALTAG); } void _send_tag_(csubstr tag) { @@ -663,12 +659,12 @@ struct EventHandlerEdn : public EventHandlerStackpos); } } - csubstr result = normalize_tag_long(tag, output); + csubstr result = c4::yml::normalize_tag_long(tag, output); _RYML_CB_CHECK(m_stack.m_callbacks, result.len > 0); _RYML_CB_CHECK(m_stack.m_callbacks, result.str); return result; @@ -677,14 +673,10 @@ struct EventHandlerEdn : public EventHandlerStack + +namespace ryml { +using namespace c4; +using namespace c4::yml; +} // namespace ryml + +using namespace ryml; + +#ifndef YS2PARSE_TIMED +#define TIMED_SECTION(name) +#else +#include +#define TIMED_SECTION(name) timed_section C4_XCAT(ts, __LINE__)(name) +struct timed_section +{ + using myclock = std::chrono::steady_clock; + using fmsecs = std::chrono::duration; + csubstr name; + myclock::time_point start; + fmsecs since() const { return myclock::now() - start; } + timed_section(csubstr n) : name(n), start(myclock::now()) {} + ~timed_section() + { + fprintf(stderr, "%.6fms: %.*s\n", since().count(), (int)name.len, name.str); + } +}; +#endif + + +#if defined(__cplusplus) +extern "C" { +#endif +// see +// https://stackoverflow.com/questions/230689/best-way-to-throw-exceptions-in-jni-code +// https://stackoverflow.com/questions/4138168/what-happens-when-i-throw-a-c-exception-from-a-native-java-method + +namespace { +C4_NORETURN void ys2evt_parse_error(const char* msg, size_t msg_len, Location location, void *user_data) +{ + Ryml2EvtParseError exc; + exc.location = location; + exc.msg.assign(msg, msg_len); + throw exc; +} +} // anon namespace + +RYML_EXPORT Ryml2Evt *ys2evt_init() +{ + TIMED_SECTION("ys2evt_init"); + Callbacks cb = {}; + cb.m_error = &ys2evt_parse_error; + set_callbacks(cb); + Ryml2Evt *ryml2evt = _RYML_CB_ALLOC(get_callbacks(), Ryml2Evt, 1); + _RYML_CB_CHECK(get_callbacks(), ryml2evt != nullptr); + new ((void*)ryml2evt) Ryml2Evt(); + return ryml2evt; +} + +RYML_EXPORT void ys2evt_destroy(Ryml2Evt *ryml2evt) +{ + TIMED_SECTION("ys2evt_destroy"); + ryml2evt->~Ryml2Evt(); + _RYML_CB_FREE(get_callbacks(), ryml2evt, Ryml2Evt, 1); +} + +RYML_EXPORT size_type ys2evt_parse(Ryml2Evt *ryml2evt, + const char *filename, + char *ys, size_type ys_size, + evt::DataType *events, size_type evt_size) +{ + TIMED_SECTION("ys2evt_parse"); + csubstr filename_ = filename ? to_csubstr(filename) : csubstr{}; + substr ys_(ys, (size_t)ys_size); + { + TIMED_SECTION("reset + reserve"); + ryml2evt->reset(ys_, events, evt_size); + ryml2evt->m_handler.reserve(256u); + } + { + TIMED_SECTION("parse_in_place"); + ryml2evt->m_parser.parse_in_place_ev(filename_, ys_); + } + return (size_type)ryml2evt->m_handler.m_evt_curr; +} + +#if defined(__cplusplus) +} +#endif diff --git a/rapidyaml/native/ysparse_evt.hpp b/rapidyaml/native/ysparse_evt.hpp new file mode 100644 index 00000000..7712f489 --- /dev/null +++ b/rapidyaml/native/ysparse_evt.hpp @@ -0,0 +1,108 @@ +#pragma once +#ifndef YSPARSE_EVT_HPP_ +#define YSPARSE_EVT_HPP_ + +#include +#include "ysparse_evt_handler.hpp" + +namespace ryml { +using namespace c4; +using namespace c4::yml; +} // namespace ryml + +#if defined(__cplusplus) +extern "C" { +#endif + +using size_type = int; + +struct RYML_EXPORT Ryml2Evt +{ + ys::EventHandlerEvt m_handler; + c4::yml::ParseEngine m_parser; + Ryml2Evt() + : m_handler() + , m_parser(&m_handler) + { + RYML_CHECK(m_parser.options().scalar_filtering()); + } + void reset(c4::csubstr src, evt::DataType *evt, int32_t evt_size) + { + m_handler.reset(src, evt, evt_size); + } +}; + +struct RYML_EXPORT Ryml2EvtParseError : public std::exception +{ + c4::yml::Location location; + std::string msg; + const char* what() const noexcept override { return msg.c_str(); } +}; + + +//----------------------------------------------------------------------------- + +/** Initialize the resources */ +RYML_EXPORT Ryml2Evt *ys2evt_init(); + +/** Destroy the resources */ +RYML_EXPORT void ys2evt_destroy(Ryml2Evt *ryml2evt); + +/** Parse YAML in the string `ys` of size `ys_size`, and write the + * result into the array of (integer) events `evt` of size + * `evt_size`. Each event is encoded as a mask of evt::EventFlags + * (note that it uses the integer evt::DataType as the underlying + * type), and when an event has an associated string, it is followed + * in the array by two extra values, which encode the offset and the + * length of the string in the `ys` string. The `ys` string is mutated + * during parsing. + * + * @return the size needed for `evt`. The caller must check if the + * returned size is larger than `evt_size`. If so, this means that + * `evt` could not accomodate all events produced from `ys`, and is + * incomplete. The caller must then (1) resize `evt` to at least the + * return value, (2) re-copy the original YS into `ys` and (3) call + * again this function, passing in the resized `evt` and the fresh + * copy in `ys`. + * + * @note nothing is written beyond `evt_size`. This means that when + * `evt_size` is 0, then `evt` can be null. This function can be + * safely called for any valid pair of `evt` and `evt_size`, and will + * always return the same required size. + * + * For example, the YAML `say: 2 + 2` produces the following sequence of + * 12 integers: + * + * ```c++ + * BSTR, + * BDOC, + * VAL|BMAP|BLCK, + * KEY|SCLR|PLAI, 0, 3, // "say" + * VAL|SCLR|PLAI, 5, 5, // "2 + 2" + * EMAP, + * EDOC, + * ESTR, + * ``` + * + * Note that the scalar events, ie "say" and "2 + 2", are followed + * each by two extra integers encoding the offset and length of the + * scalar's string. These two extra integers are present whenever the + * event has any of the bits `SCLR`, `ALIA`, `ANCH` or `TAG`. For ease + * of use, there is a bitmask `HAS_STR`, which enables quick testing + * by a simple `flags & HAS_STR`. Refer to evt::EventFlags for the + * full list of flags and their meaning. + * + * Also, where a string requires filtering, the parser filters it + * in-place in the input string, and the extra integers will pertain + * to the resulting filtered string. + */ +RYML_EXPORT size_type ys2evt_parse(Ryml2Evt *ryml2evt, + const char *filename, + char *ys, size_type ys_size, + evt::DataType *evt, size_type evt_size); + +#if defined(__cplusplus) +} +#endif + +#endif /* YSPARSE_EVT_HPP_ */ diff --git a/rapidyaml/native/ysparse_evt_handler.cpp b/rapidyaml/native/ysparse_evt_handler.cpp new file mode 100644 index 00000000..24e5fd66 --- /dev/null +++ b/rapidyaml/native/ysparse_evt_handler.cpp @@ -0,0 +1,12 @@ +#include "ysparse_evt_handler.hpp" +#include +#include + +namespace c4 { +namespace yml { + +// instantiate the template +template class ParseEngine; + +} // namespace yml +} // namespace c4 diff --git a/rapidyaml/native/ysparse_evt_handler.hpp b/rapidyaml/native/ysparse_evt_handler.hpp new file mode 100644 index 00000000..f00d7fe5 --- /dev/null +++ b/rapidyaml/native/ysparse_evt_handler.hpp @@ -0,0 +1,680 @@ +#ifndef _YSPARSE_EVT_HANDLER_HPP_ +#define _YSPARSE_EVT_HANDLER_HPP_ + +#include +#include +#include +#include +#include +#include + +C4_SUPPRESS_WARNING_GCC_CLANG_PUSH +C4_SUPPRESS_WARNING_GCC_CLANG("-Wold-style-cast") +C4_SUPPRESS_WARNING_GCC("-Wuseless-cast") + +namespace evt { +using DataType = int32_t; +typedef enum : DataType { + // --------------------- + // structure flags + KEY_ = 1 << 0, // as key + VAL_ = 1 << 1, // as value + SCLR = 1 << 2, // =VAL + BSEQ = 1 << 3, // +SEQ + ESEQ = 1 << 4, // -SEQ + BMAP = 1 << 5, // +MAP + EMAP = 1 << 6, // -MAP + ALIA = 1 << 7, // ref + ANCH = 1 << 8, // anchor + TAG_ = 1 << 9, // tag + // --------------------- + // style flags + PLAI = 1 << 10, // : (plain scalar) + SQUO = 1 << 11, // ' (single-quoted scalar) + DQUO = 1 << 12, // " (double-quoted scalar) + LITL = 1 << 13, // | (block literal scalar) + FOLD = 1 << 14, // > (block folded scalar) + FLOW = 1 << 15, // flow container: [] for seqs or {} for maps + BLCK = 1 << 16, // block container + // --------------------- + // document flags + BDOC = 1 << 17, // +DOC + EDOC = 1 << 18, // -DOC + EXPL = 1 << 21, // --- (with BDOC) or ... (with EDOC) (may be fused with FLOW if needed) + BSTR = 1 << 19, // +STR + ESTR = 1 << 20, // -STR + // --------------------- + // utility flags + LAST = EXPL, + MASK = (LAST << 1) - 1, + HAS_STR = SCLR|ALIA|ANCH|TAG_ // the event requires a string. the next two integers will provide respectively the string's offset and length +} EventFlags; +} // namespace evt + + +namespace ys { + +using c4::csubstr; +using c4::substr; +using c4::to_substr; +using c4::to_csubstr; +#ifdef RYML_DBG +using c4::_dbg_printf; +#endif + +struct EventHandlerEvtState : public c4::yml::ParserState +{ + c4::yml::type_bits evt_type; + int32_t evt_id; +}; + + +struct EventHandlerEvt : public c4::yml::EventHandlerStack +{ + + /** @name types + * @{ */ + + // our internal state must inherit from parser state + using state = EventHandlerEvtState; + + /** @} */ + +public: + + /** @cond dev */ + csubstr m_str; + evt::DataType * m_evt; + int32_t m_evt_curr; + int32_t m_evt_prev; + int32_t m_evt_size; + char m_key_tag_buf[256]; + char m_val_tag_buf[256]; + std::string m_arena; + + // undefined at the end + #define _enable_(bits) _enable__() + #define _disable_(bits) _disable__() + #define _has_any_(bits) _has_any__() + /** @endcond */ + +public: + + /** @name construction and resetting + * @{ */ + + EventHandlerEvt(c4::yml::Callbacks const& cb) + : EventHandlerStack(cb) + { + reset({}, nullptr, 0); + } + EventHandlerEvt() + : EventHandlerEvt(c4::yml::get_callbacks()) + { + } + + void reset(csubstr str, evt::DataType *dst, int32_t dst_size) + { + _stack_reset_root(); + m_curr->flags |= c4::yml::RUNK|c4::yml::RTOP; + m_curr->evt_type = {}; + m_curr->evt_id = 0; + m_arena.clear(); + m_str = str; + m_evt = dst; + m_evt_size = dst_size; + m_evt_curr = 0; + m_evt_prev = 0; + } + + void reserve(int arena_size) + { + m_arena.reserve(arena_size); + } + + /** @} */ + +public: + + /** @name parse events + * @{ */ + + void start_parse(const char* filename, c4::yml::detail::pfn_relocate_arena relocate_arena, void *relocate_arena_data) + { + this->_stack_start_parse(filename, relocate_arena, relocate_arena_data); + } + + void finish_parse() + { + this->_stack_finish_parse(); + } + + void cancel_parse() + { + while(m_stack.size() > 1) + _pop(); + } + + /** @} */ + +public: + + /** @name YAML stream events */ + /** @{ */ + + void begin_stream() + { + _send_flag_only_(evt::BSTR); + } + + void end_stream() + { + _send_flag_only_(evt::ESTR); + } + + /** @} */ + +public: + + /** @name YAML document events */ + /** @{ */ + + /** implicit doc start (without ---) */ + void begin_doc() + { + _c4dbgpf("{}/{}: begin_doc", m_evt_curr, m_evt_size); + _send_flag_only_(evt::BDOC); + if(_stack_should_push_on_begin_doc()) + { + _c4dbgp("push!"); + _push(); + } + } + /** implicit doc end (without ...) */ + void end_doc() + { + _c4dbgpf("{}/{}: end_doc", m_evt_curr, m_evt_size); + _send_flag_only_(evt::EDOC); + if(_stack_should_pop_on_end_doc()) + { + _c4dbgp("pop!"); + _pop(); + } + } + + /** explicit doc start, with --- */ + void begin_doc_expl() + { + _c4dbgpf("{}/{}: begin_doc_expl", m_evt_curr, m_evt_size); + _send_flag_only_(evt::BDOC|evt::EXPL); + if(_stack_should_push_on_begin_doc()) + { + _c4dbgp("push!"); + _push(); + } + } + /** explicit doc end, with ... */ + void end_doc_expl() + { + _c4dbgpf("{}/{}: end_doc_expl", m_evt_curr, m_evt_size); + _send_flag_only_(evt::EDOC|evt::EXPL); + if(_stack_should_pop_on_end_doc()) + { + _c4dbgp("pop!"); + _pop(); + } + } + + /** @} */ + +public: + + /** @name YAML map functions */ + /** @{ */ + + void begin_map_key_flow() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + void begin_map_key_block() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + + void begin_map_val_flow() + { + _c4dbgpf("{}/{}: bmap flow", m_evt_curr, m_evt_size); + _send_flag_only_(evt::VAL_|evt::BMAP|evt::FLOW); + _mark_parent_with_children_(); + _enable_(c4::yml::MAP|c4::yml::FLOW_SL); + _push(); + } + void begin_map_val_block() + { + _c4dbgpf("{}/{}: bmap block", m_evt_curr, m_evt_size); + _send_flag_only_(evt::VAL_|evt::BMAP|evt::BLCK); + _mark_parent_with_children_(); + _enable_(c4::yml::MAP|c4::yml::BLOCK); + _push(); + } + + void end_map() + { + _pop(); + _send_flag_only_(evt::EMAP); + } + + /** @} */ + +public: + + /** @name YAML seq events */ + /** @{ */ + + void begin_seq_key_flow() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + void begin_seq_key_block() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + + void begin_seq_val_flow() + { + _c4dbgpf("{}/{}: bseq flow", m_evt_curr, m_evt_size); + _send_flag_only_(evt::VAL_|evt::BSEQ|evt::FLOW); + _mark_parent_with_children_(); + _enable_(c4::yml::SEQ|c4::yml::FLOW_SL); + _push(); + } + void begin_seq_val_block() + { + _c4dbgpf("{}/{}: bseq block", m_evt_curr, m_evt_size); + _send_flag_only_(evt::VAL_|evt::BSEQ|evt::BLCK); + _mark_parent_with_children_(); + _enable_(c4::yml::SEQ|c4::yml::BLOCK); + _push(); + } + + void end_seq() + { + _pop(); + _send_flag_only_(evt::ESEQ); + } + + /** @} */ + +public: + + /** @name YAML structure events */ + /** @{ */ + + void add_sibling() + { + _RYML_CB_ASSERT(m_stack.m_callbacks, m_parent); + m_curr->evt_type = {}; + } + + /** set the previous val as the first key of a new map, with flow style. + * + * See the documentation for @ref doc_event_handlers, which has + * important notes about this event. + */ + void actually_val_is_first_key_of_new_map_flow() + { + _RYML_CB_ASSERT(m_stack.m_callbacks, m_evt_curr > 2); + _RYML_CB_ASSERT(m_stack.m_callbacks, m_evt_prev > 0); + _c4dbgpf("{}/{}: prev={} actually_val_is_first_key_of_new_map_flow", m_evt_curr, m_evt_size, m_evt_prev); + // BEFORE + // ... flag start len (free) + // | | + // prev curr + // AFTER + // ... flag flag start len (free) + // | | + // prev curr + if(m_evt_prev < m_evt_size) + { + _RYML_CB_ASSERT(m_stack.m_callbacks, (m_evt[m_evt_prev] & evt::HAS_STR) || m_evt_curr >= m_evt_size); + if(m_evt_curr < m_evt_size) + { + // watchout: it must be in this order! + m_evt[m_evt_curr ] = m_evt[m_evt_prev + 2]; + m_evt[m_evt_curr - 1] = m_evt[m_evt_prev + 1]; + m_evt[m_evt_curr - 2] = m_evt[m_evt_prev] | evt::KEY_; + m_evt[m_evt_curr - 2] &= ~evt::VAL_; + } + m_evt[m_evt_prev] = evt::BMAP|evt::FLOW|evt::VAL_; + } + m_curr->evt_id = m_evt_curr - 2; + ++m_evt_prev; + ++m_evt_curr; + _enable_(c4::yml::MAP|c4::yml::FLOW); + _push(); + } + + void actually_val_is_first_key_of_new_map_block() + { + _RYML_CB_ERR(m_stack.m_callbacks, "container keys not supported"); + } + + /** @} */ + +public: + + /** @name YAML scalar events */ + /** @{ */ + + + C4_ALWAYS_INLINE void set_key_scalar_plain(csubstr scalar) + { + _c4dbgpf("{}/{}: set_key_scalar_plain: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_key_scalar_(scalar, evt::PLAI); + _enable_(c4::yml::KEY|c4::yml::KEY_PLAIN); + } + C4_ALWAYS_INLINE void set_val_scalar_plain(csubstr scalar) + { + _c4dbgpf("{}/{}: set_val_scalar_plain: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_val_scalar_(scalar, evt::PLAI); + _enable_(c4::yml::VAL|c4::yml::VAL_PLAIN); + } + + + C4_ALWAYS_INLINE void set_key_scalar_dquoted(csubstr scalar) + { + _c4dbgpf("{}/{}: set_key_scalar_dquo: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_key_scalar_(scalar, evt::DQUO); + _enable_(c4::yml::KEY|c4::yml::KEY_DQUO); + } + C4_ALWAYS_INLINE void set_val_scalar_dquoted(csubstr scalar) + { + _c4dbgpf("{}/{}: set_val_scalar_dquo: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_val_scalar_(scalar, evt::DQUO); + _enable_(c4::yml::VAL|c4::yml::VAL_DQUO); + } + + + C4_ALWAYS_INLINE void set_key_scalar_squoted(csubstr scalar) + { + _c4dbgpf("{}/{}: set_key_scalar_squo: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_key_scalar_(scalar, evt::SQUO); + _enable_(c4::yml::KEY|c4::yml::KEY_SQUO); + } + C4_ALWAYS_INLINE void set_val_scalar_squoted(csubstr scalar) + { + _c4dbgpf("{}/{}: set_val_scalar_squo: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_val_scalar_(scalar, evt::SQUO); + _enable_(c4::yml::VAL|c4::yml::VAL_SQUO); + } + + + C4_ALWAYS_INLINE void set_key_scalar_literal(csubstr scalar) + { + _c4dbgpf("{}/{}: set_key_scalar_literal: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_key_scalar_(scalar, evt::LITL); + _enable_(c4::yml::KEY|c4::yml::KEY_LITERAL); + } + C4_ALWAYS_INLINE void set_val_scalar_literal(csubstr scalar) + { + _c4dbgpf("{}/{}: set_val_scalar_literal: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_val_scalar_(scalar, evt::LITL); + _enable_(c4::yml::VAL|c4::yml::VAL_LITERAL); + } + + + C4_ALWAYS_INLINE void set_key_scalar_folded(csubstr scalar) + { + _c4dbgpf("{}/{}: set_key_scalar_folded: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_key_scalar_(scalar, evt::FOLD); + _enable_(c4::yml::KEY|c4::yml::KEY_FOLDED); + } + C4_ALWAYS_INLINE void set_val_scalar_folded(csubstr scalar) + { + _c4dbgpf("{}/{}: set_val_scalar_folded: @{} [{}]~~~{}~~~", m_evt_curr, m_evt_size, scalar.str-m_str.str, scalar.len, scalar); + _send_val_scalar_(scalar, evt::FOLD); + _enable_(c4::yml::VAL|c4::yml::VAL_FOLDED); + } + + + C4_ALWAYS_INLINE void mark_key_scalar_unfiltered() + { + _RYML_CB_ERR(m_stack.m_callbacks, "all scalars must be filtered"); + } + C4_ALWAYS_INLINE void mark_val_scalar_unfiltered() + { + _RYML_CB_ERR(m_stack.m_callbacks, "all scalars must be filtered"); + } + + /** @} */ + +public: + +#define _add_scalar_(i, scalar) \ + _c4dbgpf("{}/{}: scalar!", i, m_evt_size); \ + _RYML_CB_ASSERT(m_stack.m_callbacks, scalar.is_sub(m_str)); \ + _RYML_CB_ASSERT(m_stack.m_callbacks, m_evt[i] & evt::HAS_STR); \ + _RYML_CB_ASSERT(m_stack.m_callbacks, i + 2 < m_evt_size); \ + m_evt[i + 1] = (evt::DataType)(scalar.str - m_str.str); \ + m_evt[i + 2] = (evt::DataType)scalar.len + + /** @name YAML anchor/reference events */ + /** @{ */ + + void set_key_anchor(csubstr anchor) + { + _c4dbgpf("{}/{}: set_key_anchor", m_evt_curr, m_evt_size); + _enable_(c4::yml::KEYANCH); + if(m_evt_curr + 2 < m_evt_size) + { + m_evt[m_evt_curr] = evt::KEY_|evt::ANCH; + _add_scalar_(m_evt_curr, anchor); + } + m_evt_prev = m_evt_curr; + m_evt_curr += 3; + } + void set_val_anchor(csubstr anchor) + { + _c4dbgpf("{}/{}: set_val_anchor", m_evt_curr, m_evt_size); + _enable_(c4::yml::VALANCH); + if(m_evt_curr + 2 < m_evt_size) + { + m_evt[m_evt_curr] = evt::VAL_|evt::ANCH; + _add_scalar_(m_evt_curr, anchor); + } + m_evt_prev = m_evt_curr; + m_evt_curr += 3; + } + + void set_key_ref(csubstr ref) + { + _RYML_CB_ASSERT(m_stack.m_callbacks, ref.begins_with('*')); + _enable_(c4::yml::KEY|c4::yml::KEYREF); + _send_key_scalar_(ref, evt::KEY_|evt::ALIA); + } + void set_val_ref(csubstr ref) + { + _RYML_CB_ASSERT(m_stack.m_callbacks, ref.begins_with('*')); + _enable_(c4::yml::VAL|c4::yml::VALREF); + _send_val_scalar_(ref, evt::VAL_|evt::ALIA); + } + + /** @} */ + +public: + + /** @name YAML tag events */ + /** @{ */ + + void set_key_tag(csubstr tag) + { + _enable_(c4::yml::KEYTAG); + csubstr ttag = _transform_directive(tag, m_key_tag_buf); + _RYML_CB_ASSERT(m_stack.m_callbacks, !ttag.empty()); + if(ttag.begins_with('!')) + ttag = ttag.sub(1); + if(m_evt_curr + 2 < m_evt_size) + { + m_evt[m_evt_curr] = evt::KEY_|evt::TAG_; + _add_scalar_(m_evt_curr, ttag); + } + m_evt_prev = m_evt_curr; + m_evt_curr += 3; + } + void set_val_tag(csubstr tag) + { + _enable_(c4::yml::VALTAG); + csubstr ttag = _transform_directive(tag, m_val_tag_buf); + _RYML_CB_ASSERT(m_stack.m_callbacks, !ttag.empty()); + if(ttag.begins_with('!')) + ttag = ttag.sub(1); + if(m_evt_curr + 2 < m_evt_size) + { + m_evt[m_evt_curr] = evt::VAL_|evt::TAG_; + _add_scalar_(m_evt_curr, ttag); + } + m_evt_prev = m_evt_curr; + m_evt_curr += 3; + } + + /** @} */ + +public: + + /** @name YAML directive events */ + /** @{ */ + + void add_directive(csubstr directive) + { + _RYML_CB_ERR(m_stack.m_callbacks, "tag directives not supported"); + } + + /** @} */ + +public: + + /** @name YAML arena events */ + /** @{ */ + + substr alloc_arena(size_t len) + { + const size_t sz = m_arena.size(); + csubstr prev = to_csubstr(m_arena); + m_arena.resize(sz + len); + substr out = to_substr(m_arena).sub(sz); + substr curr = to_substr(m_arena); + if(curr.str != prev.str) + _stack_relocate_to_new_arena(prev, curr); + return out; + } + + substr alloc_arena(size_t len, substr *relocated) + { + csubstr prev = to_csubstr(m_arena); + if(!prev.is_super(*relocated)) + return alloc_arena(len); + substr out = alloc_arena(len); + substr curr = to_substr(m_arena); + if(curr.str != prev.str) + *relocated = _stack_relocate_to_new_arena(*relocated, prev, curr); + return out; + } + + /** @} */ + +public: + + /** push a new parent, add a child to the new parent, and set the + * child as the current node */ + void _push() + { + _stack_push(); + m_curr->evt_type = {}; + } + + /** end the current scope */ + void _pop() + { + _stack_pop(); + } + + template C4_ALWAYS_INLINE void _enable__() noexcept + { + m_curr->evt_type |= bits; + } + template C4_ALWAYS_INLINE void _disable__() noexcept + { + m_curr->evt_type &= ~bits; + } + template C4_ALWAYS_INLINE bool _has_any__() const noexcept + { + return (m_curr->evt_type & bits) != c4::yml::type_bits(0); + } + + void _mark_parent_with_children_() + { + if(m_parent) + m_parent->has_children = true; + } + + C4_ALWAYS_INLINE void _send_flag_only_(evt::DataType flags) + { + _c4dbgpf("{}/{}: flag only", m_evt_curr, m_evt_size); + if(m_evt_curr < m_evt_size) + m_evt[m_evt_curr] = flags; + m_curr->evt_id = m_evt_curr; + m_evt_prev = m_evt_curr; + ++m_evt_curr; + } + + C4_ALWAYS_INLINE void _send_key_scalar_(csubstr scalar, evt::DataType flags) + { + _c4dbgpf("{}/{}: key scalar", m_evt_curr, m_evt_size); + if(m_evt_curr + 2 < m_evt_size) + { + m_evt[m_evt_curr] = evt::SCLR|evt::KEY_|flags; + _add_scalar_(m_evt_curr, scalar); + } + m_curr->evt_id = m_evt_curr; + m_evt_prev = m_evt_curr; + m_evt_curr += 3; + } + + C4_ALWAYS_INLINE void _send_val_scalar_(csubstr scalar, evt::DataType flags) + { + _c4dbgpf("{}/{}: val scalar", m_evt_curr, m_evt_size); + if(m_evt_curr + 2 < m_evt_size) + { + m_evt[m_evt_curr] = evt::SCLR|evt::VAL_|flags; + _add_scalar_(m_evt_curr, scalar); + } + m_curr->evt_id = m_evt_curr; + m_evt_prev = m_evt_curr; + m_evt_curr += 3; + } + + csubstr _transform_directive(csubstr tag, substr output) + { + if(tag.begins_with('!')) + { + if(c4::yml::is_custom_tag(tag)) + { + _RYML_CB_ERR_(m_stack.m_callbacks, "tag not found", m_curr->pos); + } + } + csubstr result = c4::yml::normalize_tag_long(tag, output); + _RYML_CB_CHECK(m_stack.m_callbacks, result.len > 0); + _RYML_CB_CHECK(m_stack.m_callbacks, result.str); + return result; + } +#undef _enable_ +#undef _disable_ +#undef _has_any_ + +}; + +} // namespace ys + +C4_SUPPRESS_WARNING_GCC_POP + +#endif /* _C4_YML_EVENT_HANDLER_EVT_HPP_ */ diff --git a/rapidyaml/native/ysparse_test.cpp b/rapidyaml/native/ysparse_test.cpp new file mode 100644 index 00000000..40a7e5a7 --- /dev/null +++ b/rapidyaml/native/ysparse_test.cpp @@ -0,0 +1,675 @@ +#include +#include +#include + +using c4::csubstr; +using c4::substr; + +namespace c4 +{ +template<> +c4::EnumSymbols const esyms() +{ + static constexpr typename c4::EnumSymbols::Sym syms[] = { + {evt::KEY_, "KEY_"}, + {evt::VAL_, "VAL_"}, + {evt::SCLR, "SCLR"}, + {evt::BSEQ, "BSEQ"}, + {evt::ESEQ, "ESEQ"}, + {evt::BMAP, "BMAP"}, + {evt::EMAP, "EMAP"}, + {evt::ALIA, "ALIA"}, + {evt::ANCH, "ANCH"}, + {evt::TAG_, "TAG_"}, + {evt::PLAI, "PLAI"}, + {evt::SQUO, "SQUO"}, + {evt::DQUO, "DQUO"}, + {evt::LITL, "LITL"}, + {evt::FOLD, "FOLD"}, + {evt::FLOW, "FLOW"}, + {evt::BLCK, "BLCK"}, + {evt::BDOC, "BDOC"}, + {evt::EDOC, "EDOC"}, + {evt::BSTR, "BSTR"}, + {evt::ESTR, "ESTR"}, + {evt::EXPL, "EXPL"}, + }; + return c4::EnumSymbols(syms); +} +} + + +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- + +struct Ys2EdnScoped +{ + Ryml2Edn *ryml2edn; + Ys2EdnScoped() : ryml2edn(ys2edn_init()) {} + ~Ys2EdnScoped() { if(ryml2edn) ys2edn_destroy(ryml2edn); } +}; +struct Ys2EvtScoped +{ + Ryml2Evt *ryml2evt; + Ys2EvtScoped() : ryml2evt(ys2evt_init()) {} + ~Ys2EvtScoped() { if(ryml2evt) ys2evt_destroy(ryml2evt); } +}; + + +struct TestResult +{ + uint32_t num_assertions; + uint32_t num_tests; + uint32_t num_failed_assertions; + uint32_t num_failed_tests; + operator bool() const { return num_failed_tests == 0 && num_failed_assertions == 0; } + void add(TestResult const& that) + { + num_tests += 1 + that.num_tests; + num_assertions += that.num_assertions; + num_failed_tests += (that.num_failed_assertions > 0) + that.num_failed_tests; + num_failed_assertions += that.num_failed_assertions; + } +}; + +// provide a structured input for the events, grouping the relevant +// data in a single structure +struct EvtWithScalar +{ + evt::DataType flags, str_start, str_len; + csubstr scalar; + bool needs_filter; + EvtWithScalar(evt::DataType t, evt::DataType start=0, evt::DataType len=0, csubstr sclr={}, bool needs_filter_=false) + { + flags = t; + str_start = start; + str_len = len; + scalar = sclr; + needs_filter = needs_filter_; + } + size_t required_size() const { return (flags & evt::HAS_STR) ? 3u : 1u; } +}; + +size_t expected_size(std::vector const& evt) +{ + size_t exp = 0; + for(EvtWithScalar const& e : evt) + exp += e.required_size(); + return exp; +} + +struct TestCase +{ + csubstr ys; + csubstr edn; + std::vector evt; + +public: + + #define _runtest(name, ...) \ + do { \ + printf("[ RUN ] %s ... \n", #name); \ + TestResult tr_ = name(__VA_ARGS__); \ + tr.add(tr_); \ + printf("[ %s ] %s\n", tr_?"OK ":"FAIL", #name); \ + } while(0) + #define CHECK(cond) \ + do { \ + bool pass = !!(cond); \ + ++tr.num_assertions; \ + if(!pass) { \ + printf("%s:%d: fail! %s\n", __FILE__, __LINE__, #cond); \ + ++tr.num_failed_assertions; \ + } \ + } while(0) + #define CHECK_MSG(cond, fmt, ...) \ + do { \ + bool pass = !!(cond); \ + ++tr.num_assertions; \ + if(!pass) { \ + printf("%s:%d: fail! %s:" fmt "\n", __FILE__, __LINE__, #cond, ## __VA_ARGS__); \ + ++tr.num_failed_assertions; \ + } \ + } while(0) + + TestResult test(Ryml2Edn *ryml2edn, Ryml2Evt *ryml2evt) const + { + TestResult tr = {}; + _runtest(test_edn_large_enough, ); + _runtest(test_edn_too_small, ); + _runtest(test_edn_nullptr, ); + _runtest(test_edn_large_enough_reuse, ryml2edn); + _runtest(test_edn_too_small_reuse, ryml2edn); + _runtest(test_edn_nullptr_reuse, ryml2edn); + _runtest(test_evt_large_enough, ); + _runtest(test_evt_too_small, ); + _runtest(test_evt_nullptr, ); + _runtest(test_evt_large_enough_reuse, ryml2evt); + _runtest(test_evt_too_small_reuse, ryml2evt); + _runtest(test_evt_nullptr_reuse, ryml2evt); + return tr; + } + + // happy path: large-enough destination string + TestResult test_edn_large_enough_reuse(Ryml2Edn *ryml2edn) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + std::string output; + output.resize(2 * edn.len); + size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", + input.str, (size_type)input.len, + &output[0], (size_type)output.size()); + CHECK(reqsize == edn.len+1); + CHECK(reqsize != 0); + output.resize(reqsize - 1u); + CHECK(testeq(c4::to_csubstr(output))); + return tr; + } + TestResult test_evt_large_enough_reuse(Ryml2Evt *ryml2evt) const + { + if(evt.empty()) return {}; + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + std::vector output; + output.resize(2 * expected_size(evt)); + size_type reqsize = ys2evt_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + &output[0], (size_type)output.size()); + CHECK_MSG((size_t)reqsize == expected_size(evt), "%d vs %zu", reqsize, expected_size(evt)); + CHECK(reqsize != 0); + output.resize(reqsize); + CHECK(testeq(output, input)); + return tr; + } + TestResult test_edn_large_enough() const + { + Ys2EdnScoped lib; + return test_edn_large_enough_reuse(lib.ryml2edn); + } + TestResult test_evt_large_enough() const + { + Ys2EvtScoped lib; + return test_evt_large_enough_reuse(lib.ryml2evt); + } + + // less-happy path: destination string not large enough + TestResult test_edn_too_small_reuse(Ryml2Edn *ryml2edn) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + std::string output = "?"; + size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", + input.str, (size_type)input.len, + output.data(), (size_type)output.size()); + CHECK(reqsize == edn.len+1); + CHECK(reqsize != 0); + CHECK(output != "?"); + output.resize(reqsize); + size_type getsize = ys2edn_retry_get(ryml2edn, &output[0], (size_type)output.size()); + CHECK(getsize == reqsize); + output.resize(reqsize - 1u); + CHECK(testeq(c4::to_csubstr(output))); + return tr; + } + TestResult test_evt_too_small_reuse(Ryml2Evt *ryml2evt) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + std::vector output; + output.resize(expected_size(evt)); + size_type reqsize = ys2evt_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + output.data(), (size_type)output.size()); + CHECK(reqsize == expected_size(evt)); + CHECK(reqsize != 0); + output.resize(reqsize); + input_.assign(ys.begin(), ys.end()); // FIXME + input = c4::to_substr(input_); + size_type reqsize2 = ys2evt_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + output.data(), (size_type)output.size()); + CHECK(reqsize2 == reqsize); + output.resize(reqsize2); + CHECK(testeq(output, input)); + return tr; + } + TestResult test_edn_too_small() const + { + Ys2EdnScoped lib; + return test_edn_too_small_reuse(lib.ryml2edn); + } + TestResult test_evt_too_small() const + { + Ys2EvtScoped lib; + return test_evt_too_small_reuse(lib.ryml2evt); + } + + // safe calling with nullptr + TestResult test_edn_nullptr_reuse(Ryml2Edn *ryml2edn) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + size_type reqsize = ys2edn_parse(ryml2edn, "ysfilename", + input.str, (size_type)input.len, + nullptr, 0); + CHECK(reqsize == edn.len+1); + CHECK(reqsize != 0); + return tr; + } + TestResult test_evt_nullptr_reuse(Ryml2Evt *ryml2evt) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + size_type reqsize = ys2evt_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + nullptr, 0); + CHECK(reqsize == expected_size(evt)); + CHECK(reqsize != 0); + std::vector output; + output.resize(reqsize); + input_.assign(ys.begin(), ys.end()); // FIXME + input = c4::to_substr(input_); + size_type reqsize2 = ys2evt_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + output.data(), (size_type)output.size()); + CHECK(reqsize2 == reqsize); + CHECK(reqsize2 == output.size()); + CHECK(testeq(output, input)); + return tr; + } + TestResult test_edn_nullptr() const + { + Ys2EdnScoped lib; + return test_edn_nullptr_reuse(lib.ryml2edn); + } + TestResult test_evt_nullptr() const + { + Ys2EvtScoped lib; + return test_evt_nullptr_reuse(lib.ryml2evt); + } + +public: + + bool testeq(csubstr actual) const + { + const bool status = (actual == edn); + if(!status) + printf("------\n" + "FAIL:\n" + "input:[%zu]~~~%.*s~~~\n" + "expected:[%zu]~~~%.*s~~~\n" + "actual:[%zu]~~~%.*s~~~\n", + ys.len, (int)ys.len, ys.str, + edn.len, (int)edn.len, edn.str, + actual.len, (int)actual.len, actual.str); + return status; + } + + bool testeq(std::vector const& actual, csubstr parsed_source) const + { + int status = true; + size_t num_events_expected = evt.size(); + size_t num_ints_expected = expected_size(evt); + if(actual.size() != num_ints_expected) + { + printf("------\n" + "FAIL:\n" + "input:~~~%.*s~~~\n" + "expected size:~~~%zu~~~\n" + "actual size:~~~%zu~~~\n", + (int)ys.len, ys.str, + num_ints_expected, + actual.size()); + status = false; + } + for(size_t i = 0, ie = 0; ie < num_events_expected; ++ie) + { + if(i >= actual.size()) + { + printf("fail: bad actual size. i=%zu vs %zu=actual.size()=\n", i, actual.size()); + status = false; + break; + } + #define _testcmp(fmt, cmp, ...) \ + printf("status=%d cmp=%d evt=%zu i=%zu: " fmt "\n", status, (cmp), ie, i, ## __VA_ARGS__); \ + status &= (cmp) + char actualbuf[100]; + char expectedbuf[100]; + size_t reqsize_actual = c4::bm2str(actual[i] & evt::MASK, actualbuf, sizeof(actualbuf)); + size_t reqsize_expected = c4::bm2str(evt[ie].flags & evt::MASK, expectedbuf, sizeof(expectedbuf)); + C4_CHECK(reqsize_actual < sizeof(actualbuf)); + C4_CHECK(reqsize_expected < sizeof(expectedbuf)); + _testcmp("exp=%d(%s) vs act=%d(%s)", evt[ie].flags == actual[i], evt[ie].flags, expectedbuf, actual[i], actualbuf); + status &= (evt[ie].flags == actual[i]); + if((evt[ie].flags & evt::HAS_STR) && (actual[i] & evt::HAS_STR)) + { + _testcmp(" exp=%d vs act=%d", evt[ie].str_start == actual[i + 1], evt[ie].str_start, actual[i + 1]); + _testcmp(" exp=%d vs act=%d", evt[ie].str_len == actual[i + 2], evt[ie].str_len, actual[i + 2]); + bool safeactual = (i + 2 < actual.size()) && (actual[i + 1] < (int)parsed_source.len && actual[i + 1] + actual[i + 2] <= (int)parsed_source.len); + bool safeexpected = (evt[ie].str_start < (int)parsed_source.len && evt[ie].str_start + evt[ie].str_len <= (int)parsed_source.len); + _testcmp(" safeactual=%d", safeactual, safeactual); + _testcmp(" safeactual=%d safeexpected=%d", safeactual == safeexpected, safeactual, safeexpected); + if(safeactual && safeexpected) + { + csubstr evtstr = parsed_source.sub((size_t)evt[ie].str_start, (size_t)evt[ie].str_len); + csubstr actualstr = parsed_source.sub((size_t)actual[i + 1], (size_t)actual[i + 2]); + _testcmp(" ref=[%zu]~~~%.*s~~~ vs act=[%zu]~~~%.*s~~~", + evt[ie].scalar == actualstr, + evt[ie].scalar.len, (int)evt[ie].scalar.len, evt[ie].scalar.str, + actualstr.len, (int)actualstr.len, actualstr.str); + if( ! evt[ie].needs_filter) + { + _testcmp(" exp=[%zu]~~~%.*s~~~ vs act=[%zu]~~~%.*s~~~", + evtstr == actualstr, + evtstr.len, (int)evtstr.len, evtstr.str, + actualstr.len, (int)actualstr.len, actualstr.str); + } + } + } + i += (actual[i] & evt::HAS_STR) ? 3 : 1; + } + if(!status) + printf("------\n" + "FAIL:\n" + "input:~~~%.*s~~~\n", + (int)ys.len, ys.str); + return status; + } +}; + + +//----------------------------------------------------------------------------- + +namespace { +// make the declarations shorter +#define tc(ys, edn, ...) {ys, edn, std::vector(__VA_ARGS__)} +#define e(...) EvtWithScalar{__VA_ARGS__} +using namespace evt; +inline constexpr bool needs_filter = true; +const TestCase test_cases[] = { + // case ------------------------------------------------- + tc("a: 1", + R"(( +{:+ "+MAP"} +{:+ "=VAL", := "a"} +{:+ "=VAL", := "1"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)", + { + e(BSTR), + e(BDOC), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 0, 1, "a"), + e(VAL_|SCLR|PLAI, 3, 1, "1"), + e(EMAP), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- + tc("say: 2 + 2", + R"(( +{:+ "+MAP"} +{:+ "=VAL", := "say"} +{:+ "=VAL", := "2 + 2"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)", + { + e(BSTR), + e(BDOC), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 0, 3, "say"), + e(VAL_|SCLR|PLAI, 5, 5, "2 + 2"), + e(EMAP), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- + tc("𝄞: ✅", + R"(( +{:+ "+MAP"} +{:+ "=VAL", := "𝄞"} +{:+ "=VAL", := "✅"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)", + { + e(BSTR), + e(BDOC), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 0, 4, "𝄞"), + e(VAL_|SCLR|PLAI, 6, 3, "✅"), + e(EMAP), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- + tc("[a, b, c]", + R"(( +{:+ "+SEQ", :flow true} +{:+ "=VAL", := "a"} +{:+ "=VAL", := "b"} +{:+ "=VAL", := "c"} +{:+ "-SEQ"} +{:+ "-DOC"} +) +)", + { + e(BSTR), + e(BDOC), + e(VAL_|BSEQ|FLOW), + e(VAL_|SCLR|PLAI, 1, 1, "a"), + e(VAL_|SCLR|PLAI, 4, 1, "b"), + e(VAL_|SCLR|PLAI, 7, 1, "c"), + e(ESEQ), + e(EDOC), + e(ESTR), + }), + // case ------------------------------ + tc("[a: b]", + R"(( +{:+ "+SEQ", :flow true} +{:+ "+MAP", :flow true} +{:+ "=VAL", := "a"} +{:+ "=VAL", := "b"} +{:+ "-MAP"} +{:+ "-SEQ"} +{:+ "-DOC"} +) +)", + { + e(BSTR), + e(BDOC), + e(VAL_|BSEQ|FLOW), + e(VAL_|BMAP|FLOW), + e(KEY_|SCLR|PLAI, 1, 1, "a"), + e(VAL_|SCLR|PLAI, 4, 1, "b"), + e(EMAP), + e(ESEQ), + e(EDOC), + e(ESTR), + }), + // case ------------------------------ + tc(R"(--- !yamlscript/v0 +foo: ! +- {x: y} +- [x, y] +- foo +- 'foo' +- "foo" +- | + foo +- > + foo +- [1, 2, true, false, null] +- &anchor-1 !tag-1 foobar +--- +another: doc +)", + R"(( +{:+ "+MAP", :! "yamlscript/v0"} +{:+ "=VAL", := "foo"} +{:+ "+SEQ", :! ""} +{:+ "+MAP", :flow true} +{:+ "=VAL", := "x"} +{:+ "=VAL", := "y"} +{:+ "-MAP"} +{:+ "+SEQ", :flow true} +{:+ "=VAL", := "x"} +{:+ "=VAL", := "y"} +{:+ "-SEQ"} +{:+ "=VAL", := "foo"} +{:+ "=VAL", :' "foo"} +{:+ "=VAL", :$ "foo"} +{:+ "=VAL", :| "foo\n"} +{:+ "=VAL", :> "foo\n"} +{:+ "+SEQ", :flow true} +{:+ "=VAL", := "1"} +{:+ "=VAL", := "2"} +{:+ "=VAL", := "true"} +{:+ "=VAL", := "false"} +{:+ "=VAL", := "null"} +{:+ "-SEQ"} +{:+ "=VAL", :& "anchor-1", :! "tag-1", := "foobar"} +{:+ "-SEQ"} +{:+ "-MAP"} +{:+ "-DOC"} +{:+ "+DOC"} +{:+ "+MAP"} +{:+ "=VAL", := "another"} +{:+ "=VAL", := "doc"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)", + { + e(BSTR), + e(BDOC|EXPL), + e(VAL_|TAG_, 5, 13, "yamlscript/v0"), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 19, 3, "foo"), + e(VAL_|TAG_, 25, 0, ""), + e(VAL_|BSEQ|BLCK), + e(VAL_|BMAP|FLOW), + e(KEY_|SCLR|PLAI, 29, 1, "x"), + e(VAL_|SCLR|PLAI, 32, 1, "y"), + e(EMAP), + e(VAL_|BSEQ|FLOW), + e(VAL_|SCLR|PLAI, 38, 1, "x"), + e(VAL_|SCLR|PLAI, 41, 1, "y"), + e(ESEQ), + e(VAL_|SCLR|PLAI, 46, 3, "foo"), + e(VAL_|SCLR|SQUO, 53, 3, "foo"), + e(VAL_|SCLR|DQUO, 61, 3, "foo"), + e(VAL_|SCLR|LITL, 70, 4, "foo\n", needs_filter), + e(VAL_|SCLR|FOLD, 80, 4, "foo\n", needs_filter), + e(VAL_|BSEQ|FLOW), + e(VAL_|SCLR|PLAI, 89, 1, "1"), + e(VAL_|SCLR|PLAI, 92, 1, "2"), + e(VAL_|SCLR|PLAI, 95, 4, "true"), + e(VAL_|SCLR|PLAI, 101, 5, "false"), + e(VAL_|SCLR|PLAI, 108, 4, "null"), + e(ESEQ), + e(VAL_|TAG_, 127, 5, "tag-1"), + e(VAL_|ANCH, 117, 8, "anchor-1"), + e(VAL_|SCLR|PLAI, 133, 6, "foobar"), + e(ESEQ), + e(EMAP), + e(EDOC), + e(BDOC|EXPL), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 144, 7, "another"), + e(VAL_|SCLR|PLAI, 153, 3, "doc"), + e(EMAP), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- + tc(R"(plain: well + a + b + c +squo: 'single''quote' +dquo: "x\t\ny" +lit: | + X + Y + Z +fold: > + U + V + W +)", + R"(( +{:+ "+MAP"} +{:+ "=VAL", := "plain"} +{:+ "=VAL", := "well a b c"} +{:+ "=VAL", := "squo"} +{:+ "=VAL", :' "single'quote"} +{:+ "=VAL", := "dquo"} +{:+ "=VAL", :$ "x\t\ny"} +{:+ "=VAL", := "lit"} +{:+ "=VAL", :| "X\nY\nZ\n"} +{:+ "=VAL", := "fold"} +{:+ "=VAL", :> "U V W\n"} +{:+ "-MAP"} +{:+ "-DOC"} +) +)", + { + e(BSTR), + e(BDOC), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 0, 5, "plain"), + e(VAL_|SCLR|PLAI, 7, 10, "well a b c"), + e(KEY_|SCLR|PLAI, 24, 4, "squo"), + e(VAL_|SCLR|SQUO, 31, 12, "single'quote", needs_filter), + e(KEY_|SCLR|PLAI, 46, 4, "dquo"), + e(VAL_|SCLR|DQUO, 53, 4, "x\t\ny", needs_filter), + e(KEY_|SCLR|PLAI, 61, 3, "lit"), + e(VAL_|SCLR|LITL, 68, 6, "X\nY\nZ\n", needs_filter), + e(KEY_|SCLR|PLAI, 89, 4, "fold"), + e(VAL_|SCLR|FOLD, 97, 6, "U V W\n", needs_filter), + e(EMAP), + e(EDOC), + e(ESTR), + }), +}; +} // namespace + +int main() +{ + Ys2EdnScoped ys2edn; + Ys2EvtScoped ys2evt; + TestResult total = {}; + size_t failed_cases = {}; + size_t num_cases = C4_COUNTOF(test_cases); + for(size_t i = 0; i < C4_COUNTOF(test_cases); ++i) + { + printf("-----------------------------------------\n" + "case %zu/%zu ...\n" + "[%zu]~~~%.*s~~~\n", i, num_cases, test_cases[i].ys.len, (int)test_cases[i].ys.len, test_cases[i].ys.str); + const TestResult tr = test_cases[i].test(ys2edn.ryml2edn, ys2evt.ryml2evt); + total.add(tr); + failed_cases += (!tr); + printf("case %zu/%zu: %s\n", i, C4_COUNTOF(test_cases), tr ? "ok!" : "failed"); + } + printf("assertions: %u/%u pass %u/%u fail\n", total.num_assertions - total.num_failed_assertions, total.num_assertions, total.num_failed_assertions, total.num_assertions); + printf("tests: %u/%u pass %u/%u fail\n", total.num_tests - total.num_failed_tests, total.num_tests, total.num_failed_tests, total.num_tests); + printf("cases: %zu/%zu pass %zu/%zu fail\n", num_cases-failed_cases, num_cases, failed_cases, num_cases); + if(total) + printf("TESTS SUCCEED!\n"); + return total ? 0 : -1; +} diff --git a/rapidyaml/src/main/java/org/rapidyaml/Evt.java b/rapidyaml/src/main/java/org/rapidyaml/Evt.java new file mode 100644 index 00000000..4f06211f --- /dev/null +++ b/rapidyaml/src/main/java/org/rapidyaml/Evt.java @@ -0,0 +1,37 @@ +package org.rapidyaml; + +public class Evt { + // --------------------- + // structure flags + public static final int KEY_ = 1 << 0; // as key + public static final int VAL_ = 1 << 1; // as value + public static final int SCLR = 1 << 2; // =VAL + public static final int BSEQ = 1 << 3; // +SEQ + public static final int ESEQ = 1 << 4; // -SEQ + public static final int BMAP = 1 << 5; // +MAP + public static final int EMAP = 1 << 6; // -MAP + public static final int ALIA = 1 << 7; // ref + public static final int ANCH = 1 << 8; // anchor + public static final int TAG_ = 1 << 9; // tag + // --------------------- + // style flags + public static final int PLAI = 1 << 10; // : (plain scalar) + public static final int SQUO = 1 << 11; // ' (single-quoted scalar) + public static final int DQUO = 1 << 12; // " (double-quoted scalar) + public static final int LITL = 1 << 13; // | (block literal scalar) + public static final int FOLD = 1 << 14; // > (block folded scalar) + public static final int FLOW = 1 << 15; // flow container: [] for seqs or {} for maps + public static final int BLCK = 1 << 16; // block container + // --------------------- + // document flags + public static final int BDOC = 1 << 17; // +DOC + public static final int EDOC = 1 << 18; // -DOC + public static final int EXPL = 1 << 21; // --- (with BDOC) or ... (with EDOC) (may be fused with FLOW if needed) + public static final int BSTR = 1 << 19; // +STR + public static final int ESTR = 1 << 20; // -STR + // --------------------- + // utility flags + public static final int LAST = ESTR; + public static final int MASK = ((LAST << 1) - 1); + public static final int HAS_STR = SCLR|ALIA|ANCH|TAG_; +} diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java index ccf7d437..c36eb768 100644 --- a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -6,7 +6,6 @@ /** * Interface with the shared librapidyaml library - * */ public class Rapidyaml { public static String RAPIDYAML_VERSION = "0.7.2"; @@ -19,13 +18,20 @@ private native int ys2edn_parse(long ryml2edn, String filename, private native int ys2edn_retry_get( long ryml2edn, byte[] edn, int edn_size ); - private final long ryml2edn; + private native long ys2evt_init(); + private native void ys2evt_destroy(long ryml2evt); + private native int ys2evt_parse(long ryml2evt, String filename, + byte[] ys, int ys_length, + int[] evt, int evt_length); + private final long ryml2evt; + public Rapidyaml() { String library_name = "rapidyaml"; // ." + RAPIDYAML_VERSION; System.loadLibrary(library_name); this.ryml2edn = this.ys2edn_init(); + this.ryml2evt = this.ys2evt_init(); } // Likely bad idea to implement finalize: @@ -35,13 +41,14 @@ public Rapidyaml() { protected void finalize() throws Throwable { try { this.ys2edn_destroy(this.ryml2edn); + this.ys2evt_destroy(this.ryml2evt); } finally { super.finalize(); } } - public String parseYS(String srcstr) throws RuntimeException, org.rapidyaml.YamlParseErrorException { + public String parseYsToEdn(String srcstr) throws RuntimeException, org.rapidyaml.YamlParseErrorException { String filename = "yamlscript"; // fixme byte[] src = srcstr.getBytes(StandardCharsets.UTF_8); int edn_size = 10 * src.length; @@ -58,4 +65,10 @@ public String parseYS(String srcstr) throws RuntimeException, org.rapidyaml.Yaml String ret = new String(edn, 0, required_size-1, StandardCharsets.UTF_8); return ret; } + + public int parseYsToEvt(byte[] src, int[] evts) throws RuntimeException, org.rapidyaml.YamlParseErrorException { + String filename = "yamlscript"; // fixme + int required_size = ys2evt_parse(this.ryml2evt, filename, src, src.length, evts, evts.length); + return required_size; + } } diff --git a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java index 48e9eaf8..db62cc2b 100644 --- a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java +++ b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java @@ -1,6 +1,8 @@ package org.rapidyaml; import org.rapidyaml.*; +import java.nio.charset.StandardCharsets; +import java.nio.ByteBuffer; import junit.framework.Test; import junit.framework.TestCase; import junit.framework.TestSuite; @@ -32,10 +34,110 @@ private void testEdn_(String ys, String expected) { Rapidyaml rapidyaml = new Rapidyaml(); try { - String actual = rapidyaml.parseYS(ys); + String actual = rapidyaml.parseYsToEdn(ys); try { - assertEquals(expected.length(), actual.length()); assertEquals(expected, actual); + assertEquals(expected.length(), actual.length()); + } + catch (Exception e) { + System.err.println("expected:"); + System.err.println(expected); + System.err.println("actual"); + System.err.println(actual); + throw e; + } + } + catch (YamlParseErrorException e) { + fail("parse error:\n" + e.getMessage()); + } + } + + // the result is an array of integers, but we use this to simplify + // running the tests + private class ExpectedEvent + { + int flags; + int str_start; + int str_len; + String str; + ExpectedEvent(int flags) + { + this.flags = flags; + this.str_start = 0; + this.str_len = 0; + this.str = ""; + } + ExpectedEvent(int flags, int str_start, int str_len, String str) + { + this.flags = flags; + this.str_start = str_start; + this.str_len = str_len; + this.str = str; + } + int required_size() { return ((flags & Evt.HAS_STR) != 0) ? 3 : 1; } + }; + private static int required_size_(ExpectedEvent[] evts) + { + int sz = 0; + for(int i = 0; i < evts.length; ++i) { + sz += evts[i].required_size(); + } + return sz; + } + + private void testEvt_(String ys, ExpectedEvent[] expected) + { + Rapidyaml rapidyaml = new Rapidyaml(); + try { + int[] actual = new int[2 * required_size_(expected)]; + byte[] src = ys.getBytes(StandardCharsets.UTF_8); + int numEvts = rapidyaml.parseYsToEvt(src, actual); + assertTrue(numEvts < actual.length); + try { + int ia = 0; + int ie = 0; + int status = 1; + while(true) { + if((ia < numEvts) != (ie < expected.length)) { + status = 0; + break; + } + if(ia >= numEvts) + break; + if(ie >= expected.length) + break; + int cmp = 1; + System.out.printf("status=%d evt=%d pos=%d expflags=%d actualflags=%d", status, ie, ia, expected[ie].flags, actual[ia]); + cmp &= (expected[ie].flags == actual[ia]) ? 1 : 0; + if(((actual[ia] & Evt.HAS_STR) != 0) && ((expected[ie].flags & Evt.HAS_STR)) != 0) { + cmp &= (ia + 2 < numEvts) ? 1 : 0; + if(cmp != 0) { + cmp &= (expected[ie].str_start == actual[ia + 1]) ? 1 : 0; + cmp &= (expected[ie].str_len == actual[ia + 2]) ? 1 : 0; + System.out.printf(" exp=(%d,%d) actual=(%d,%d)", expected[ie].str_start, expected[ie].str_len, actual[ia + 1], actual[ia + 2]); + if(cmp != 0) { + cmp &= (actual[ia + 1] >= 0) ? 1 : 0; + cmp &= (actual[ia + 2] >= 0) ? 1 : 0; + cmp &= (actual[ia + 1] + actual[ia + 2] <= src.length) ? 1 : 0; + if(cmp != 0) { + String actualStr = new String(src, actual[ia + 1], actual[ia + 2], StandardCharsets.UTF_8); + cmp &= actualStr.equals(expected[ie].str) ? 1 : 0; + System.out.printf(" exp=~~~%s~~~ actual=~~~%s~~~", expected[ie].str, actualStr); + } + else { + System.out.printf(" BAD RANGE len=%d yslen=%d", src.length, ys.length()); + } + } + } + } + System.out.printf(" --> %s\n", cmp != 0 ? "ok!" : "FAIL"); + status &= cmp; + ia += ((actual[ia] & Evt.HAS_STR) != 0) ? 3 : 1; + ++ie; + } + if(required_size_(expected) != numEvts) + status = 0; + assertEquals(1, status); } catch (Exception e) { System.err.println("expected:"); @@ -52,9 +154,8 @@ private void testEdn_(String ys, String expected) public void testPlainMap() { - testEdn_( - "a: 1" - , + String ys = "a: 1"; + testEdn_(ys, "(\n" + "{:+ \"+MAP\"}\n" + "{:+ \"=VAL\", := \"a\"}\n" + @@ -63,13 +164,23 @@ public void testPlainMap() "{:+ \"-DOC\"}\n" + ")\n" ); + ExpectedEvent[] expected = { + new ExpectedEvent(Evt.BSTR), + new ExpectedEvent(Evt.BDOC), + new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 1, "a"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 3, 1, "1"), + new ExpectedEvent(Evt.EMAP), + new ExpectedEvent(Evt.EDOC), + new ExpectedEvent(Evt.ESTR), + }; + testEvt_(ys, expected); } public void testUtf8() { - testEdn_( - "𝄞: ✅" - , + String ys = "𝄞: ✅"; + testEdn_(ys, "(\n" + "{:+ \"+MAP\"}\n" + "{:+ \"=VAL\", := \"𝄞\"}\n" + @@ -78,11 +189,22 @@ public void testUtf8() "{:+ \"-DOC\"}\n" + ")\n" ); + ExpectedEvent[] expected = { + new ExpectedEvent(Evt.BSTR), + new ExpectedEvent(Evt.BDOC), + new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 4, "𝄞"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 6, 3, "✅"), + new ExpectedEvent(Evt.EMAP), + new ExpectedEvent(Evt.EDOC), + new ExpectedEvent(Evt.ESTR), + }; + testEvt_(ys, expected); } public void testLargeCase() { - testEdn_( + String ys = "--- !yamlscript/v0\n" + "foo: !\n" + "- {x: y}\n" + "- [x, y]\n" + @@ -96,10 +218,10 @@ public void testLargeCase() "- [1, 2, true, false, null]\n" + "- &anchor-1 !tag-1 foobar\n" + "---\n" + - "another: doc\n" - , + "another: doc\n"; + testEdn_(ys, "(\n" + - "{:+ \"+MAP\"}\n" + + "{:+ \"+MAP\", :! \"yamlscript/v0\"}\n" + "{:+ \"=VAL\", := \"foo\"}\n" + "{:+ \"+SEQ\", :! \"\"}\n" + "{:+ \"+MAP\", :flow true}\n" + @@ -134,6 +256,103 @@ public void testLargeCase() "{:+ \"-DOC\"}\n" + ")\n" ); + ExpectedEvent[] expected = { + new ExpectedEvent(Evt.BSTR), + new ExpectedEvent(Evt.BDOC|Evt.EXPL), + new ExpectedEvent(Evt.VAL_|Evt.TAG_, 5, 13, "yamlscript/v0"), + new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 19, 3, "foo"), + new ExpectedEvent(Evt.VAL_|Evt.TAG_, 25, 0, ""), + new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.BLCK), + new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.FLOW), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 29, 1, "x"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 32, 1, "y"), + new ExpectedEvent(Evt.EMAP), + new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.FLOW), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 38, 1, "x"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 41, 1, "y"), + new ExpectedEvent(Evt.ESEQ), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 46, 3, "foo"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.SQUO, 53, 3, "foo"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.DQUO, 61, 3, "foo"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.LITL, 70, 4, "foo\n"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.FOLD, 80, 4, "foo\n"), + new ExpectedEvent(Evt.VAL_|Evt.BSEQ|Evt.FLOW), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 89, 1, "1"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 92, 1, "2"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 95, 4, "true"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 101, 5, "false"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 108, 4, "null"), + new ExpectedEvent(Evt.ESEQ), + new ExpectedEvent(Evt.VAL_|Evt.TAG_, 127, 5, "tag-1"), + new ExpectedEvent(Evt.VAL_|Evt.ANCH, 117, 8, "anchor-1"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 133, 6, "foobar"), + new ExpectedEvent(Evt.ESEQ), + new ExpectedEvent(Evt.EMAP), + new ExpectedEvent(Evt.EDOC), + new ExpectedEvent(Evt.BDOC|Evt.EXPL), + new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 144, 7, "another"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 153, 3, "doc"), + new ExpectedEvent(Evt.EMAP), + new ExpectedEvent(Evt.EDOC), + new ExpectedEvent(Evt.ESTR), + }; + testEvt_(ys, expected); + } + + public void testFilterCase() + { + String ys = "" + + "plain: well\n" + + " a\n" + + " b\n" + + " c\n" + + "squo: 'single''quote'\n" + + "dquo: \"x\\t\\ny\"\n" + + "lit: |\n" + + " X\n" + + " Y\n" + + " Z\n" + + "fold: >\n" + + " U\n" + + " V\n" + + " W\n"; + testEdn_(ys, + "(\n" + + "{:+ \"+MAP\"}\n" + + "{:+ \"=VAL\", := \"plain\"}\n" + + "{:+ \"=VAL\", := \"well a b c\"}\n" + + "{:+ \"=VAL\", := \"squo\"}\n" + + "{:+ \"=VAL\", :' \"single'quote\"}\n" + + "{:+ \"=VAL\", := \"dquo\"}\n" + + "{:+ \"=VAL\", :$ \"x\\t\\ny\"}\n" + + "{:+ \"=VAL\", := \"lit\"}\n" + + "{:+ \"=VAL\", :| \"X\\nY\\nZ\\n\"}\n" + + "{:+ \"=VAL\", := \"fold\"}\n" + + "{:+ \"=VAL\", :> \"U V W\\n\"}\n" + + "{:+ \"-MAP\"}\n" + + "{:+ \"-DOC\"}\n" + + ")\n"); + ExpectedEvent[] expected = { + new ExpectedEvent(Evt.BSTR), + new ExpectedEvent(Evt.BDOC), + new ExpectedEvent(Evt.VAL_|Evt.BMAP|Evt.BLCK), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 5, "plain"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.PLAI, 7, 10, "well a b c"), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 24, 4, "squo"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.SQUO, 31, 12, "single'quote"), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 46, 4, "dquo"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.DQUO, 53, 4, "x\t\ny"), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 61, 3, "lit"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.LITL, 68, 6, "X\nY\nZ\n"), + new ExpectedEvent(Evt.KEY_|Evt.SCLR|Evt.PLAI, 89, 4, "fold"), + new ExpectedEvent(Evt.VAL_|Evt.SCLR|Evt.FOLD, 97, 6, "U V W\n"), + new ExpectedEvent(Evt.EMAP), + new ExpectedEvent(Evt.EDOC), + new ExpectedEvent(Evt.ESTR), + }; + testEvt_(ys, expected); } public void testFailure() @@ -142,7 +361,7 @@ public void testFailure() String ys = ": : : :"; boolean gotit = false; try { - rapidyaml.parseYS(ys); + rapidyaml.parseYsToEdn(ys); } catch(YamlParseErrorException e) { gotit = true;