rapidyaml: Add parsing of ys to integer-based events

Adds a parser producing integer events, and integers signifying strings
by indexing into the parsed YS string.

Each event is provided as an integer bitmask. The event bits are
defined in both the C++ side (ysparse_evt_handler.hpp) and the Java
side (org.rapidyaml.Evt), and they need to stay consistent on both
ends. When a string is associated with an event, it is provided as an
integer offset and length.

For example, the YAML `say: 2 + 2` produces the following sequence of
integers:

```c++
BSTR,
BDOC,
VAL|BMAP|BLCK,
KEY|SCLR|PLAI, 0, 3, // "say"
VAL|SCLR|PLAI, 5, 5, // "2 + 2"
EMAP,
EDOC,
ESTR,
```

Note that the scalar events, ie "say" and "2 + 2", are followed each
by two extra integers encoding the offset and length of the scalar's
string. These two extra integers are present whenever the event has
any of the bits SCLR, ALIA, ANCH or TAG. For ease of use, there is a
bitmask HAS_STR, which enables quick testing by a simple `flags & HAS_STR`.

Also, where a string requires filtering, the parser filters it
in-place in the input string, and returns the extra integers
pertaining to the resulting filtered string.

The existing EDN-producing parser was not removed, and the
EVT-producing parser was added both on C++ and on the Java JNI bridge.

Tests were enlarged to cover the new event parsing, both for C++ and
Java.

To benefit from this approach, the YS side must implement a mechanism
to convert the int event sequence into its internal data-structure,
using the symbols in the class org.rapidyaml.Evt.

Other changes:

- rapidyaml->ysparse: start renaming the C++ library providing the
  JNI. ysparse is a more accurate description of what the library does.
- Directly use only the required rapidyaml source files instead of
  amalgamating into a single header.
- Makefile:
  - Add target to generate the JNI header
  - Improve target dependencies

common/vars.mk

@@ -226,6 +226,8 @@ RAPIDYAML := $(ROOT)/rapidyaml
 RAPIDYAML_VERSION := 0.7.2
 RAPIDYAML_TAG ?= v$(RAPIDYAML_VERSION)
 RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml
+RAPIDYAML_BUILD_TYPE := Release
+RAPIDYAML_DBG := 0
 RAPIDYAML_TIMED := 0
 RAPIDYAML_JAVA := \
   $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java \

rapidyaml/Makefile

@@ -10,9 +10,7 @@ RAPIDYAML_JAR_DEPS := \
       pom.xml \
   )
 
-RAPIDYAML_CLASS := \
-  src/main/java/org/rapidyaml/Rapidyaml.class \
-  src/main/java/org/rapidyaml/YamlParseErrorException.class
+RAPIDYAML_CLASS := $(RAPIDYAML_JAVA:.java=.class)
 
 
 #------------------------------------------------------------------------------
@@ -23,7 +21,7 @@ default::
 x:
 	YS -ce 'foo: var'
 
-build:: $(RAPIDYAML_SO) $(RAPIDYAML_LIB)
+build:: $(RAPIDYAML_JNI_H) $(RAPIDYAML_SO) $(RAPIDYAML_LIB)
 	@:
 
 jar: $(RAPIDYAML_JAR)
@@ -51,7 +49,7 @@ sysclean::
 
 
 #------------------------------------------------------------------------------
-$(RAPIDYAML_SO):
+$(RAPIDYAML_SO): $(RAPIDYAML_JNI_H)
 	$(MAKE) -C native $@
 $(RAPIDYAML_LIB):
 	$(MAKE) -C native $@
@@ -63,9 +61,12 @@ $(RAPIDYAML_JAR): $(RAPIDYAML_CLASS) $(JAVA_INSTALLED)
 	jar -cvf $@ $<
 
 $(RAPIDYAML_CLASS): $(RAPIDYAML_JAVA) $(JAVA_INSTALLED)
-	# this doesn't work:
-	#javac $<
-	# ... but this does:
+	@# this doesn't work:
+	@#javac $<
+	@# ... but this does:
 	javac $(RAPIDYAML_JAVA)
 
+$(RAPIDYAML_JNI_H): $(RAPIDYAML_JAVA)
+	$(MAKE) -C native $@
+
 crl: $(RAPIDYAML_CLASS)

rapidyaml/native/.gitignore

@@ -1,4 +1,5 @@
 /rapidyaml/
-/rapidyaml-build/
-/rapidyaml_all.hpp
+/_build/
 /librapidyaml.*
+/.cache
+/compile_commands.json

rapidyaml/native/CMakeLists.txt

@@ -1,45 +1,69 @@
 cmake_minimum_required(VERSION 3.12)
-project(rapidyaml
-    DESCRIPTION "rapidyaml -> yamlscript"
+project(ysparse
+    DESCRIPTION "ysparse: rapidyaml -> yamlscript"
     HOMEPAGE_URL "https://github.com/biojppm/rapidyaml -> https://github.com/yaml/yamlscript"
     LANGUAGES CXX)
 
 find_package(JNI REQUIRED)
 
-option(YS2EDN_TIMED "add timings to sections" OFF)
+option(YSPARSE_TIMED "add timings to sections" OFF)
+option(YSPARSE_DBG "enable debug logs" OFF)
 
 if(UNIX)
     set(CMAKE_SHARED_LIBRARY_SUFFIX .so)
 endif()
 
-add_library(rapidyaml
+set(libname rapidyaml) # TODO rename to ysparse
+
+add_library(${libname}
+    #
+    # JNI bridge
     org_rapidyaml_Rapidyaml.h
     org_rapidyaml_Rapidyaml.cpp
-    rapidyaml_all.hpp
-    rapidyaml_edn_handler.hpp
-    rapidyaml_edn_handler.cpp
-    rapidyaml_edn.hpp
-    rapidyaml_edn.cpp
+    #
+    # ysparse files
+    ysparse_edn_handler.hpp
+    ysparse_edn_handler.cpp
+    ysparse_edn.hpp
+    ysparse_edn.cpp
+    ysparse_evt_handler.hpp
+    ysparse_evt_handler.cpp
+    ysparse_evt.hpp
+    ysparse_evt.cpp
+    #
+    # files required from rapidyaml
+    rapidyaml/src/c4/yml/common.cpp
+    rapidyaml/src/c4/yml/node_type.cpp
+    rapidyaml/src/c4/yml/parse.cpp
+    rapidyaml/src/c4/yml/tree.cpp
+    rapidyaml/src/c4/yml/tag.cpp
+    rapidyaml/src/c4/yml/reference_resolver.cpp
+    #
+    # files required from rapidyaml/ext/c4core
+    rapidyaml/ext/c4core/src/c4/base64.cpp
+    rapidyaml/ext/c4core/src/c4/error.cpp
+    rapidyaml/ext/c4core/src/c4/language.cpp
+    rapidyaml/ext/c4core/src/c4/utf.cpp
 )
-target_include_directories(rapidyaml PUBLIC
+target_include_directories(${libname} PUBLIC
     ${CMAKE_CURRENT_LIST_DIR}
-    ${CMAKE_CURRENT_LIST_DIR}/rapidyaml/test
+    ${CMAKE_CURRENT_LIST_DIR}/rapidyaml/src
+    ${CMAKE_CURRENT_LIST_DIR}/rapidyaml/ext/c4core/src
 )
-target_compile_definitions(rapidyaml PUBLIC
+target_compile_definitions(${libname} PUBLIC
     RYML_WITH_TAB_TOKENS
     RYML_DEFAULT_CALLBACK_USES_EXCEPTIONS
-    RYML_SINGLE_HEADER
-    $<$<BOOL:${YS2EDN_TIMED}>:YS2EDN_TIMED>
+    $<$<BOOL:${YSPARSE_TIMED}>:YSPARSE_TIMED>
+    $<$<BOOL:${YSPARSE_DBG}>:RYML_DBG>
 )
-set_target_properties(rapidyaml PROPERTIES CXX_STANDARD 17)
+set_target_properties(${libname} PROPERTIES CXX_STANDARD 17)
 
-# target_link_libraries(rapidyaml PUBLIC JNI::JNI)
-target_include_directories(rapidyaml PUBLIC ${JNI_INCLUDE_DIRS})
+target_include_directories(${libname} PUBLIC ${JNI_INCLUDE_DIRS})
 
-add_executable(rapidyaml-test rapidyaml_test.cpp)
-target_link_libraries(rapidyaml-test rapidyaml)
-add_custom_target(rapidyaml-test-run
-    DEPENDS rapidyaml-test
-    COMMAND $<TARGET_FILE:rapidyaml-test>
+add_executable(${libname}-test ysparse_test.cpp)
+target_link_libraries(${libname}-test ${libname})
+add_custom_target(${libname}-test-run
+    DEPENDS ${libname}-test
+    COMMAND $<TARGET_FILE:${libname}-test>
     COMMENT "running C++ tests"
 )

rapidyaml/native/Makefile

@@ -3,24 +3,27 @@ include $(COMMON)/clojure.mk
 include $(COMMON)/java.mk
 include $(COMMON)/python.mk
 
-RAPIDYAML_H := org_rapidyaml_Rapidyaml.h
-RAPIDYAML_HPP := rapidyaml_all.hpp
-RAPIDYAML_AMALGAMATE_OPTS := --stl
+# TODO change to static library!
+# https://www.graalvm.org/latest/reference-manual/native-image/guides/build-static-executables/
+# https://www.blog.akhil.cc/static-jni
+# https://stackoverflow.com/questions/24493337/linking-static-library-with-jni
+
+THIS_DIR := $(shell pwd)
+BDIR := $(THIS_DIR)/_build/$(BUILD_TYPE)-shared$(SHARED)-timed$(TIMED)-dbg$(DBG)
 
 RAPIDYAML_DEPS := \
   Makefile \
   CMakeLists.txt \
   $(JAVA_HOME) \
-  $(RAPIDYAML_H) \
-  $(RAPIDYAML_HPP) \
+  $(RAPIDYAML_JNI_H) \
   $(wildcard ./*pp) \
 
-CMK_FLAGS :=
-CMK_FLAGS += -D YS2EDN_TIMED=$(RAPIDYAML_TIMED)
-
-BDIR := rapidyaml-build
-
-CMK_BUILD :=
+CMK_ENV :=
+CMK_FLAGS := \
+  -D CMAKE_BUILD_TYPE=$(RAPIDYAML_BUILD_TYPE) \
+  -D YSPARSE_TIMED=$(RAPIDYAML_TIMED) \
+  -D YSPARSE_DBG=$(RAPIDYAML_DBG) \
+  -D CMAKE_EXPORT_COMPILE_COMMANDS=ON
 
 
 #------------------------------------------------------------------------------
@@ -39,7 +42,6 @@ rapidyaml:
 
 clean::
 	$(RM) librapidyaml.*
-	$(RM) $(RAPIDYAML_HPP)
 	$(RM) -r $(BDIR)
 	$(RM) -r rapidyaml-install
 
@@ -67,6 +69,3 @@ $(RAPIDYAML_SO): $(RAPIDYAML_DEPS)
 
 $(RAPIDYAML_H): $(RAPIDYAML_JAVA)
 	javac -h . $(RAPIDYAML_JAVA) # $^ doesn't work
-
-$(RAPIDYAML_HPP): rapidyaml
-	$(PYTHON) rapidyaml/tools/amalgamate.py $(RAPIDYAML_AMALGAMATE_OPTS) > $@

rapidyaml/native/org_rapidyaml_Rapidyaml.cpp

@@ -1,5 +1,6 @@
 #include <jni.h>
-#include "./rapidyaml_edn.hpp"
+#include "ysparse_edn.hpp"
+#include "ysparse_evt.hpp"
 #include <stdio.h>
 
 #ifndef _Included_org_rapidyaml_Rapidyaml
@@ -50,6 +51,18 @@ Java_org_rapidyaml_Rapidyaml_ys2edn_1init(JNIEnv *, jobject)
     return (jlong)obj;
 }
 
+/*
+ * Class:     org_rapidyaml_Rapidyaml
+ * Method:    ys2evt_init
+ * Signature: ()J
+ */
+JNIEXPORT jlong JNICALL
+Java_org_rapidyaml_Rapidyaml_ys2evt_1init(JNIEnv *env, jobject)
+{
+    Ryml2Evt *obj = ys2evt_init();
+    return (jlong)obj;
+}
+
 /*
  * Class:     org_rapidyaml_Rapidyaml
  * Method:    ys2edn_destroy
@@ -61,6 +74,17 @@ Java_org_rapidyaml_Rapidyaml_ys2edn_1destroy(JNIEnv *, jobject, jlong obj)
     ys2edn_destroy((Ryml2Edn*)obj);
 }
 
+/*
+ * Class:     org_rapidyaml_Rapidyaml
+ * Method:    ys2evt_destroy
+ * Signature: (Ljava/lang/Object;)V
+ */
+JNIEXPORT void JNICALL
+Java_org_rapidyaml_Rapidyaml_ys2evt_1destroy(JNIEnv *, jobject, jlong obj)
+{
+    ys2evt_destroy((Ryml2Evt*)obj);
+}
+
 /*
  * Class:     org_rapidyaml_Rapidyaml
  * Method:    ys2edn_parse
@@ -93,6 +117,38 @@ Java_org_rapidyaml_Rapidyaml_ys2edn_1parse(JNIEnv *env, jobject,
     return rc;
 }
 
+/*
+ * Class:     org_rapidyaml_Rapidyaml
+ * Method:    ys2evt_parse
+ * Signature: (Ljava/lang/Object;Ljava/lang/String;[BI[BI)I
+ */
+JNIEXPORT jint JNICALL
+Java_org_rapidyaml_Rapidyaml_ys2evt_1parse(JNIEnv *env, jobject,
+                                           jlong obj, jstring jfilename,
+                                           jbyteArray src, jint src_len,
+                                           jintArray dst, jint dst_len)
+{
+    jboolean src_is_copy, dst_is_copy;
+    jbyte* src_ = env->GetByteArrayElements(src, &src_is_copy);
+    int* dst_ = env->GetIntArrayElements(dst, &dst_is_copy);
+    const char *filename = env->GetStringUTFChars(jfilename, 0);
+    int rc = 0;
+    try
+    {
+        rc = ys2evt_parse((Ryml2Evt*)obj, filename,
+                          (char*)src_, src_len,
+                          dst_, dst_len);
+    }
+    catch (Ryml2EvtParseError const& exc)
+    {
+        throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str());
+    }
+    env->ReleaseByteArrayElements(src, src_, 0);
+    env->ReleaseIntArrayElements(dst, dst_, 0);
+    env->ReleaseStringUTFChars(jfilename, filename);
+    return rc;
+}
+
 /*
  * Class:     org_rapidyaml_Rapidyaml
  * Method:    ys2edn_retry_get