Table extension from c068469 reworked

Note this includes a hack to the core code to escape pipes in the
'commonmark' renderer.  This is to fix test cases with the table
extension; i.e. we treat pipes as special characters that need escaping.

We use the cmark_mem of the parser in order to ensure we use the arena
allocator when necessary.  A very flexible table format is supported;
see test/extensions.txt for examples.  Leading and trailing pipes can be
omitted, and alignment specifiers can be used in the separator between
the header and body.  Table bodies don't need to be a consistent width.
Embedded HTML is OK.

Note we reuse the inline parser from cmark to parse tables -- this is to
ensure pipes e.g. in the middle of an inline code block don't
prematurely terminate a table cell.

Makefile

@@ -1,10 +1,12 @@
 SRCDIR=src
+EXTDIR=extensions
 DATADIR=data
 BUILDDIR?=build
 GENERATOR?=Unix Makefiles
 MINGW_BUILDDIR?=build-mingw
 MINGW_INSTALLDIR?=windows
 SPEC=test/spec.txt
+EXTENSIONS_SPEC=test/extensions.txt
 SITE=_site
 SPECVERSION=$(shell perl -ne 'print $$1 if /^version: *([0-9.]+)/' $(SPEC))
 FUZZCHARS?=2000000  # for fuzztest
@@ -80,7 +82,7 @@ afl:
 	    -o test/afl_results \
 	    -x test/fuzzing_dictionary \
 	    -t 100 \
-	    $(CMARK) $(CMARK_OPTS)
+	    $(CMARK) -e table $(CMARK_OPTS)
 
 libFuzzer:
 	@[ -n "$(LIB_FUZZER_PATH)" ] || { echo '$$LIB_FUZZER_PATH not set'; false; }
@@ -126,6 +128,19 @@ $(SRCDIR)/scanners.c: $(SRCDIR)/scanners.re
 		--encoding-policy substitute -o $@ $<
 	$(CLANG_FORMAT) $@
 
+# We include scanners.c in the repository, so this shouldn't
+# normally need to be generated.
+$(EXTDIR)/ext_scanners.c: $(EXTDIR)/ext_scanners.re
+	@case "$$(re2c -v)" in \
+	    *\ 0.13.*|*\ 0.14|*\ 0.14.1) \
+		echo "re2c >= 0.14.2 is required"; \
+		false; \
+		;; \
+	esac
+	re2c --case-insensitive -b -i --no-generation-date -8 \
+		--encoding-policy substitute -o $@ $<
+	clang-format -style llvm -i $@
+
 # We include entities.inc in the repository, so normally this
 # doesn't need to be regenerated:
 $(SRCDIR)/entities.inc: tools/make_entities_inc.py
@@ -138,14 +153,19 @@ update-spec:
 test: $(SPEC) cmake_build
 	$(MAKE) -C $(BUILDDIR) test || (cat $(BUILDDIR)/Testing/Temporary/LastTest.log && exit 1)
 
-$(ALLTESTS): $(SPEC)
-	python3 test/spec_tests.py --spec $< --dump-tests | python3 -c 'import json; import sys; tests = json.loads(sys.stdin.read()); print("\n".join([test["markdown"] for test in tests]))' > $@
+$(ALLTESTS): $(SPEC) $(EXTENSIONS_SPEC)
+	( \
+	  python3 test/spec_tests.py --spec $(SPEC) --dump-tests | \
+	    python3 -c 'import json; import sys; tests = json.loads(sys.stdin.read()); print("\n".join([test["markdown"] for test in tests]))'; \
+	  python3 test/spec_tests.py --spec $(EXTENSIONS_SPEC) --dump-tests | \
+	    python3 -c 'import json; import sys; tests = json.loads(sys.stdin.read()); print("\n".join([test["markdown"] for test in tests]))'; \
+	) > $@
 
 leakcheck: $(ALLTESTS)
 	for format in html man xml latex commonmark; do \
 	  for opts in "" "--smart"; do \
-	     echo "cmark -t $$format $$opts" ; \
-	     valgrind -q --leak-check=full --dsymutil=yes --error-exitcode=1 $(PROG) -t $$format $$opts $(ALLTESTS) >/dev/null || exit 1;\
+	     echo "cmark -t $$format -e table $$opts" ; \
+	     valgrind -q --leak-check=full --dsymutil=yes --suppressions=suppressions --error-exitcode=1 $(PROG) -t $$format -e table $$opts $(ALLTESTS) >/dev/null || exit 1;\
           done; \
 	done;
 

extensions/CMakeLists.txt

@@ -2,6 +2,10 @@ cmake_minimum_required(VERSION 2.8)
 set(STATICLIBRARY "libcmarkextensions_static")
 set(LIBRARY_SOURCES
     core-extensions.c
+    table.c
+    ext_scanners.c
+    ext_scanners.re
+    ext_scanners.h
    )
 
 include_directories(

extensions/core-extensions.c

@@ -1,3 +1,7 @@
 #include "core-extensions.h"
+#include "table.h"
 
-int core_extensions_registration(cmark_plugin *plugin) { return 1; }
+int core_extensions_registration(cmark_plugin *plugin) {
+  cmark_plugin_register_syntax_extension(plugin, create_table_extension());
+  return 1;
+}