Fix the handle_backticks issue

.gitignore

@@ -32,3 +32,14 @@ bstrlib.txt
 build
 cmark.dSYM/*
 cmark
+.vscode
+
+# Testing and benchmark
+alltests.md
+progit/
+bench/benchinput.md
+test/afl_results/
+
+# Build directories for SwiftPM and Xcode
+.swiftpm
+.build

CMakeLists.txt

@@ -1,18 +1,27 @@
 cmake_minimum_required(VERSION 3.0)
-project(cmark VERSION 0.29.0)
+project(cmark-gfm)
+
+set(PROJECT_VERSION_MAJOR 0)
+set(PROJECT_VERSION_MINOR 29)
+set(PROJECT_VERSION_PATCH 0)
+set(PROJECT_VERSION_GFM 0)
+set(PROJECT_VERSION ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}.gfm.${PROJECT_VERSION_GFM})
 
 include("FindAsan.cmake")
+include("CheckFileOffsetBits.cmake")
 
 if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}")
     message(FATAL_ERROR "Do not build in-source.\nPlease remove CMakeCache.txt and the CMakeFiles/ directory.\nThen: mkdir build ; cd build ; cmake .. ; make")
 endif()
 
-option(CMARK_TESTS "Build cmark tests and enable testing" ON)
-option(CMARK_STATIC "Build static libcmark library" ON)
-option(CMARK_SHARED "Build shared libcmark library" ON)
+option(CMARK_TESTS "Build cmark-gfm tests and enable testing" ON)
+option(CMARK_STATIC "Build static libcmark-gfm library" ON)
+option(CMARK_SHARED "Build shared libcmark-gfm library" ON)
 option(CMARK_LIB_FUZZER "Build libFuzzer fuzzing harness" OFF)
+option(CMARK_THREADING "Add locks around static accesses via pthreads" OFF)
 
 add_subdirectory(src)
+add_subdirectory(extensions)
 if(CMARK_TESTS AND (CMARK_SHARED OR CMARK_STATIC))
   add_subdirectory(api_test)
 endif()

CheckFileOffsetBits.c

@@ -0,0 +1,14 @@
+#include <sys/types.h>
+
+#define KB ((off_t)1024)
+#define MB ((off_t)1024 * KB)
+#define GB ((off_t)1024 * MB)
+#define TB ((off_t)1024 * GB)
+int t2[(((64 * GB -1) % 671088649) == 268434537)
+       && (((TB - (64 * GB -1) + 255) % 1792151290) == 305159546)? 1: -1];
+
+int main()
+{
+  ;
+  return 0;
+}

CheckFileOffsetBits.cmake

@@ -0,0 +1,43 @@
+# - Check if _FILE_OFFSET_BITS macro needed for large files
+# CHECK_FILE_OFFSET_BITS ()
+#
+# The following variables may be set before calling this macro to
+# modify the way the check is run:
+#
+#  CMAKE_REQUIRED_FLAGS = string of compile command line flags
+#  CMAKE_REQUIRED_DEFINITIONS = list of macros to define (-DFOO=bar)
+#  CMAKE_REQUIRED_INCLUDES = list of include directories
+# Copyright (c) 2009, Michihiro NAKAJIMA
+#
+# Redistribution and use is allowed according to the terms of the BSD license.
+# For details see the accompanying COPYING-CMAKE-SCRIPTS file.
+
+#INCLUDE(CheckCSourceCompiles)
+
+GET_FILENAME_COMPONENT(_selfdir_CheckFileOffsetBits
+	 "${CMAKE_CURRENT_LIST_FILE}" PATH)
+
+MACRO (CHECK_FILE_OFFSET_BITS)
+  IF(NOT DEFINED _FILE_OFFSET_BITS)
+    MESSAGE(STATUS "Checking _FILE_OFFSET_BITS for large files")
+    TRY_COMPILE(__WITHOUT_FILE_OFFSET_BITS_64
+      ${CMAKE_CURRENT_BINARY_DIR}
+      ${_selfdir_CheckFileOffsetBits}/CheckFileOffsetBits.c
+      COMPILE_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS})
+    IF(NOT __WITHOUT_FILE_OFFSET_BITS_64)
+      TRY_COMPILE(__WITH_FILE_OFFSET_BITS_64
+        ${CMAKE_CURRENT_BINARY_DIR}
+        ${_selfdir_CheckFileOffsetBits}/CheckFileOffsetBits.c
+        COMPILE_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS} -D_FILE_OFFSET_BITS=64)
+    ENDIF(NOT __WITHOUT_FILE_OFFSET_BITS_64)
+
+    IF(NOT __WITHOUT_FILE_OFFSET_BITS_64 AND __WITH_FILE_OFFSET_BITS_64)
+      SET(_FILE_OFFSET_BITS 64 CACHE INTERNAL "_FILE_OFFSET_BITS macro needed for large files")
+      MESSAGE(STATUS "Checking _FILE_OFFSET_BITS for large files - needed")
+    ELSE(NOT __WITHOUT_FILE_OFFSET_BITS_64 AND __WITH_FILE_OFFSET_BITS_64)
+      SET(_FILE_OFFSET_BITS "" CACHE INTERNAL "_FILE_OFFSET_BITS macro needed for large files")
+      MESSAGE(STATUS "Checking _FILE_OFFSET_BITS for large files - not needed")
+    ENDIF(NOT __WITHOUT_FILE_OFFSET_BITS_64 AND __WITH_FILE_OFFSET_BITS_64)
+  ENDIF(NOT DEFINED _FILE_OFFSET_BITS)
+
+ENDMACRO (CHECK_FILE_OFFSET_BITS)

Makefile

@@ -1,31 +1,33 @@
 SRCDIR=src
+EXTDIR=extensions
 DATADIR=data
 BUILDDIR?=build
 GENERATOR?=Unix Makefiles
 MINGW_BUILDDIR?=build-mingw
 MINGW_INSTALLDIR?=windows
 SPEC=test/spec.txt
+EXTENSIONS_SPEC=test/extensions.txt
 SITE=_site
 SPECVERSION=$(shell perl -ne 'print $$1 if /^version: *([0-9.]+)/' $(SPEC))
 FUZZCHARS?=2000000  # for fuzztest
 BENCHDIR=bench
 BENCHSAMPLES=$(wildcard $(BENCHDIR)/samples/*.md)
 BENCHFILE=$(BENCHDIR)/benchinput.md
 ALLTESTS=alltests.md
-NUMRUNS?=10
-CMARK=$(BUILDDIR)/src/cmark
+NUMRUNS?=20
+CMARK=$(BUILDDIR)/src/cmark-gfm
 CMARK_FUZZ=$(BUILDDIR)/src/cmark-fuzz
 PROG?=$(CMARK)
 VERSION?=$(SPECVERSION)
 RELEASE?=CommonMark-$(VERSION)
 INSTALL_PREFIX?=/usr/local
 CLANG_CHECK?=clang-check
-CLANG_FORMAT=clang-format -style llvm -sort-includes=0 -i
+CLANG_FORMAT=clang-format-3.5 -style llvm -sort-includes=0 -i
 AFL_PATH?=/usr/local/bin
 
-.PHONY: all cmake_build leakcheck clean fuzztest test debug ubsan asan mingw archive newbench bench format update-spec afl clang-check libFuzzer
+.PHONY: all cmake_build leakcheck clean fuzztest test debug ubsan asan mingw archive newbench bench format update-spec afl clang-check docker libFuzzer
 
-all: cmake_build man/man3/cmark.3
+all: cmake_build man/man3/cmark-gfm.3
 
 $(CMARK): cmake_build
 
@@ -79,8 +81,9 @@ afl:
 	    -i test/afl_test_cases \
 	    -o test/afl_results \
 	    -x test/fuzzing_dictionary \
+	    $(AFL_OPTIONS) \
 	    -t 100 \
-	    $(CMARK) $(CMARK_OPTS)
+	    $(CMARK) -e table -e strikethrough -e autolink -e tagfilter $(CMARK_OPTS)
 
 libFuzzer:
 	@[ -n "$(LIB_FUZZER_PATH)" ] || { echo '$$LIB_FUZZER_PATH not set'; false; }
@@ -98,7 +101,7 @@ mingw:
 	cmake .. -DCMAKE_TOOLCHAIN_FILE=../toolchain-mingw32.cmake -DCMAKE_INSTALL_PREFIX=$(MINGW_INSTALLDIR) ;\
 	$(MAKE) && $(MAKE) install
 
-man/man3/cmark.3: src/cmark.h | $(CMARK)
+man/man3/cmark-gfm.3: src/cmark-gfm.h | $(CMARK)
 	python man/make_man_page.py $< > $@ \
 
 archive:
@@ -126,6 +129,19 @@ $(SRCDIR)/scanners.c: $(SRCDIR)/scanners.re
 		--encoding-policy substitute -o $@ $<
 	$(CLANG_FORMAT) $@
 
+# We include scanners.c in the repository, so this shouldn't
+# normally need to be generated.
+$(EXTDIR)/ext_scanners.c: $(EXTDIR)/ext_scanners.re
+	@case "$$(re2c -v)" in \
+	    *\ 0.13.*|*\ 0.14|*\ 0.14.1) \
+		echo "re2c >= 0.14.2 is required"; \
+		false; \
+		;; \
+	esac
+	re2c --case-insensitive -b -i --no-generation-date -8 \
+		--encoding-policy substitute -o $@ $<
+	clang-format-3.5 -style llvm -i $@
+
 # We include entities.inc in the repository, so normally this
 # doesn't need to be regenerated:
 $(SRCDIR)/entities.inc: tools/make_entities_inc.py
@@ -138,14 +154,19 @@ update-spec:
 test: $(SPEC) cmake_build
 	$(MAKE) -C $(BUILDDIR) test || (cat $(BUILDDIR)/Testing/Temporary/LastTest.log && exit 1)
 
-$(ALLTESTS): $(SPEC)
-	python3 test/spec_tests.py --spec $< --dump-tests | python3 -c 'import json; import sys; tests = json.loads(sys.stdin.read()); print("\n".join([test["markdown"] for test in tests]))' > $@
+$(ALLTESTS): $(SPEC) $(EXTENSIONS_SPEC)
+	( \
+	  python3 test/spec_tests.py --spec $(SPEC) --dump-tests | \
+	    python3 -c 'import json; import sys; tests = json.loads(sys.stdin.read()); u8s = open(1, "w", encoding="utf-8", closefd=False); print("\n".join([test["markdown"] for test in tests]), file=u8s)'; \
+	  python3 test/spec_tests.py --spec $(EXTENSIONS_SPEC) --dump-tests | \
+	    python3 -c 'import json; import sys; tests = json.loads(sys.stdin.read()); u8s = open(1, "w", encoding="utf-8", closefd=False); print("\n".join([test["markdown"] for test in tests]), file=u8s)'; \
+	) > $@
 
 leakcheck: $(ALLTESTS)
 	for format in html man xml latex commonmark; do \
 	  for opts in "" "--smart"; do \
-	     echo "cmark -t $$format $$opts" ; \
-	     valgrind -q --leak-check=full --dsymutil=yes --error-exitcode=1 $(PROG) -t $$format $$opts $(ALLTESTS) >/dev/null || exit 1;\
+	     echo "cmark-gfm -t $$format -e table -e strikethrough -e autolink -e tagfilter $$opts" ; \
+	     valgrind -q --leak-check=full --dsymutil=yes --suppressions=suppressions --error-exitcode=1 $(PROG) -t $$format -e table -e strikethrough -e autolink -e tagfilter $$opts $(ALLTESTS) >/dev/null || exit 1;\
           done; \
 	done;
 
@@ -189,10 +210,17 @@ newbench:
 format:
 	$(CLANG_FORMAT) src/*.c src/*.h api_test/*.c api_test/*.h
 
+format-extensions:
+	clang-format-3.5 -style llvm -i extensions/*.c extensions/*.h
+
 operf: $(CMARK)
 	operf $< < $(BENCHFILE) > /dev/null
 
 distclean: clean
 	-rm -rf *.dSYM
 	-rm -f README.html
 	-rm -rf $(BENCHFILE) $(ALLTESTS) progit
+
+docker:
+	docker build -t cmark-gfm $(CURDIR)/tools
+	docker run --privileged -t -i -v $(CURDIR):/src/cmark-gfm -w /src/cmark-gfm cmark-gfm /bin/bash

Makefile.nmake

@@ -3,7 +3,7 @@ DATADIR=data
 BUILDDIR=build
 INSTALLDIR=windows
 SPEC=test/spec.txt
-PROG=$(BUILDDIR)\src\cmark.exe
+PROG=$(BUILDDIR)\src\cmark-gfm.exe
 GENERATOR=NMake Makefiles
 
 all: $(BUILDDIR)/CMakeFiles
@@ -16,6 +16,8 @@ $(BUILDDIR)/CMakeFiles:
 	    -G "$(GENERATOR)" \
 	    -D CMAKE_BUILD_TYPE=$(BUILD_TYPE) \
 	    -D CMAKE_INSTALL_PREFIX=$(INSTALLDIR) \
+	    -D CMARK_STATIC=ON \
+	    -D CMARK_SHARED=OFF \
 	    .. && \
 	cd ..
 

Package.swift

@@ -0,0 +1,66 @@
+// swift-tools-version:5.3
+// The swift-tools-version declares the minimum version of Swift required to build this package.
+
+import PackageDescription
+
+let package = Package(
+    name: "cmark-gfm",
+    products: [
+        // Products define the executables and libraries a package produces, and make them visible to other packages.
+        .library(
+            name: "cmark-gfm",
+            targets: ["cmark-gfm"]),
+        .library(
+            name: "cmark-gfm-extensions",
+            targets: ["cmark-gfm-extensions"]),
+        .executable(
+            name: "cmark-gfm-bin",
+            targets: ["cmark-gfm-bin"]),
+        .executable(name: "api_test",
+            targets: ["api_test"])
+    ],
+    targets: [
+        .target(name: "cmark-gfm",
+          path: "src",
+          exclude: [
+            "scanners.re",
+            "libcmark-gfm.pc.in",
+            "config.h.in",
+            "CMakeLists.txt",
+            "cmark-gfm_version.h.in",
+            "case_fold_switch.inc",
+            "entities.inc",
+          ]
+        ),
+        .target(name: "cmark-gfm-extensions",
+          dependencies: [
+            "cmark-gfm",
+          ],
+          path: "extensions",
+          exclude: [
+            "CMakeLists.txt",
+            "ext_scanners.re",
+          ]
+        ),
+        .target(name: "cmark-gfm-bin",
+          dependencies: [
+            "cmark-gfm",
+            "cmark-gfm-extensions",
+          ],
+          path: "bin",
+          sources: [
+            "main.c",
+          ]
+        ),
+        .target(name: "api_test",
+          dependencies: [
+            "cmark-gfm",
+            "cmark-gfm-extensions",
+          ],
+          path: "api_test",
+          exclude: [
+            "CMakeLists.txt",
+          ]
+        )
+    ]
+)

README.md

@@ -1,13 +1,25 @@
-cmark
-=====
+cmark-gfm
+=========
 
-[![Build Status]](https://travis-ci.org/commonmark/cmark)
-[![Windows Build Status]](https://ci.appveyor.com/project/jgm/cmark-0ub06)
+`cmark-gfm` is an extended version of the C reference implementation of
+[CommonMark], a rationalized version of Markdown syntax with a spec.  This
+repository adds GitHub Flavored Markdown extensions to
+[the upstream implementation], as defined in [the spec].
 
-`cmark` is the C reference implementation of [CommonMark], a
-rationalized version of Markdown syntax with a [spec][the spec].
-(For the JavaScript reference implementation, see
-[commonmark.js].)
+Changes upstream in `cmark` will be pulled into this `cmark-gfm` project repository
+as the upstream project evolves. The original `cmark` repository can be found here:
+<https://github.com/commonmark/cmark>.
+
+The rest of the README is preserved as-is from the upstream source.  Note that
+the library and binaries produced by this fork are suffixed with `-gfm` in
+order to distinguish them from the upstream.
+
+## License
+
+The original `cmark` code is released under a BSD2 license. This same license
+applies to the Swift code included in this `cmark-gfm` repository.
+
+---
 
 It provides a shared library (`libcmark`) with functions for parsing
 CommonMark documents to an abstract syntax tree (AST), manipulating
@@ -59,8 +71,10 @@ There are also libraries that wrap `libcmark` for
 [Lua](https://github.com/jgm/cmark-lua),
 [Perl](https://metacpan.org/release/CommonMark),
 [Python](https://pypi.python.org/pypi/paka.cmark),
-[R](https://cran.r-project.org/package=commonmark) and
-[Scala](https://github.com/sparsetech/cmark-scala).
+[R](https://cran.r-project.org/package=commonmark),
+[Tcl](https://github.com/apnadkarni/tcl-cmark),
+[Scala](https://github.com/sparsetech/cmark-scala) and
+[Node.js](https://github.com/killa123/node-cmark).
 
 Installing
 ----------
@@ -187,12 +201,13 @@ Nick Wellnhofer contributed many improvements, including
 most of the C library's API and its test harness.
 
 [benchmarks]: benchmarks.md
-[the spec]: http://spec.commonmark.org
+[the spec]: https://github.github.com/gfm/
+[the upstream implementation]: https://github.com/jgm/cmark
 [CommonMark]: http://commonmark.org
 [cmake]: http://www.cmake.org/download/
 [re2c]: http://re2c.org
 [commonmark.js]: https://github.com/commonmark/commonmark.js
-[Build Status]: https://img.shields.io/travis/commonmark/cmark/master.svg?style=flat
-[Windows Build Status]: https://ci.appveyor.com/api/projects/status/h3fd91vtd1xfmp69?svg=true
+[Build Status]: https://img.shields.io/travis/github/cmark-gfm/master.svg?style=flat
+[Windows Build Status]: https://ci.appveyor.com/api/projects/status/wv7ifhqhv5itm3d5?svg=true
 [american fuzzy lop]: http://lcamtuf.coredump.cx/afl/
 [libFuzzer]: http://llvm.org/docs/LibFuzzer.html

api_test/CMakeLists.txt

@@ -5,13 +5,15 @@ add_executable(api_test
   main.c
 )
 include_directories(
-  ${PROJECT_SOURCE_DIR}/src
-  ${PROJECT_BINARY_DIR}/src
+  ${PROJECT_SOURCE_DIR}/src/include
+  ${PROJECT_BINARY_DIR}/src/include
+  ${PROJECT_SOURCE_DIR}/extensions/include
+  ${PROJECT_BINARY_DIR}/extensions/include
 )
 if(CMARK_SHARED)
-  target_link_libraries(api_test libcmark)
+  target_link_libraries(api_test libcmark-gfm-extensions libcmark-gfm)
 else()
-  target_link_libraries(api_test libcmark_static)
+  target_link_libraries(api_test libcmark-gfm-extensions_static libcmark-gfm_static)
 endif()
 
 # Compiler flags