New pcre2

.gitattributes

@@ -34,8 +34,8 @@ fish.spec.in export-ignore
 /.github/* export-ignore
 
 # for linguist; let github identify our project as C++ instead of C due to pcre2
-/pcre2-10.21/ linguist-vendored
-/pcre2-10.21/* linguist-vendored
+/pcre2-10.22/ linguist-vendored
+/pcre2-10.22/* linguist-vendored
 angular.js linguist-vendored
 /doc_src/* linguist-documentation
 *.fish linguist-language=fish

Makefile.in

@@ -70,7 +70,7 @@ extra_confdir = @extra_confdir@
 # pcre2
 #
 PCRE2_WIDTH = @WCHAR_T_BITS@
-PCRE2_DIR = pcre2-10.21
+PCRE2_DIR = pcre2-10.22
 PCRE2_LIBDIR = $(PCRE2_DIR)/.libs
 PCRE2_LIB = $(PCRE2_LIBDIR)/libpcre2-$(PCRE2_WIDTH).a
 PCRE2_H = $(PCRE2_DIR)/src/pcre2.h

build_tools/lint.fish

@@ -135,14 +135,14 @@ if set -q c_files[1]
                 oclint-xcodebuild xcodebuild.log >/dev/null
             end
             if test $all = yes
-                oclint-json-compilation-database -e '/pcre2-10.21/' -- -enable-global-analysis 2>&1
+                oclint-json-compilation-database -e '/pcre2-10.22/' -- -enable-global-analysis 2>&1
             else
                 set i_files
                 for f in $c_files
                     set i_files $i_files -i $f
                 end
-                echo oclint-json-compilation-database -e '/pcre2-10.21/' $i_files
-                oclint-json-compilation-database -e '/pcre2-10.21/' $i_files 2>&1
+                echo oclint-json-compilation-database -e '/pcre2-10.22/' $i_files
+                oclint-json-compilation-database -e '/pcre2-10.22/' $i_files 2>&1
             end
         else
             # Presumably we're on Linux or other platform not requiring special

configure.ac

@@ -659,7 +659,7 @@ if test "x$included_pcre2" = "xyes"; then
   AC_MSG_NOTICE([using included PCRE2 library])
   # unfortunately these get added to the global configuration
   ac_configure_args="$ac_configure_args --disable-pcre2-8 --enable-pcre2-$WCHAR_T_BITS --disable-shared"
-  AC_CONFIG_SUBDIRS([pcre2-10.21])
+  AC_CONFIG_SUBDIRS([pcre2-10.22])
 
   PCRE2_CXXFLAGS='-I$(PCRE2_DIR)/src'
   PCRE2_LIBS='-L$(PCRE2_LIBDIR) -lpcre2-$(PCRE2_WIDTH)'

fish.xcodeproj/project.pbxproj

@@ -817,7 +817,7 @@
 				D04F7FF71BA4E82C00B0F227 /* pcre2_chartables.c.dist */,
 			);
 			name = pcre;
-			path = "pcre2-10.21/src";
+			path = "pcre2-10.22/src";
 			sourceTree = SOURCE_ROOT;
 		};
 		D08A328E17B4455100F3A533 /* fish_tests */ = {

pcre2-10.21/CMakeLists.txt → pcre2-10.22/CMakeLists.txt

@@ -71,6 +71,9 @@
 # 2015-07-16 PH updated for new pcre2_find_bracket source module
 # 2015-08-24 PH correct C_FLAGS setting (patch from Roy Ivy III)
 # 2015-10=16 PH added support for never-backslash-C
+# 2016-03-01 PH applied Chris Wilson's patch for MSVC static
+# 2016-06-24 PH applied Chris Wilson's second patch, putting the first under
+#            a new option instead of being unconditional.
 
 PROJECT(PCRE2 C)
 
@@ -157,6 +160,9 @@ SET(PCRE2_SUPPORT_JIT OFF CACHE BOOL
 SET(PCRE2_SUPPORT_PCRE2GREP_JIT ON CACHE BOOL
     "Enable use of Just-in-time compiling in pcre2grep.")
 
+SET(PCRE2_SUPPORT_PCRE2GREP_CALLOUT ON CACHE BOOL
+    "Enable callout string support in pcre2grep.")
+
 SET(PCRE2_SUPPORT_UNICODE ON CACHE BOOL
     "Enable support for Unicode and UTF-8/UTF-16/UTF-32 encoding.")
 
@@ -184,6 +190,9 @@ IF (MINGW)
 ENDIF(MINGW)
 
 IF(MSVC)
+  OPTION(PCRE_STATIC_RUNTIME OFF CACHE BOOL
+	"ON=Compile against the static runtime (/MT)."
+	OFF)
   OPTION(INSTALL_MSVC_PDB
          "ON=Install .pdb files built by MSVC, if generated"
          OFF)
@@ -272,6 +281,10 @@ IF(PCRE2_SUPPORT_PCRE2GREP_JIT)
         SET(SUPPORT_PCRE2GREP_JIT 1)
 ENDIF(PCRE2_SUPPORT_PCRE2GREP_JIT)
 
+IF(PCRE2_SUPPORT_PCRE2GREP_CALLOUT)
+        SET(SUPPORT_PCRE2GREP_CALLOUT 1)
+ENDIF(PCRE2_SUPPORT_PCRE2GREP_CALLOUT)
+
 IF(PCRE2_SUPPORT_VALGRIND)
         SET(SUPPORT_VALGRIND 1)
 ENDIF(PCRE2_SUPPORT_VALGRIND)
@@ -456,6 +469,18 @@ SET(PCRE2POSIX_SOURCES
 ENDIF (EXISTS ${PROJECT_SOURCE_DIR}/pcre2posix.rc)
 ENDIF(MSVC AND NOT PCRE2_STATIC)
 
+# Fix static compilation with MSVC: https://bugs.exim.org/show_bug.cgi?id=1681
+# This code was taken from the CMake wiki, not from WebM.
+
+IF(MSVC AND PCRE2_STATIC_RUNTIME)
+  MESSAGE(STATUS "** MSVC and PCRE2_STATIC_RUNTIME: modifying compiler flags to use static runtime library")
+  foreach(flag_var
+          CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
+          CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO)
+    string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
+  endforeach()
+ENDIF(MSVC AND PCRE2_STATIC_RUNTIME)
+
 # Build setup
 
 ADD_DEFINITIONS(-DHAVE_CONFIG_H)
@@ -740,6 +765,7 @@ IF(PCRE2_SHOW_REPORT)
   MESSAGE(STATUS "  Build static libs ............... : ${BUILD_STATIC_LIBS}")
   MESSAGE(STATUS "  Build pcre2grep ................. : ${PCRE2_BUILD_PCRE2GREP}")
   MESSAGE(STATUS "  Enable JIT in pcre2grep ......... : ${PCRE2_SUPPORT_PCRE2GREP_JIT}")
+  MESSAGE(STATUS "  Enable callouts in pcre2grep .... : ${PCRE2_SUPPORT_PCRE2GREP_CALLOUT}")
   MESSAGE(STATUS "  Buffer size for pcre2grep ....... : ${PCRE2GREP_BUFSIZE}")
   MESSAGE(STATUS "  Build tests (implies pcre2test  . : ${PCRE2_BUILD_TESTS}")
   MESSAGE(STATUS "               and pcre2grep)")

pcre2-10.21/ChangeLog → pcre2-10.22/ChangeLog

@@ -1,6 +1,182 @@
 Change Log for PCRE2
 --------------------
 
+
+Version 10.22 29-July-2016
+--------------------------
+
+1. Applied Jason Hood's patches to RunTest.bat and testdata/wintestoutput3
+to fix problems with running the tests under Windows.
+
+2. Implemented a facility for quoting literal characters within hexadecimal
+patterns in pcre2test, to make it easier to create patterns with just a few
+non-printing characters.
+
+3. Binary zeros are not supported in pcre2test input files. It now detects them
+and gives an error.
+
+4. Updated the valgrind parameters in RunTest: (a) changed smc-check=all to
+smc-check=all-non-file; (b) changed obj:* in the suppression file to obj:??? so
+that it matches only unknown objects.
+
+5. Updated the maintenance script maint/ManyConfigTests to make it easier to
+select individual groups of tests.
+
+6. When the POSIX wrapper function regcomp() is called, the REG_NOSUB option
+used to set PCRE2_NO_AUTO_CAPTURE when calling pcre2_compile(). However, this
+disables the use of back references (and subroutine calls), which are supported
+by other implementations of regcomp() with RE_NOSUB. Therefore, REG_NOSUB no
+longer causes PCRE2_NO_AUTO_CAPTURE to be set, though it still ignores nmatch
+and pmatch when regexec() is called.
+
+7. Because of 6 above, pcre2test has been modified with a new modifier called
+posix_nosub, to call regcomp() with REG_NOSUB. Previously the no_auto_capture
+modifier had this effect. That option is now ignored when the POSIX API is in
+use.
+
+8. Minor tidies to the pcre2demo.c sample program, including more comments
+about its 8-bit-ness.
+
+9. Detect unmatched closing parentheses and give the error in the pre-scan
+instead of later. Previously the pre-scan carried on and could give a
+misleading incorrect error message. For example, /(?J)(?'a'))(?'a')/ gave a
+message about invalid duplicate group names.
+
+10. It has happened that pcre2test was accidentally linked with another POSIX
+regex library instead of libpcre2-posix. In this situation, a call to regcomp()
+(in the other library) may succeed, returning zero, but of course putting its
+own data into the regex_t block. In one example the re_pcre2_code field was
+left as NULL, which made pcre2test think it had not got a compiled POSIX regex,
+so it treated the next line as another pattern line, resulting in a confusing
+error message. A check has been added to pcre2test to see if the data returned
+from a successful call of regcomp() are valid for PCRE2's regcomp(). If they
+are not, an error message is output and the pcre2test run is abandoned. The
+message points out the possibility of a mis-linking. Hopefully this will avoid
+some head-scratching the next time this happens.
+
+11. A pattern such as /(?<=((?C)0))/, which has a callout inside a lookbehind
+assertion, caused pcre2test to output a very large number of spaces when the
+callout was taken, making the program appearing to loop.
+
+12. A pattern that included (*ACCEPT) in the middle of a sufficiently deeply
+nested set of parentheses of sufficient size caused an overflow of the
+compiling workspace (which was diagnosed, but of course is not desirable).
+
+13. Detect missing closing parentheses during the pre-pass for group
+identification.
+
+14. Changed some integer variable types and put in a number of casts, following
+a report of compiler warnings from Visual Studio 2013 and a few tests with
+gcc's -Wconversion (which still throws up a lot).
+
+15. Implemented pcre2_code_copy(), and added pushcopy and #popcopy to pcre2test
+for testing it.
+
+16. Change 66 for 10.21 introduced the use of snprintf() in PCRE2's version of
+regerror(). When the error buffer is too small, my version of snprintf() puts a
+binary zero in the final byte. Bug #1801 seems to show that other versions do
+not do this, leading to bad output from pcre2test when it was checking for
+buffer overflow. It no longer assumes a binary zero at the end of a too-small
+regerror() buffer.
+
+17. Fixed typo ("&&" for "&") in pcre2_study(). Fortunately, this could not
+actually affect anything, by sheer luck.
+
+18. Two minor fixes for MSVC compilation: (a) removal of apparently incorrect
+"const" qualifiers in pcre2test and (b) defining snprintf as _snprintf for
+older MSVC compilers. This has been done both in src/pcre2_internal.h for most
+of the library, and also in src/pcre2posix.c, which no longer includes
+pcre2_internal.h (see 24 below).
+
+19. Applied Chris Wilson's patch (Bugzilla #1681) to CMakeLists.txt for MSVC
+static compilation. Subsequently applied Chris Wilson's second patch, putting
+the first patch under a new option instead of being unconditional when
+PCRE_STATIC is set.
+
+20. Updated pcre2grep to set stdout as binary when run under Windows, so as not
+to convert \r\n at the ends of reflected lines into \r\r\n. This required
+ensuring that other output that is written to stdout (e.g. file names) uses the
+appropriate line terminator: \r\n for Windows, \n otherwise.
+
+21. When a line is too long for pcre2grep's internal buffer, show the maximum
+length in the error message.
+
+22. Added support for string callouts to pcre2grep (Zoltan's patch with PH
+additions).
+
+23. RunTest.bat was missing a "set type" line for test 22.
+
+24. The pcre2posix.c file was including pcre2_internal.h, and using some
+"private" knowledge of the data structures. This is unnecessary; the code has
+been re-factored and no longer includes pcre2_internal.h.
+
+25. A racing condition is fixed in JIT reported by Mozilla.
+
+26. Minor code refactor to avoid "array subscript is below array bounds"
+compiler warning.
+
+27. Minor code refactor to avoid "left shift of negative number" warning.
+
+28. Add a bit more sanity checking to pcre2_serialize_decode() and document
+that it expects trusted data.
+
+29. Fix typo in pcre2_jit_test.c
+
+30. Due to an oversight, pcre2grep was not making use of JIT when available.
+This is now fixed.
+
+31. The RunGrepTest script is updated to use the valgrind suppressions file
+when testing with JIT under valgrind (compare 10.21/51 below). The suppressions
+file is updated so that is now the same as for PCRE1: it suppresses the
+Memcheck warnings Addr16 and Cond in unknown objects (that is, JIT-compiled
+code). Also changed smc-check=all to smc-check=all-non-file as was done for
+RunTest (see 4 above).
+
+32. Implemented the PCRE2_NO_JIT option for pcre2_match().
+
+33. Fix typo that gave a compiler error when JIT not supported.
+
+34. Fix comment describing the returns from find_fixedlength().
+
+35. Fix potential negative index in pcre2test.
+
+36. Calls to pcre2_get_error_message() with error numbers that are never
+returned by PCRE2 functions were returning empty strings. Now the error code
+PCRE2_ERROR_BADDATA is returned. A facility has been added to pcre2test to
+show the texts for given error numbers (i.e. to call pcre2_get_error_message()
+and display what it returns) and a few representative error codes are now
+checked in RunTest.
+
+37. Added "&& !defined(__INTEL_COMPILER)" to the test for __GNUC__ in
+pcre2_match.c, in anticipation that this is needed for the same reason it was
+recently added to pcrecpp.cc in PCRE1.
+
+38. Using -o with -M in pcre2grep could cause unnecessary repeated output when
+the match extended over a line boundary, as it tried to find more matches "on
+the same line" - but it was already over the end.
+
+39. Allow \C in lookbehinds and DFA matching in UTF-32 mode (by converting it
+to the same code as '.' when PCRE2_DOTALL is set).
+
+40. Fix two clang compiler warnings in pcre2test when only one code unit width
+is supported.
+
+41. Upgrade RunTest to automatically re-run test 2 with a large (64M) stack if
+it fails when running the interpreter with a 16M stack (and if changing the
+stack size via pcre2test is possible). This avoids having to manually set a
+large stack size when testing with clang.
+
+42. Fix register overwite in JIT when SSE2 acceleration is enabled.
+
+43. Detect integer overflow in pcre2test pattern and data repetition counts.
+
+44. In pcre2test, ignore "allcaptures" after DFA matching.
+
+45. Fix unaligned accesses on x86. Patch by Marc Mutz.
+
+46. Fix some more clang compiler warnings.
+
+
 Version 10.21 12-January-2016
 -----------------------------
 
@@ -371,7 +547,7 @@ space or a #-type comment that was followed by (?-x), which turns off
 PCRE2_EXTENDED, and there was no subsequent (?x) to turn it on again,
 pcre2_compile() assumed that (?-x) applied to the whole pattern and
 consequently mis-compiled it. This bug was found by the LLVM fuzzer. The fix
-for this bug means that a setting of any of the (?imsxU) options at the start
+for this bug means that a setting of any of the (?imsxJU) options at the start
 of a pattern is no longer transferred to the options that are returned by
 PCRE2_INFO_ALLOPTIONS. In fact, this was an anachronism that should have
 changed when the effects of those options were all moved to compile time.

pcre2-10.21/HACKING → pcre2-10.22/HACKING

@@ -228,6 +228,11 @@ OP_ASSERT_ACCEPT is used when (*ACCEPT) is encountered within an assertion.
 This ends the assertion, not the entire pattern match. The assertion (?!) is 
 always optimized to OP_FAIL.
 
+OP_ALLANY is used for '.' when PCRE2_DOTALL is set. It is also used for \C in
+non-UTF modes and in UTF-32 mode (since one code unit still equals one 
+character). Another use is for [^] when empty classes are permitted
+(PCRE2_ALLOW_EMPTY_CLASS is set).
+
 
 Backtracking control verbs with optional data
 ---------------------------------------------
@@ -601,4 +606,4 @@ not a real opcode, but is used to check that tables indexed by opcode are the
 correct length, in order to catch updating errors.
 
 Philip Hazel
-June 2015
+June 2016

pcre2-10.21/Makefile.am → pcre2-10.22/Makefile.am

@@ -25,6 +25,7 @@ dist_html_DATA = \
   doc/html/pcre2-config.html \
   doc/html/pcre2.html \
   doc/html/pcre2_callout_enumerate.html \
+  doc/html/pcre2_code_copy.html \
   doc/html/pcre2_code_free.html \
   doc/html/pcre2_compile.html \
   doc/html/pcre2_compile_context_copy.html \
@@ -105,6 +106,7 @@ dist_man_MANS = \
   doc/pcre2-config.1 \
   doc/pcre2.3 \
   doc/pcre2_callout_enumerate.3 \
+  doc/pcre2_code_copy.3 \
   doc/pcre2_code_free.3 \
   doc/pcre2_compile.3 \
   doc/pcre2_compile_context_copy.3 \
@@ -568,6 +570,7 @@ EXTRA_DIST += \
   testdata/greplist \
   testdata/grepoutput \
   testdata/grepoutput8 \
+  testdata/grepoutputC \
   testdata/grepoutputN \
   testdata/greppatN4 \
   testdata/testinput1 \