Merge pull request #1247 from EdgarModesto23/edgar-add-support-zstd

Added support for zstd compression

README.md

@@ -217,6 +217,7 @@ $ meson setup build                                 \
     -DPISTACHE_BUILD_DOCS=false                     \
     -DPISTACHE_USE_CONTENT_ENCODING_BROTLI=true     \
     -DPISTACHE_USE_CONTENT_ENCODING_DEFLATE=true    \
+    -DPISTACHE_USE_CONTENT_ENCODING_ZSTD=true    \
     --prefix="$PWD/prefix"
 $ meson compile -C build
 $ meson install -C build
@@ -240,6 +241,7 @@ Some other Meson options:
 | PISTACHE_BUILD_DOCS                   | False   | Build Doxygen docs                             |
 | PISTACHE_USE_CONTENT_ENCODING_BROTLI  | False   | Build with Brotli content encoding support     |
 | PISTACHE_USE_CONTENT_ENCODING_DEFLATE | False   | Build with deflate content encoding support    |
+| PISTACHE_USE_CONTENT_ENCODING_ZSTD    | False   | Build with zstd content encoding support       |
 
 ## Example
 

bldscripts/mesbuild.sh

@@ -22,6 +22,7 @@ else
     -DPISTACHE_BUILD_TESTS=true \
     -DPISTACHE_BUILD_DOCS=false \
     -DPISTACHE_USE_CONTENT_ENCODING_DEFLATE=true \
+    -DPISTACHE_USE_CONTENT_ENCODING_ZSTD=true \
     --prefix="${MESON_PREFIX_DIR}" \
 #    -DPISTACHE_FORCE_LIBEVENT=true
 

bldscripts/mesbuilddebug.sh

@@ -22,6 +22,7 @@ else
     -DPISTACHE_BUILD_TESTS=true \
     -DPISTACHE_BUILD_DOCS=false \
     -DPISTACHE_USE_CONTENT_ENCODING_DEFLATE=true \
+    -DPISTACHE_USE_CONTENT_ENCODING_ZSTD=true \
     -DPISTACHE_DEBUG=true \
     --prefix="${MESON_PREFIX_DIR}" \
 #    -DPISTACHE_FORCE_LIBEVENT=true

include/pistache/http.h

@@ -48,6 +48,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef PISTACHE_USE_CONTENT_ENCODING_ZSTD
+#include <zstd.h>
+#endif
+
 namespace Pistache
 {
     namespace Tcp
@@ -530,6 +534,14 @@ namespace Pistache
                 contentEncodingBrotliLevel_ = _contentEncodingBrotliLevel;
             }
 #endif
+#ifdef PISTACHE_USE_CONTENT_ENCODING_ZSTD
+
+            void setCompressionZstdLevel(const int contentEncodingZstdLevel)
+            {
+                contentEncodingZstdLevel_ = contentEncodingZstdLevel;
+            }
+
+#endif
 
 #ifdef PISTACHE_USE_CONTENT_ENCODING_DEFLATE
             // Set the compression level for deflate algorithm. Defaults to
@@ -562,6 +574,10 @@ namespace Pistache
             int contentEncodingBrotliLevel_ = BROTLI_DEFAULT_QUALITY;
 #endif
 
+#ifdef PISTACHE_USE_CONTENT_ENCODING_ZSTD
+            int contentEncodingZstdLevel_ = ZSTD_defaultCLevel();
+#endif
+
 #ifdef PISTACHE_USE_CONTENT_ENCODING_DEFLATE
             int contentEncodingDeflateLevel_ = Z_DEFAULT_COMPRESSION;
 #endif

include/pistache/http_header.h

@@ -68,6 +68,7 @@ namespace Pistache::Http::Header
                           Br,
                           Compress,
                           Deflate,
+                          Zstd,
                           Identity,
                           Chunked,
                           Unknown };

meson.build

@@ -62,6 +62,15 @@ if get_option('PISTACHE_USE_RAPIDJSON')
 	public_deps += rapidjson_dep
 endif
 
+# Support Zstd compressed Content-Encoding responses...
+if get_option('PISTACHE_USE_CONTENT_ENCODING_ZSTD')
+
+  #Need Zstd encoder for library...
+  zstd_dep = dependency('libzstd')
+  deps_libpistache += zstd_dep
+  public_deps += zstd_dep
+endif
+
 # Support Brotli compressed Content-Encoding responses...
 if get_option('PISTACHE_USE_CONTENT_ENCODING_BROTLI')
 

meson_options.txt

@@ -9,6 +9,7 @@ option('PISTACHE_INSTALL', type: 'boolean', value: true, description: 'add pista
 option('PISTACHE_USE_SSL', type: 'boolean', value: false, description: 'add support for SSL server')
 option('PISTACHE_USE_RAPIDJSON', type: 'boolean', value: true, description: 'add support for rapidjson')
 option('PISTACHE_USE_CONTENT_ENCODING_BROTLI', type: 'boolean', value: false, description: 'add support for Brotli compressed content encoding')
+option('PISTACHE_USE_CONTENT_ENCODING_ZSTD', type: 'boolean', value: false, description: 'add support for Zstandard compressed content encoding')
 option('PISTACHE_USE_CONTENT_ENCODING_DEFLATE', type: 'boolean', value: false, description: 'add support for deflate compressed content encoding')
 option('PISTACHE_DEBUG', type: 'boolean', value: false, description: 'with debugging code')
 option('PISTACHE_LOG_AND_STDOUT', type: 'boolean', value: false, description: 'send log msgs to stdout too')

src/common/http.cc

@@ -10,6 +10,7 @@
    Http layer implementation
 */
 
+#include <pistache/http_header.h>
 #include <pistache/config.h>
 #include <pistache/eventmeth.h>
 #include <pistache/http.h>
@@ -929,6 +930,33 @@ namespace Pistache::Http
         }
 #endif
 
+#ifdef PISTACHE_USE_CONTENT_ENCODING_ZSTD
+
+        case Http::Header::Encoding::Zstd: {
+            // Get max compressed size
+            size_t estimated_size = ZSTD_compressBound(size);
+            // Allocate a smart buffer to contain compressed data...
+            std::unique_ptr compressedData = std::make_unique<std::byte[]>(estimated_size);
+
+            // Compress data using default compression level: https://raw.githack.com/facebook/zstd/release/doc/zstd_manual.html#Chapter3
+            auto compress_size = ZSTD_compress(reinterpret_cast<void*>(compressedData.get()), estimated_size,
+                                               data, size, ZSTD_defaultCLevel());
+            if (ZSTD_isError(compress_size))
+            {
+                throw std::runtime_error(
+                    std::string("failed to compress data to ZSTD on ZSTD_compress(), returning: ") + std::to_string(compress_size));
+            }
+            headers().add<Http::Header::ContentEncoding>(
+                Http::Header::Encoding::Zstd);
+
+            // Send compressed data back to client...
+            return putOnWire(
+                reinterpret_cast<const char*>(compressedData.get()),
+                compress_size);
+        }
+
+#endif
+
 #ifdef PISTACHE_USE_CONTENT_ENCODING_DEFLATE
         // User requested deflate compression...
         case Http::Header::Encoding::Deflate: {
@@ -1091,6 +1119,12 @@ namespace Pistache::Http
             break;
 #endif
 
+#ifdef PISTACHE_USE_CONTENT_ENCODING_ZSTD
+        case Http::Header::Encoding::Zstd:
+            contentEncoding_ = Http::Header::Encoding::Zstd;
+            break;
+#endif
+
 #ifdef PISTACHE_USE_CONTENT_ENCODING_DEFLATE
         // Application requested deflate compression...
         case Http::Header::Encoding::Deflate:

src/common/http_header.cc

@@ -53,6 +53,8 @@ namespace Pistache::Http::Header
             return "gzip";
         case Encoding::Br:
             return "br";
+        case Encoding::Zstd:
+            return "zstd";
         case Encoding::Compress:
             return "compress";
         case Encoding::Deflate:
@@ -74,6 +76,11 @@ namespace Pistache::Http::Header
             return Encoding::Unknown;
         }
 
+        if (!strncasecmp(str.data(), "zstd", str.length()))
+        {
+            return Encoding::Zstd;
+        }
+
         if (!strncasecmp(str.data(), "gzip", str.length()))
         {
             return Encoding::Gzip;
@@ -108,6 +115,11 @@ namespace Pistache::Http::Header
     {
         switch (encoding)
         {
+
+#ifdef PISTACHE_USE_CONTENT_ENCODING_ZSTD
+        case Encoding::Zstd:
+            /* @fallthrough@ */
+#endif
 #ifdef PISTACHE_USE_CONTENT_ENCODING_BROTLI
         case Encoding::Br:
             /* @fallthrough@ */

src/meson.build

@@ -45,6 +45,10 @@ if get_option('PISTACHE_USE_CONTENT_ENCODING_BROTLI')
 	public_args += '-DPISTACHE_USE_CONTENT_ENCODING_BROTLI'
 endif
 
+if get_option('PISTACHE_USE_CONTENT_ENCODING_ZSTD')
+	public_args += '-DPISTACHE_USE_CONTENT_ENCODING_ZSTD'
+endif
+
 if get_option('PISTACHE_USE_CONTENT_ENCODING_DEFLATE')
 	public_args += '-DPISTACHE_USE_CONTENT_ENCODING_DEFLATE'
 endif

tests/http_server_test.cc

@@ -25,6 +25,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef PISTACHE_USE_CONTENT_ENCODING_ZSTD
+#include <zstd.h>
+#endif
+
 #include <algorithm>
 #include <chrono>
 #include <condition_variable>
@@ -1297,6 +1301,12 @@ struct ContentEncodingHandler : public Http::Handler
         switch (encoding)
         {
 
+#ifdef PISTACHE_USE_CONTENT_ENCODING_ZSTD
+        case Http::Header::Encoding::Zstd:
+            writer.setCompressionZstdLevel(ZSTD_maxCLevel());
+            break;
+#endif
+
 #ifdef PISTACHE_USE_CONTENT_ENCODING_BROTLI
         // Set maximum compression if using Brotli
         case Http::Header::Encoding::Br:
@@ -1320,6 +1330,151 @@ struct ContentEncodingHandler : public Http::Handler
     }
 };
 
+#ifdef PISTACHE_USE_CONTENT_ENCODING_ZSTD
+TEST(http_server_test, server_with_content_encoding_zstd)
+{
+    { // encapsulate
+
+        // Data to send to server to expect it to return compressed...
+
+        // Allocate storage...
+        std::vector<std::byte> originalUncompressedData(1024);
+
+        // Random bytes engine...
+        using random_bytes_engine_type = std::independent_bits_engine<
+            std::default_random_engine, CHAR_BIT, unsigned char>;
+        random_bytes_engine_type randomEngine;
+
+        // Fill with random bytes...
+        std::generate(
+            std::begin(originalUncompressedData),
+            std::end(originalUncompressedData),
+            [&randomEngine]() { return static_cast<std::byte>(randomEngine()); });
+
+        // Bind server to localhost on a random port...
+        const Pistache::Address address("localhost", Pistache::Port(0));
+
+        // Initialize server...
+        Http::Endpoint server(address);
+        auto flags       = Tcp::Options::ReuseAddr;
+        auto server_opts = Http::Endpoint::options().flags(flags);
+        server_opts.maxRequestSize(1024 * 1024 * 20);
+        server_opts.maxResponseSize(1024 * 1024 * 20);
+        server.init(server_opts);
+        server.setHandler(Http::make_handler<ContentEncodingHandler>());
+        server.serveThreaded();
+
+        // Verify server is running...
+        ASSERT_TRUE(server.isBound());
+
+        // Log server coordinates...
+        const std::string server_address = "localhost:" + server.getPort().toString();
+        LOGGER("test", "Server address: " << server_address);
+
+        // Initialize client...
+
+        // Construct and initialize...
+        Http::Experimental::Client client;
+        client.init();
+
+        // Set server to connect to and get request builder object...
+        auto rb = client.get(server_address);
+
+        // Set data to send as body...
+        rb.body(
+            std::string(
+                reinterpret_cast<const char*>(originalUncompressedData.data()),
+                originalUncompressedData.size()));
+
+        // Request server send back response Zstd compressed...
+        rb.header<Http::Header::AcceptEncoding>(Http::Header::Encoding::Zstd);
+
+        // Send client request. Note that Transport::asyncSendRequestImpl() is
+        //  buggy, or at least with Pistache::Client, when the amount of data being
+        //  sent is large. When that happens send() breaks in asyncSendRequestImpl()
+        //  receiving an errno=EAGAIN...
+        auto response = rb.send();
+
+        // Storage for server response body...
+
+        std::string resultStringData;
+
+        // Verify response code, expected header, and store its body...
+        response.then(
+            [&resultStringData](Http::Response resp) {
+                // Log response code...
+                LOGGER("client", "Response code: " << resp.code());
+
+                // Log Content-Encoding header value, if present...
+                if (resp.headers().tryGetRaw("Content-Encoding").has_value())
+                {
+                    LOGGER("client", "Content-Encoding: " << resp.headers().tryGetRaw("Content-Encoding").value().value());
+                }
+
+                // Preserve body only if response code as expected...
+                if (resp.code() == Http::Code::Ok)
+                    resultStringData = resp.body();
+
+                // Get response headers...
+                const auto& headers = resp.headers();
+
+                // Verify Content-Encoding header was present...
+                ASSERT_TRUE(headers.has<Http::Header::ContentEncoding>());
+
+                // Verify Content-Encoding was set to Brotli...
+                const auto ce = headers.get<Http::Header::ContentEncoding>().get();
+                ASSERT_EQ(ce->encoding(), Http::Header::Encoding::Zstd);
+            },
+            Async::Throw);
+
+        // Wait for response to complete...
+        Async::Barrier<Http::Response> barrier(response);
+        barrier.wait();
+
+        // Cleanup client and server...
+        client.shutdown();
+        server.shutdown();
+
+        // Get server response body in vector...
+        std::vector<std::byte> newlyCompressedResponse(resultStringData.size());
+        std::transform(
+            std::cbegin(resultStringData),
+            std::cend(resultStringData),
+            std::begin(newlyCompressedResponse),
+            [](const char character) { return static_cast<std::byte>(character); });
+
+        // The data the server responded with should be compressed, and therefore
+        //  different from the original uncompressed sent during the request...
+        ASSERT_NE(originalUncompressedData, newlyCompressedResponse);
+
+        // Decompress response body...
+
+        // Storage for decompressed data...
+        std::vector<std::byte> newlyDecompressedData(
+            originalUncompressedData.size());
+
+        // Size of destination buffer, but will be updated by uncompress() to
+        //  actual size used...
+        size_t destinationLength = originalUncompressedData.size();
+
+        // Decompress...
+        const auto compressionStatus = ZSTD_getFrameContentSize(newlyDecompressedData.data(), newlyDecompressedData.size());
+
+        const auto decompressed_size = ZSTD_decompress((void*)newlyDecompressedData.data(), compressionStatus, newlyCompressedResponse.data(), newlyCompressedResponse.size());
+
+        ASSERT_EQ(ZSTD_isError(decompressed_size), 0);
+
+        // The sizes of both the original uncompressed data we sent the server
+        //  and the result of decompressing what it sent back should match...
+        ASSERT_EQ(originalUncompressedData.size(), destinationLength);
+
+        // Check to ensure the compressed data received back from server after
+        //  decompression matches exactly what we originally sent it...
+        ASSERT_EQ(originalUncompressedData, newlyDecompressedData);
+    }
+}
+#endif
+
 #ifdef PISTACHE_USE_CONTENT_ENCODING_BROTLI
 TEST(http_server_test, server_with_content_encoding_brotli)
 {