kram - add ZipStream and Perf

These are a gzip stream that can compress data passed to it.  Perfetto can take in gz and zip compressed files.  Gzip has a 10B header + 8B footer, and this uses miniz deflate calls.  Had to disable the default zlib header/footer, but would be interesting to support that too.  Use this to unpack and keep the original file, since this is compress only right now.   gzunzip -k file.gz

Needed to expose some of the thread calls for Perf to obtain the id and name.

kramv/KramViewerBase.cpp

@@ -1179,7 +1179,21 @@ bool Data::isArchive() const
 bool Data::loadFile()
 {
     if (isArchive()) {
-        return loadFileFromArchive();
+        // This test perf layer and the ZipStream
+        Perf* perf = nullptr; // Perf::instance();
+
+        // TODO: have to have permision to write file
+        if (perf) {
+            if (!perf->start("/Users/Alec/Library/Containers/com.hialec.kramv/Data/Traces/"
+                             "load.perftrace.gz"))
+                perf = nullptr;
+        }
+        bool success = loadFileFromArchive();
+
+        if (perf)
+            perf->stop();
+
+        return success;
     }
 
     // now lookup the filename and data at that entry
@@ -1379,6 +1393,8 @@ bool Data::loadFileFromArchive()
         return false;
     }
 
+    KPERFT("loadFileFromArchive");
+
     const uint8_t* imageData = nullptr;
     uint64_t imageDataLength = 0;
 
@@ -1391,13 +1407,17 @@ bool Data::loadFileFromArchive()
     vector<uint8_t> bufferForImage;
 
     if (isFileUncompressed) {
+        KPERFT("ZipExtractRaw");
+
         // search for main file - can be albedo or normal
         if (!zip.extractRaw(filename, &imageData, imageDataLength)) {
             return false;
         }
 
     }
     else {
+        KPERFT("ZipExtract");
+
         // need to decompress first
         if (!zip.extract(filename, bufferForImage)) {
             return false;
@@ -1430,10 +1450,14 @@ bool Data::loadFileFromArchive()
                 bool isNormalUncompressed = normalEntry->compressedSize == entry->uncompressedSize;
 
                 if (isNormalUncompressed) {
+                    KPERFT("ZipExtractRawNormal");
+
                     zip.extractRaw(name.c_str(), &imageNormalData,
                                    imageNormalDataLength);
                 }
                 else {
+                    KPERFT("ZipExtractNormal");
+
                     // need to decompress first
                     if (!zip.extract(filename, bufferForNormal)) {
                         return false;
@@ -1459,29 +1483,43 @@ bool Data::loadFileFromArchive()
 
     // TODO: do imageDiff here?
 
+    KPERFT_START(1, "KTXOpen");
+
     if (!imageDataKTX.open(imageData, imageDataLength, image)) {
         return false;
     }
 
-    if (hasNormal && imageNormalDataKTX.open(
-                         imageNormalData, imageNormalDataLength, imageNormal)) {
-        // shaders only pull from albedo + normal on these texture types
-        if (imageNormal.textureType == image.textureType &&
-            (imageNormal.textureType == MyMTLTextureType2D ||
-             imageNormal.textureType == MyMTLTextureType2DArray)) {
-            // hasNormal = true;
-        }
-        else {
-            hasNormal = false;
-        }
+    KPERFT_STOP(1);
+
+
+    if (hasNormal) {
+        KPERFT("KTXOpenNormal");
+
+        if (imageNormalDataKTX.open(
+            imageNormalData, imageNormalDataLength, imageNormal)) {
+                // shaders only pull from albedo + normal on these texture types
+                if (imageNormal.textureType == image.textureType &&
+                    (imageNormal.textureType == MyMTLTextureType2D ||
+                     imageNormal.textureType == MyMTLTextureType2DArray)) {
+                    // hasNormal = true;
+                }
+                else {
+                    hasNormal = false;
+                }
+            }
     }
 
+
     //---------------------------------
 
+    KPERFT_START(3, "KTXLoad");
+
     if (!_delegate.loadTextureFromImage(fullFilename.c_str(), (double)timestamp, image, hasNormal ? &imageNormal : nullptr, nullptr, true)) {
         return false;
     }
 
+    KPERFT_STOP(3);
+
     //---------------------------------
 
    // NSArray<NSURL*>* urls_ = (NSArray<NSURL*>*)_delegate._urls;

libkram/json11/json11.cpp

@@ -284,7 +284,7 @@ void JsonWriter::writeFormat(const char* fmt, ...) {
     if (_stream && _out->size() >= _stream->compressLimit())
     {
         // flush the output to a compression stream
-        _stream->compress(Slice((uint8_t*)_out->data(), _out->size())); // losing const
+        _stream->compress(Slice((uint8_t*)_out->data(), _out->size()), false); // losing const
 
         // reset the buffer
         _out->clear();
@@ -295,7 +295,7 @@ JsonWriter::~JsonWriter()
 {
     if (_stream)  {
         if (!_out->empty()) {
-            _stream->compress(Slice((uint8_t*)_out->data(), _out->size())); // losing const
+            _stream->compress(Slice((uint8_t*)_out->data(), _out->size()), true); // losing const
         }
     }
 }

libkram/kram/Kram.cpp

@@ -919,7 +919,7 @@ bool SavePNG(Image& image, const char* filename)
 
 bool SetupTmpFile(FileHelper& tmpFileHelper, const char* suffix)
 {
-    return tmpFileHelper.openTemporaryFile(suffix, "w+b");
+    return tmpFileHelper.openTemporaryFile("kramimage-", suffix, "w+b");
 }
 
 bool SetupSourceImage(const string& srcFilename, Image& sourceImage,

libkram/kram/KramFileHelper.cpp

@@ -65,7 +65,7 @@ static FILE* fopen_mkdir(const char* path, const char* mode)
 FileHelper::~FileHelper() { close(); }
 
 // no current extension
-bool FileHelper::openTemporaryFile(const char* suffix, const char* access)
+bool FileHelper::openTemporaryFile(const char* prefix, const char* suffix, const char* access)
 {
     close();
 
@@ -82,7 +82,7 @@ bool FileHelper::openTemporaryFile(const char* suffix, const char* access)
     int keep = 0;
 
     // Note: can't pass . either, always opened as rw
-    _fp = tmpfileplus("/tmp/", "kramimage-", suffix, &pathname, keep);
+    _fp = tmpfileplus("/tmp/", prefix, suffix, &pathname, keep);
     if (!_fp) {
         return false;
     }
@@ -139,8 +139,10 @@ size_t FileHelper::pagesize()
 
 bool FileHelper::copyTemporaryFileTo(const char* dstFilename)
 {
-    if (!_fp) return false;
-    if (_filename.empty()) return false;
+    if (!_fp) 
+        return false;
+    if (_filename.empty()) 
+        return false;
 
     // since we're not closing, need to flush output
     fflush(_fp);

libkram/kram/KramFileHelper.h

@@ -29,7 +29,7 @@ class FileHelper {
     bool open(const char* filename, const char* access);
 
     // this file is auto-deleted by close(), is that okay with renameFile use?
-    bool openTemporaryFile(const char* suffix, const char* access);
+    bool openTemporaryFile(const char* prefix, const char* suffix, const char* access);
 
     // mainly for tmp files, file can be closed, but this does rename of tmp file.
     // may fail if tmp file and dst are different volumes.

libkram/kram/KramLog.cpp

@@ -43,13 +43,10 @@
 #include "KramFmt.h"
 #include "KramTimer.h"
 #include "format.h" // really fmt/format.h
+#include "TaskSystem.h"
 
 namespace kram {
 
-// Pulled in from TaskSystem.cpp
-constexpr const uint32_t kMaxThreadName = 32;
-extern void getCurrentThreadName(char name[kMaxThreadName]);
-
 using mymutex = std::recursive_mutex;
 using mylock = std::unique_lock<mymutex>;