Implement network serializer (rebased)

libraries/gpu/src/gpu/Texture.h

@@ -24,6 +24,7 @@
 #include "Forward.h"
 #include "Resource.h"
 #include "Metric.h"
+#include "SerDes.h"
 
 const int ABSOLUTE_MAX_TEXTURE_NUM_PIXELS = 8192 * 8192;
 
@@ -91,6 +92,37 @@ class SphericalHarmonics {
 };
 typedef std::shared_ptr< SphericalHarmonics > SHPointer;
 
+
+inline DataSerializer &operator<<(DataSerializer &ser, const SphericalHarmonics &h) {
+    DataSerializer::SizeTracker tracker(ser);
+
+    ser << h.L00  << h.spare0;
+    ser << h.L1m1 << h.spare1;
+    ser << h.L10  << h.spare2;
+    ser << h.L11  << h.spare3;
+    ser << h.L2m2 << h.spare4;
+    ser << h.L2m1 << h.spare5;
+    ser << h.L20  << h.spare6;
+    ser << h.L21  << h.spare7;
+    ser << h.L22  << h.spare8;
+    return ser;
+}
+
+inline DataDeserializer &operator>>(DataDeserializer &des, SphericalHarmonics &h) {
+    DataDeserializer::SizeTracker tracker(des);
+
+    des >> h.L00  >> h.spare0;
+    des >> h.L1m1 >> h.spare1;
+    des >> h.L10  >> h.spare2;
+    des >> h.L11  >> h.spare3;
+    des >> h.L2m2 >> h.spare4;
+    des >> h.L2m1 >> h.spare5;
+    des >> h.L20  >> h.spare6;
+    des >> h.L21  >> h.spare7;
+    des >> h.L22  >> h.spare8;
+    return des;
+}
+
 class Sampler {
 public:
 
@@ -136,7 +168,7 @@ class Sampler {
         uint8 _wrapModeU = WRAP_REPEAT;
         uint8 _wrapModeV = WRAP_REPEAT;
         uint8 _wrapModeW = WRAP_REPEAT;
-            
+
         uint8 _mipOffset = 0;
         uint8 _minMip = 0;
         uint8 _maxMip = MAX_MIP_LEVEL;
@@ -193,6 +225,35 @@ class Sampler {
     friend class Deserializer;
 };
 
+inline DataSerializer &operator<<(DataSerializer &ser, const Sampler::Desc &d) {
+    DataSerializer::SizeTracker tracker(ser);
+    ser << d._borderColor;
+    ser << d._maxAnisotropy;
+    ser << d._filter;
+    ser << d._comparisonFunc;
+    ser << d._wrapModeU;
+    ser << d._wrapModeV;
+    ser << d._wrapModeW;
+    ser << d._mipOffset;
+    ser << d._minMip;
+    ser << d._maxMip;
+    return ser;
+}
+
+inline DataDeserializer &operator>>(DataDeserializer &dsr, Sampler::Desc &d) {
+    DataDeserializer::SizeTracker tracker(dsr);
+    dsr >> d._borderColor;
+    dsr >> d._maxAnisotropy;
+    dsr >> d._filter;
+    dsr >> d._comparisonFunc;
+    dsr >> d._wrapModeU;
+    dsr >> d._wrapModeV;
+    dsr >> d._wrapModeW;
+    dsr >> d._mipOffset;
+    dsr >> d._minMip;
+    dsr >> d._maxMip;
+    return dsr;
+}
 enum class TextureUsageType : uint8 {
     RENDERBUFFER,       // Used as attachments to a framebuffer
     RESOURCE,           // Resource textures, like materials... subject to memory manipulation
@@ -230,7 +291,7 @@ class Texture : public Resource {
             NORMAL,      // Texture is a normal map
             ALPHA,      // Texture has an alpha channel
             ALPHA_MASK,       // Texture alpha channel is a Mask 0/1
-            NUM_FLAGS,  
+            NUM_FLAGS,
         };
 
         typedef std::bitset<NUM_FLAGS> Flags;
@@ -478,7 +539,7 @@ class Texture : public Resource {
     uint16 evalMipDepth(uint16 level) const { return std::max(_depth >> level, 1); }
 
     // The true size of an image line or surface depends on the format, tiling and padding rules
-    // 
+    //
     // Here are the static function to compute the different sizes from parametered dimensions and format
     // Tile size must be a power of 2
     static uint16 evalTiledPadding(uint16 length, int tile) { int tileMinusOne = (tile - 1); return (tileMinusOne - (length + tileMinusOne) % tile); }
@@ -507,7 +568,7 @@ class Texture : public Resource {
     uint32 evalMipFaceNumTexels(uint16 level) const { return evalMipWidth(level) * evalMipHeight(level) * evalMipDepth(level); }
     uint32 evalMipNumTexels(uint16 level) const { return evalMipFaceNumTexels(level) * getNumFaces(); }
 
-    // For convenience assign a source name 
+    // For convenience assign a source name
     const std::string& source() const { return _source; }
     void setSource(const std::string& source) { _source = source; }
     const std::string& sourceHash() const { return _sourceHash; }
@@ -633,7 +694,7 @@ class Texture : public Resource {
     uint16 _maxMipLevel { 0 };
 
     uint16 _minMip { 0 };
- 
+
     Type _type { TEX_1D };
 
     Usage _usage;
@@ -643,7 +704,7 @@ class Texture : public Resource {
     bool _isIrradianceValid = false;
     bool _defined = false;
     bool _important = false;
-   
+
     static TexturePointer create(TextureUsageType usageType, Type type, const Element& texelFormat, uint16 width, uint16 height, uint16 depth, uint16 numSamples, uint16 numSlices, uint16 numMips, const Sampler& sampler);
 
     Size resize(Type type, const Element& texelFormat, uint16 width, uint16 height, uint16 depth, uint16 numSamples, uint16 numSlices, uint16 numMips);

libraries/gpu/src/gpu/Texture_ktx.cpp

@@ -18,6 +18,7 @@
 #include <ktx/KTX.h>
 
 #include "GPULogging.h"
+#include "SerDes.h"
 
 using namespace gpu;
 
@@ -27,71 +28,94 @@ using KtxStorage = Texture::KtxStorage;
 std::vector<std::pair<std::shared_ptr<storage::FileStorage>, std::shared_ptr<std::mutex>>> KtxStorage::_cachedKtxFiles;
 std::mutex KtxStorage::_cachedKtxFilesMutex;
 
+
+/**
+ * @brief Payload for a KTX (texture)
+ *
+ * This contains a ready to use texture. This is both used for the local cache, and for baked textures.
+ *
+ * @note The usage for textures means breaking compatibility is a bad idea, and that the implementation
+ * should just keep on adding extra data at the bottom of the structure, and remain able to read old
+ * formats. In fact, version 1 KTX can be found in older baked assets.
+ */
 struct GPUKTXPayload {
     using Version = uint8;
 
     static const std::string KEY;
     static const Version CURRENT_VERSION { 2 };
     static const size_t PADDING { 2 };
-    static const size_t SIZE { sizeof(Version) + sizeof(Sampler::Desc) + sizeof(uint32) + sizeof(TextureUsageType) + sizeof(glm::ivec2) + PADDING };
+    static const size_t SIZE { sizeof(Version) + sizeof(Sampler::Desc) + sizeof(uint32_t) + sizeof(TextureUsageType) + sizeof(glm::ivec2) + PADDING };
+
     static_assert(GPUKTXPayload::SIZE == 44, "Packing size may differ between platforms");
-    static_assert(GPUKTXPayload::SIZE % 4 == 0, "GPUKTXPayload is not 4 bytes aligned");
 
     Sampler::Desc _samplerDesc;
     Texture::Usage _usage;
     TextureUsageType _usageType;
     glm::ivec2 _originalSize { 0, 0 };
 
-    Byte* serialize(Byte* data) const {
-        *(Version*)data = CURRENT_VERSION;
-        data += sizeof(Version);
+    /**
+     * @brief Serialize the KTX payload
+     *
+     * @warning Be careful modifying this code, as it influences baked assets.
+     * Backwards compatibility must be maintained.
+     *
+     * @param ser Destination serializer
+     */
+    void serialize(DataSerializer &ser) {
 
-        memcpy(data, &_samplerDesc, sizeof(Sampler::Desc));
-        data += sizeof(Sampler::Desc);
+        ser << CURRENT_VERSION;
 
-        // We can't copy the bitset in Texture::Usage in a crossplateform manner
-        // So serialize it manually
-        uint32 usageData = _usage._flags.to_ulong();
-        memcpy(data, &usageData, sizeof(uint32));
-        data += sizeof(uint32);
+        ser << _samplerDesc;
 
-        memcpy(data, &_usageType, sizeof(TextureUsageType));
-        data += sizeof(TextureUsageType);
+        uint32_t usageData = (uint32_t)_usage._flags.to_ulong();
+        ser << usageData;
+        ser << ((uint8_t)_usageType);
+        ser << _originalSize;
 
-        memcpy(data, glm::value_ptr(_originalSize), sizeof(glm::ivec2));
-        data += sizeof(glm::ivec2);
+        ser.addPadding(PADDING);
 
-        return data + PADDING;
+        assert(ser.length() == GPUKTXPayload::SIZE);
     }
 
-    bool unserialize(const Byte* data, size_t size) {
-        Version version = *(const Version*)data;
-        data += sizeof(Version);
+    /**
+     * @brief Deserialize the KTX payload
+     *
+     * @warning Be careful modifying this code, as it influences baked assets.
+     * Backwards compatibility must be maintained.
+     *
+     * @param dsr Deserializer object
+     * @return true Successful
+     * @return false Version check failed
+     */
+    bool unserialize(DataDeserializer &dsr) {
+        Version version = 0;
+        uint32_t usageData = 0;
+        uint8_t usagetype = 0;
+
+        dsr >> version;
 
         if (version > CURRENT_VERSION) {
             // If we try to load a version that we don't know how to parse,
             // it will render incorrectly
+            qCWarning(gpulogging) << "KTX version" << version << "is newer than our own," << CURRENT_VERSION;
+            qCWarning(gpulogging) << dsr;
             return false;
         }
 
-        memcpy(&_samplerDesc, data, sizeof(Sampler::Desc));
-        data += sizeof(Sampler::Desc);
+        dsr >> _samplerDesc;
 
-        // We can't copy the bitset in Texture::Usage in a crossplateform manner
-        // So unserialize it manually
-        uint32 usageData;
-        memcpy(&usageData, data, sizeof(uint32));
-        _usage = Texture::Usage(usageData);
-        data += sizeof(uint32);
+        dsr >> usageData;
+        _usage = gpu::Texture::Usage(usageData);
 
-        memcpy(&_usageType, data, sizeof(TextureUsageType));
-        data += sizeof(TextureUsageType);
+        dsr >> usagetype;
+        _usageType = (TextureUsageType)usagetype;
 
         if (version >= 2) {
-            memcpy(&_originalSize, data, sizeof(glm::ivec2));
-            data += sizeof(glm::ivec2);
+            dsr >> _originalSize;
         }
 
+        dsr.skipPadding(PADDING);
+
         return true;
     }
 
@@ -103,7 +127,8 @@ struct GPUKTXPayload {
         auto found = std::find_if(keyValues.begin(), keyValues.end(), isGPUKTX);
         if (found != keyValues.end()) {
             auto value = found->_value;
-            return payload.unserialize(value.data(), value.size());
+            DataDeserializer dsr(value.data(), value.size());
+            return payload.unserialize(dsr);
         }
         return false;
     }
@@ -123,29 +148,24 @@ struct IrradianceKTXPayload {
 
     SphericalHarmonics _irradianceSH;
 
-    Byte* serialize(Byte* data) const {
-        *(Version*)data = CURRENT_VERSION;
-        data += sizeof(Version);
-
-        memcpy(data, &_irradianceSH, sizeof(SphericalHarmonics));
-        data += sizeof(SphericalHarmonics);
-
-        return data + PADDING;
+    void serialize(DataSerializer &ser) const {
+        ser << CURRENT_VERSION;
+        ser << _irradianceSH;
+        ser.addPadding(PADDING);
     }
 
-    bool unserialize(const Byte* data, size_t size) {
-        if (size != SIZE) {
+    bool unserialize(DataDeserializer &des) {
+        Version version;
+        if (des.length() != SIZE) {
             return false;
         }
 
-        Version version = *(const Version*)data;
+        des >> version;
         if (version != CURRENT_VERSION) {
             return false;
         }
-        data += sizeof(Version);
-
-        memcpy(&_irradianceSH, data, sizeof(SphericalHarmonics));
 
+        des >> _irradianceSH;
         return true;
     }
 
@@ -157,7 +177,8 @@ struct IrradianceKTXPayload {
         auto found = std::find_if(keyValues.begin(), keyValues.end(), isIrradianceKTX);
         if (found != keyValues.end()) {
             auto value = found->_value;
-            return payload.unserialize(value.data(), value.size());
+            DataDeserializer des(value.data(), value.size());
+            return payload.unserialize(des);
         }
         return false;
     }
@@ -467,7 +488,9 @@ ktx::KTXUniquePointer Texture::serialize(const Texture& texture, const glm::ivec
     gpuKeyval._originalSize = originalSize;
 
     Byte keyvalPayload[GPUKTXPayload::SIZE];
-    gpuKeyval.serialize(keyvalPayload);
+    DataSerializer ser(keyvalPayload, sizeof(keyvalPayload));
+
+    gpuKeyval.serialize(ser);
 
     ktx::KeyValues keyValues;
     keyValues.emplace_back(GPUKTXPayload::KEY, (uint32)GPUKTXPayload::SIZE, (ktx::Byte*) &keyvalPayload);
@@ -477,7 +500,8 @@ ktx::KTXUniquePointer Texture::serialize(const Texture& texture, const glm::ivec
         irradianceKeyval._irradianceSH = *texture.getIrradiance();
 
         Byte irradianceKeyvalPayload[IrradianceKTXPayload::SIZE];
-        irradianceKeyval.serialize(irradianceKeyvalPayload);
+        DataSerializer ser(irradianceKeyvalPayload, sizeof(irradianceKeyvalPayload));
+        irradianceKeyval.serialize(ser);
 
         keyValues.emplace_back(IrradianceKTXPayload::KEY, (uint32)IrradianceKTXPayload::SIZE, (ktx::Byte*) &irradianceKeyvalPayload);
     }

libraries/octree/src/OctreePacketData.cpp

@@ -17,6 +17,7 @@
 #include "OctreeLogging.h"
 #include "NumericalConstants.h"
 #include <glm/gtc/type_ptr.hpp>
+#include "SerDes.h"
 
 bool OctreePacketData::_debug = false;
 AtomicUIntStat OctreePacketData::_totalBytesOfOctalCodes { 0 };
@@ -847,10 +848,10 @@ int OctreePacketData::unpackDataFromBytes(const unsigned char* dataBytes, QByteA
 }
 
 int OctreePacketData::unpackDataFromBytes(const unsigned char* dataBytes, AACube& result) {
-    aaCubeData cube;
-    memcpy(&cube, dataBytes, sizeof(aaCubeData));
-    result = AACube(cube.corner, cube.scale);
-    return sizeof(aaCubeData);
+    DataDeserializer des(dataBytes, sizeof(aaCubeData));
+    des >> result;
+
+    return des.length();
 }
 
 int OctreePacketData::unpackDataFromBytes(const unsigned char* dataBytes, QRect& result) {