From cd8ad23dd1860e7118a4da42d53381574c72f3a7 Mon Sep 17 00:00:00 2001 From: ksuprynowicz Date: Tue, 5 Mar 2024 01:09:57 +0100 Subject: [PATCH 1/4] Fix invalid reference --- assignment-client/src/avatars/ScriptableAvatar.cpp | 9 +++++---- libraries/ui/src/QmlWindowClass.cpp | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/assignment-client/src/avatars/ScriptableAvatar.cpp b/assignment-client/src/avatars/ScriptableAvatar.cpp index c919d2a2d73..8a15f05bf11 100644 --- a/assignment-client/src/avatars/ScriptableAvatar.cpp +++ b/assignment-client/src/avatars/ScriptableAvatar.cpp @@ -138,7 +138,8 @@ static AnimPose composeAnimPose(const HFMJoint& joint, const glm::quat rotation, void ScriptableAvatar::update(float deltatime) { // Run animation - if (_animation && _animation->isLoaded() && _animation->getFrames().size() > 0 && !_bind.isNull() && _bind->isLoaded()) { + auto frames = _animation->getFrames(); + if (_animation && _animation->isLoaded() && frames.size() > 0 && !_bind.isNull() && _bind->isLoaded()) { if (!_animSkeleton) { _animSkeleton = std::make_shared(_bind->getHFMModel()); } @@ -157,9 +158,9 @@ void ScriptableAvatar::update(float deltatime) { _jointData.resize(nJoints); } - const int frameCount = _animation->getFrames().size(); - const HFMAnimationFrame& floorFrame = _animation->getFrames().at((int)glm::floor(currentFrame) % frameCount); - const HFMAnimationFrame& ceilFrame = _animation->getFrames().at((int)glm::ceil(currentFrame) % frameCount); + const int frameCount = frames.size(); + const HFMAnimationFrame& floorFrame = frames.at((int)glm::floor(currentFrame) % frameCount); + const HFMAnimationFrame& ceilFrame = frames.at((int)glm::ceil(currentFrame) % frameCount); const float frameFraction = glm::fract(currentFrame); std::vector poses = _animSkeleton->getRelativeDefaultPoses(); diff --git a/libraries/ui/src/QmlWindowClass.cpp b/libraries/ui/src/QmlWindowClass.cpp index 474a8f467d4..74734fdc43c 100644 --- a/libraries/ui/src/QmlWindowClass.cpp +++ b/libraries/ui/src/QmlWindowClass.cpp @@ -279,6 +279,7 @@ bool QmlWindowClass::isVisible() { return quickItem->isVisible(); } else { qDebug() << "QmlWindowClass::isVisible: asQuickItem() returned NULL"; + return false; } } From 805cd78728b975e01e8d88c7c7fbc03652934563 Mon Sep 17 00:00:00 2001 From: ksuprynowicz Date: Wed, 6 Mar 2024 00:35:19 +0100 Subject: [PATCH 2/4] Switched to animation frames reference --- assignment-client/src/avatars/ScriptableAvatar.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/avatars/ScriptableAvatar.cpp b/assignment-client/src/avatars/ScriptableAvatar.cpp index 8a15f05bf11..f321e3c48fb 100644 --- a/assignment-client/src/avatars/ScriptableAvatar.cpp +++ b/assignment-client/src/avatars/ScriptableAvatar.cpp @@ -138,7 +138,9 @@ static AnimPose composeAnimPose(const HFMJoint& joint, const glm::quat rotation, void ScriptableAvatar::update(float deltatime) { // Run animation - auto frames = _animation->getFrames(); + Q_ASSERT(QThread::currentThread() == thread()); + Q_ASSERT(thread() == _animation->thread()); + auto frames = _animation->getFramesReference(); if (_animation && _animation->isLoaded() && frames.size() > 0 && !_bind.isNull() && _bind->isLoaded()) { if (!_animSkeleton) { _animSkeleton = std::make_shared(_bind->getHFMModel()); From 78978c709722b30021a2bdd5caaff50cafdb568c Mon Sep 17 00:00:00 2001 From: ksuprynowicz Date: Fri, 8 Mar 2024 17:28:07 +0100 Subject: [PATCH 3/4] Initial fix for animations --- assignment-client/src/Agent.cpp | 2 + .../src/avatars/ScriptableAvatar.cpp | 141 +++++++++++------- .../src/avatars/ScriptableAvatar.h | 12 +- 3 files changed, 96 insertions(+), 59 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 246fd22cbb0..3ac8eaaff79 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -108,6 +108,7 @@ Agent::Agent(ReceivedMessage& message) : DependencyManager::set(); DependencyManager::set(); + DependencyManager::set(); // Needed to ensure the creation of the DebugDraw instance on the main thread DebugDraw::getInstance(); @@ -848,6 +849,7 @@ void Agent::aboutToFinish() { DependencyManager::destroy(); DependencyManager::destroy(); + DependencyManager::destroy(); DependencyManager::destroy(); diff --git a/assignment-client/src/avatars/ScriptableAvatar.cpp b/assignment-client/src/avatars/ScriptableAvatar.cpp index f321e3c48fb..3a3b2cf517a 100644 --- a/assignment-client/src/avatars/ScriptableAvatar.cpp +++ b/assignment-client/src/avatars/ScriptableAvatar.cpp @@ -32,6 +32,10 @@ ScriptableAvatar::ScriptableAvatar(): _scriptEngine(newScriptEngine()) { _clientTraitsHandler.reset(new ClientTraitsHandler(this)); + static std::once_flag once; + std::call_once(once, [] { + qRegisterMetaType("HFMModel::Pointer"); + }); } QByteArray ScriptableAvatar::toByteArrayStateful(AvatarDataDetail dataDetail, bool dropFaceTracking) { @@ -52,6 +56,7 @@ void ScriptableAvatar::startAnimation(const QString& url, float fps, float prior _animation = DependencyManager::get()->getAnimation(url); _animationDetails = AnimationDetails("", QUrl(url), fps, 0, loop, hold, false, firstFrame, lastFrame, true, firstFrame, false); _maskedJoints = maskedJoints; + _isAnimationRigValid = false; } void ScriptableAvatar::stopAnimation() { @@ -89,11 +94,12 @@ QStringList ScriptableAvatar::getJointNames() const { } void ScriptableAvatar::setSkeletonModelURL(const QUrl& skeletonModelURL) { - _bind.reset(); - _animSkeleton.reset(); + _avatarAnimSkeleton.reset(); + _geometryResource.reset(); AvatarData::setSkeletonModelURL(skeletonModelURL); updateJointMappings(); + _isRigValid = false; } int ScriptableAvatar::sendAvatarDataPacket(bool sendAll) { @@ -137,68 +143,90 @@ static AnimPose composeAnimPose(const HFMJoint& joint, const glm::quat rotation, } void ScriptableAvatar::update(float deltatime) { + // TODO: the current decision to use geometry resource results in loading textures, but it works way better + // than previous choice of loading avatar as animation, which was missing data such as joint names hash, + // and also didn't support glTF models. Optimizing this will be left fot future PR. + if (!_geometryResource && !_skeletonModelFilenameURL.isEmpty()) { // AvatarData will parse the .fst, but not get the .fbx skeleton. + _geometryResource = DependencyManager::get()->getGeometryResource(_skeletonModelFilenameURL); + } + // Run animation Q_ASSERT(QThread::currentThread() == thread()); - Q_ASSERT(thread() == _animation->thread()); - auto frames = _animation->getFramesReference(); - if (_animation && _animation->isLoaded() && frames.size() > 0 && !_bind.isNull() && _bind->isLoaded()) { - if (!_animSkeleton) { - _animSkeleton = std::make_shared(_bind->getHFMModel()); - } - float currentFrame = _animationDetails.currentFrame + deltatime * _animationDetails.fps; - if (_animationDetails.loop || currentFrame < _animationDetails.lastFrame) { - while (currentFrame >= _animationDetails.lastFrame) { - currentFrame -= (_animationDetails.lastFrame - _animationDetails.firstFrame); + if (_animation && _animation->isLoaded()) { + Q_ASSERT(thread() == _animation->thread()); + auto frames = _animation->getFramesReference(); + if (frames.size() > 0 && _geometryResource && _geometryResource->isHFMModelLoaded()) { + if (!_isRigValid) { + _rig.reset(_geometryResource->getHFMModel()); + _isRigValid = true; } - _animationDetails.currentFrame = currentFrame; - - const QVector& modelJoints = _bind->getHFMModel().joints; - QStringList animationJointNames = _animation->getJointNames(); - - const int nJoints = modelJoints.size(); - if (_jointData.size() != nJoints) { - _jointData.resize(nJoints); + if (!_isAnimationRigValid) { + _animationRig.reset(_animation->getHFMModel()); + _isAnimationRigValid = true; } - - const int frameCount = frames.size(); - const HFMAnimationFrame& floorFrame = frames.at((int)glm::floor(currentFrame) % frameCount); - const HFMAnimationFrame& ceilFrame = frames.at((int)glm::ceil(currentFrame) % frameCount); - const float frameFraction = glm::fract(currentFrame); - std::vector poses = _animSkeleton->getRelativeDefaultPoses(); - - const float UNIT_SCALE = 0.01f; - - for (int i = 0; i < animationJointNames.size(); i++) { - const QString& name = animationJointNames[i]; - // As long as we need the model preRotations anyway, let's get the jointIndex from the bind skeleton rather than - // trusting the .fst (which is sometimes not updated to match changes to .fbx). - int mapping = _bind->getHFMModel().getJointIndex(name); - if (mapping != -1 && !_maskedJoints.contains(name)) { - - AnimPose floorPose = composeAnimPose(modelJoints[mapping], floorFrame.rotations[i], floorFrame.translations[i] * UNIT_SCALE); - AnimPose ceilPose = composeAnimPose(modelJoints[mapping], ceilFrame.rotations[i], floorFrame.translations[i] * UNIT_SCALE); - blend(1, &floorPose, &ceilPose, frameFraction, &poses[mapping]); - } + if (!_avatarAnimSkeleton) { + _avatarAnimSkeleton = std::make_shared(_geometryResource->getHFMModel()); } + float currentFrame = _animationDetails.currentFrame + deltatime * _animationDetails.fps; + if (_animationDetails.loop || currentFrame < _animationDetails.lastFrame) { + while (currentFrame >= _animationDetails.lastFrame) { + currentFrame -= (_animationDetails.lastFrame - _animationDetails.firstFrame); + } + _animationDetails.currentFrame = currentFrame; + + const QVector& modelJoints = _geometryResource->getHFMModel().joints; + QStringList animationJointNames = _animation->getJointNames(); - std::vector absPoses = poses; - _animSkeleton->convertRelativePosesToAbsolute(absPoses); - for (int i = 0; i < nJoints; i++) { - JointData& data = _jointData[i]; - AnimPose& absPose = absPoses[i]; - if (data.rotation != absPose.rot()) { - data.rotation = absPose.rot(); - data.rotationIsDefaultPose = false; + const int nJoints = modelJoints.size(); + if (_jointData.size() != nJoints) { + _jointData.resize(nJoints); } - AnimPose& relPose = poses[i]; - if (data.translation != relPose.trans()) { - data.translation = relPose.trans(); - data.translationIsDefaultPose = false; + + const int frameCount = frames.size(); + const HFMAnimationFrame& floorFrame = frames.at((int)glm::floor(currentFrame) % frameCount); + const HFMAnimationFrame& ceilFrame = frames.at((int)glm::ceil(currentFrame) % frameCount); + const float frameFraction = glm::fract(currentFrame); + std::vector poses = _avatarAnimSkeleton->getRelativeDefaultPoses(); + + // TODO: this needs more testing, it's possible that we need not only scale but also rotation and translation + // According to tests with unmatching avatar and animation armatures, sometimes bones are not rotated correctly. + // Matching armatures already work very well now. + const float UNIT_SCALE = _animationRig.GetScaleFactorGeometryToUnscaledRig() / _rig.GetScaleFactorGeometryToUnscaledRig(); + + for (int i = 0; i < animationJointNames.size(); i++) { + const QString& name = animationJointNames[i]; + // As long as we need the model preRotations anyway, let's get the jointIndex from the bind skeleton rather than + // trusting the .fst (which is sometimes not updated to match changes to .fbx). + int mapping = _geometryResource->getHFMModel().getJointIndex(name); + if (mapping != -1 && !_maskedJoints.contains(name)) { + AnimPose floorPose = composeAnimPose(modelJoints[mapping], floorFrame.rotations[i], + floorFrame.translations[i] * UNIT_SCALE); + AnimPose ceilPose = composeAnimPose(modelJoints[mapping], ceilFrame.rotations[i], + ceilFrame.translations[i] * UNIT_SCALE); + blend(1, &floorPose, &ceilPose, frameFraction, &poses[mapping]); + } } - } - } else { - _animation.clear(); + std::vector absPoses = poses; + Q_ASSERT(_avatarAnimSkeleton != nullptr); + _avatarAnimSkeleton->convertRelativePosesToAbsolute(absPoses); + for (int i = 0; i < nJoints; i++) { + JointData& data = _jointData[i]; + AnimPose& absPose = absPoses[i]; + if (data.rotation != absPose.rot()) { + data.rotation = absPose.rot(); + data.rotationIsDefaultPose = false; + } + AnimPose& relPose = poses[i]; + if (data.translation != relPose.trans()) { + data.translation = relPose.trans(); + data.translationIsDefaultPose = false; + } + } + + } else { + _animation.clear(); + } } } @@ -248,6 +276,7 @@ void ScriptableAvatar::setJointMappingsFromNetworkReply() { networkReply->deleteLater(); return; } + // TODO: this works only with .fst files currently, not directly with FBX and GLB models { QWriteLocker writeLock(&_jointDataLock); QByteArray line; @@ -256,7 +285,7 @@ void ScriptableAvatar::setJointMappingsFromNetworkReply() { if (line.startsWith("filename")) { int filenameIndex = line.indexOf('=') + 1; if (filenameIndex > 0) { - _skeletonFBXURL = _skeletonModelURL.resolved(QString(line.mid(filenameIndex).trimmed())); + _skeletonModelFilenameURL = _skeletonModelURL.resolved(QString(line.mid(filenameIndex).trimmed())); } } if (!line.startsWith("jointIndex")) { diff --git a/assignment-client/src/avatars/ScriptableAvatar.h b/assignment-client/src/avatars/ScriptableAvatar.h index 703a0a9f64c..192de31a269 100644 --- a/assignment-client/src/avatars/ScriptableAvatar.h +++ b/assignment-client/src/avatars/ScriptableAvatar.h @@ -19,6 +19,8 @@ #include #include #include +#include "model-networking/ModelCache.h" +#include "Rig.h" /*@jsdoc * The Avatar API is used to manipulate scriptable avatars on the domain. This API is a subset of the @@ -217,11 +219,15 @@ public slots: AnimationPointer _animation; AnimationDetails _animationDetails; QStringList _maskedJoints; - AnimationPointer _bind; // a sleazy way to get the skeleton, given the various library/cmake dependencies - std::shared_ptr _animSkeleton; + GeometryResource::Pointer _geometryResource; + Rig _rig; + bool _isRigValid{false}; + Rig _animationRig; + bool _isAnimationRigValid{false}; + std::shared_ptr _avatarAnimSkeleton; QHash _fstJointIndices; ///< 1-based, since zero is returned for missing keys QStringList _fstJointNames; ///< in order of depth-first traversal - QUrl _skeletonFBXURL; + QUrl _skeletonModelFilenameURL; // This contains URL from filename field in fst file mutable ScriptEnginePointer _scriptEngine; std::map _entities; From a0aae43386dc8cf6ca4ea2ab6da141b4d691a5c8 Mon Sep 17 00:00:00 2001 From: ksuprynowicz Date: Mon, 25 Mar 2024 00:39:04 +0100 Subject: [PATCH 4/4] Disabled texture loading for agent --- assignment-client/src/Agent.cpp | 2 -- assignment-client/src/avatars/ScriptableAvatar.cpp | 3 --- .../model-networking/src/model-networking/ModelCache.cpp | 8 ++++++-- .../procedural/src/procedural/ProceduralMaterialCache.cpp | 8 +++++++- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 3ac8eaaff79..246fd22cbb0 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -108,7 +108,6 @@ Agent::Agent(ReceivedMessage& message) : DependencyManager::set(); DependencyManager::set(); - DependencyManager::set(); // Needed to ensure the creation of the DebugDraw instance on the main thread DebugDraw::getInstance(); @@ -849,7 +848,6 @@ void Agent::aboutToFinish() { DependencyManager::destroy(); DependencyManager::destroy(); - DependencyManager::destroy(); DependencyManager::destroy(); diff --git a/assignment-client/src/avatars/ScriptableAvatar.cpp b/assignment-client/src/avatars/ScriptableAvatar.cpp index 3a3b2cf517a..929cd68f7c4 100644 --- a/assignment-client/src/avatars/ScriptableAvatar.cpp +++ b/assignment-client/src/avatars/ScriptableAvatar.cpp @@ -143,9 +143,6 @@ static AnimPose composeAnimPose(const HFMJoint& joint, const glm::quat rotation, } void ScriptableAvatar::update(float deltatime) { - // TODO: the current decision to use geometry resource results in loading textures, but it works way better - // than previous choice of loading avatar as animation, which was missing data such as joint names hash, - // and also didn't support glTF models. Optimizing this will be left fot future PR. if (!_geometryResource && !_skeletonModelFilenameURL.isEmpty()) { // AvatarData will parse the .fst, but not get the .fbx skeleton. _geometryResource = DependencyManager::get()->getGeometryResource(_skeletonModelFilenameURL); } diff --git a/libraries/model-networking/src/model-networking/ModelCache.cpp b/libraries/model-networking/src/model-networking/ModelCache.cpp index 738c61874f4..a488098b1bd 100644 --- a/libraries/model-networking/src/model-networking/ModelCache.cpp +++ b/libraries/model-networking/src/model-networking/ModelCache.cpp @@ -355,8 +355,12 @@ void GeometryResource::deleter() { void GeometryResource::setTextures() { if (_hfmModel) { - for (const HFMMaterial& material : _hfmModel->materials) { - _materials.push_back(std::make_shared(material, _textureBaseURL)); + if (DependencyManager::get()) { + for (const HFMMaterial& material : _hfmModel->materials) { + _materials.push_back(std::make_shared(material, _textureBaseURL)); + } + } else { + qDebug() << "GeometryResource::setTextures: TextureCache dependency not available, skipping textures"; } } } diff --git a/libraries/procedural/src/procedural/ProceduralMaterialCache.cpp b/libraries/procedural/src/procedural/ProceduralMaterialCache.cpp index 02ec234266d..fe5d85d0ef4 100644 --- a/libraries/procedural/src/procedural/ProceduralMaterialCache.cpp +++ b/libraries/procedural/src/procedural/ProceduralMaterialCache.cpp @@ -865,7 +865,13 @@ graphics::TextureMapPointer NetworkMaterial::fetchTextureMap(const QUrl& baseUrl } const auto url = getTextureUrl(baseUrl, hfmTexture); - const auto texture = DependencyManager::get()->getTexture(url, type, hfmTexture.content, hfmTexture.maxNumPixels, hfmTexture.sourceChannel); + auto textureCache = DependencyManager::get(); + NetworkTexturePointer texture; + if (textureCache) { + textureCache->getTexture(url, type, hfmTexture.content, hfmTexture.maxNumPixels, hfmTexture.sourceChannel); + } else { + qDebug() << "GeometryResource::setTextures: TextureCache dependency not available, skipping textures"; + } _textures[channel] = Texture { hfmTexture.name, texture }; auto map = std::make_shared();