Skip to content

Commit

Permalink
Merge pull request #854 from overte-org/fix/anim_reference
Browse files Browse the repository at this point in the history 
Fix invalid animation reference in assignment client
  • Loading branch information
@ksuprynowicz
ksuprynowicz authored Mar 25, 2024
2 parents 922ae91 + a0aae43 commit 7fdd4ae
Show file tree
Hide file tree
Showing 5 changed files with 106 additions and 60 deletions.
137 changes: 83 additions & 54 deletions assignment-client/src/avatars/ScriptableAvatar.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@

ScriptableAvatar::ScriptableAvatar(): _scriptEngine(newScriptEngine()) {
_clientTraitsHandler.reset(new ClientTraitsHandler(this));
static std::once_flag once;
std::call_once(once, [] {
qRegisterMetaType<HFMModel::Pointer>("HFMModel::Pointer");
});
}

QByteArray ScriptableAvatar::toByteArrayStateful(AvatarDataDetail dataDetail, bool dropFaceTracking) {
Expand All @@ -52,6 +56,7 @@ void ScriptableAvatar::startAnimation(const QString& url, float fps, float prior
_animation = DependencyManager::get<AnimationCache>()->getAnimation(url);
_animationDetails = AnimationDetails("", QUrl(url), fps, 0, loop, hold, false, firstFrame, lastFrame, true, firstFrame, false);
_maskedJoints = maskedJoints;
_isAnimationRigValid = false;
}

void ScriptableAvatar::stopAnimation() {
Expand Down Expand Up @@ -89,11 +94,12 @@ QStringList ScriptableAvatar::getJointNames() const {
}

void ScriptableAvatar::setSkeletonModelURL(const QUrl& skeletonModelURL) {
_bind.reset();
_animSkeleton.reset();
_avatarAnimSkeleton.reset();
_geometryResource.reset();

AvatarData::setSkeletonModelURL(skeletonModelURL);
updateJointMappings();
_isRigValid = false;
}

int ScriptableAvatar::sendAvatarDataPacket(bool sendAll) {
Expand Down Expand Up @@ -137,65 +143,87 @@ static AnimPose composeAnimPose(const HFMJoint& joint, const glm::quat rotation,
}

void ScriptableAvatar::update(float deltatime) {
if (!_geometryResource && !_skeletonModelFilenameURL.isEmpty()) { // AvatarData will parse the .fst, but not get the .fbx skeleton.
_geometryResource = DependencyManager::get<ModelCache>()->getGeometryResource(_skeletonModelFilenameURL);
}

// Run animation
if (_animation && _animation->isLoaded() && _animation->getFrames().size() > 0 && !_bind.isNull() && _bind->isLoaded()) {
if (!_animSkeleton) {
_animSkeleton = std::make_shared<AnimSkeleton>(_bind->getHFMModel());
}
float currentFrame = _animationDetails.currentFrame + deltatime * _animationDetails.fps;
if (_animationDetails.loop || currentFrame < _animationDetails.lastFrame) {
while (currentFrame >= _animationDetails.lastFrame) {
currentFrame -= (_animationDetails.lastFrame - _animationDetails.firstFrame);
Q_ASSERT(QThread::currentThread() == thread());
if (_animation && _animation->isLoaded()) {
Q_ASSERT(thread() == _animation->thread());
auto frames = _animation->getFramesReference();
if (frames.size() > 0 && _geometryResource && _geometryResource->isHFMModelLoaded()) {
if (!_isRigValid) {
_rig.reset(_geometryResource->getHFMModel());
_isRigValid = true;
}
_animationDetails.currentFrame = currentFrame;

const QVector<HFMJoint>& modelJoints = _bind->getHFMModel().joints;
QStringList animationJointNames = _animation->getJointNames();

const int nJoints = modelJoints.size();
if (_jointData.size() != nJoints) {
_jointData.resize(nJoints);
if (!_isAnimationRigValid) {
_animationRig.reset(_animation->getHFMModel());
_isAnimationRigValid = true;
}

const int frameCount = _animation->getFrames().size();
const HFMAnimationFrame& floorFrame = _animation->getFrames().at((int)glm::floor(currentFrame) % frameCount);
const HFMAnimationFrame& ceilFrame = _animation->getFrames().at((int)glm::ceil(currentFrame) % frameCount);
const float frameFraction = glm::fract(currentFrame);
std::vector<AnimPose> poses = _animSkeleton->getRelativeDefaultPoses();

const float UNIT_SCALE = 0.01f;

for (int i = 0; i < animationJointNames.size(); i++) {
const QString& name = animationJointNames[i];
// As long as we need the model preRotations anyway, let's get the jointIndex from the bind skeleton rather than
// trusting the .fst (which is sometimes not updated to match changes to .fbx).
int mapping = _bind->getHFMModel().getJointIndex(name);
if (mapping != -1 && !_maskedJoints.contains(name)) {

AnimPose floorPose = composeAnimPose(modelJoints[mapping], floorFrame.rotations[i], floorFrame.translations[i] * UNIT_SCALE);
AnimPose ceilPose = composeAnimPose(modelJoints[mapping], ceilFrame.rotations[i], floorFrame.translations[i] * UNIT_SCALE);
blend(1, &floorPose, &ceilPose, frameFraction, &poses[mapping]);
}
if (!_avatarAnimSkeleton) {
_avatarAnimSkeleton = std::make_shared<AnimSkeleton>(_geometryResource->getHFMModel());
}
float currentFrame = _animationDetails.currentFrame + deltatime * _animationDetails.fps;
if (_animationDetails.loop || currentFrame < _animationDetails.lastFrame) {
while (currentFrame >= _animationDetails.lastFrame) {
currentFrame -= (_animationDetails.lastFrame - _animationDetails.firstFrame);
}
_animationDetails.currentFrame = currentFrame;

const QVector<HFMJoint>& modelJoints = _geometryResource->getHFMModel().joints;
QStringList animationJointNames = _animation->getJointNames();

std::vector<AnimPose> absPoses = poses;
_animSkeleton->convertRelativePosesToAbsolute(absPoses);
for (int i = 0; i < nJoints; i++) {
JointData& data = _jointData[i];
AnimPose& absPose = absPoses[i];
if (data.rotation != absPose.rot()) {
data.rotation = absPose.rot();
data.rotationIsDefaultPose = false;
const int nJoints = modelJoints.size();
if (_jointData.size() != nJoints) {
_jointData.resize(nJoints);
}
AnimPose& relPose = poses[i];
if (data.translation != relPose.trans()) {
data.translation = relPose.trans();
data.translationIsDefaultPose = false;

const int frameCount = frames.size();
const HFMAnimationFrame& floorFrame = frames.at((int)glm::floor(currentFrame) % frameCount);
const HFMAnimationFrame& ceilFrame = frames.at((int)glm::ceil(currentFrame) % frameCount);
const float frameFraction = glm::fract(currentFrame);
std::vector<AnimPose> poses = _avatarAnimSkeleton->getRelativeDefaultPoses();

// TODO: this needs more testing, it's possible that we need not only scale but also rotation and translation
// According to tests with unmatching avatar and animation armatures, sometimes bones are not rotated correctly.
// Matching armatures already work very well now.
const float UNIT_SCALE = _animationRig.GetScaleFactorGeometryToUnscaledRig() / _rig.GetScaleFactorGeometryToUnscaledRig();

for (int i = 0; i < animationJointNames.size(); i++) {
const QString& name = animationJointNames[i];
// As long as we need the model preRotations anyway, let's get the jointIndex from the bind skeleton rather than
// trusting the .fst (which is sometimes not updated to match changes to .fbx).
int mapping = _geometryResource->getHFMModel().getJointIndex(name);
if (mapping != -1 && !_maskedJoints.contains(name)) {
AnimPose floorPose = composeAnimPose(modelJoints[mapping], floorFrame.rotations[i],
floorFrame.translations[i] * UNIT_SCALE);
AnimPose ceilPose = composeAnimPose(modelJoints[mapping], ceilFrame.rotations[i],
ceilFrame.translations[i] * UNIT_SCALE);
blend(1, &floorPose, &ceilPose, frameFraction, &poses[mapping]);
}
}
}

} else {
_animation.clear();
std::vector<AnimPose> absPoses = poses;
Q_ASSERT(_avatarAnimSkeleton != nullptr);
_avatarAnimSkeleton->convertRelativePosesToAbsolute(absPoses);
for (int i = 0; i < nJoints; i++) {
JointData& data = _jointData[i];
AnimPose& absPose = absPoses[i];
if (data.rotation != absPose.rot()) {
data.rotation = absPose.rot();
data.rotationIsDefaultPose = false;
}
AnimPose& relPose = poses[i];
if (data.translation != relPose.trans()) {
data.translation = relPose.trans();
data.translationIsDefaultPose = false;
}
}

} else {
_animation.clear();
}
}
}

Expand Down Expand Up @@ -245,6 +273,7 @@ void ScriptableAvatar::setJointMappingsFromNetworkReply() {
networkReply->deleteLater();
return;
}
// TODO: this works only with .fst files currently, not directly with FBX and GLB models
{
QWriteLocker writeLock(&_jointDataLock);
QByteArray line;
Expand All @@ -253,7 +282,7 @@ void ScriptableAvatar::setJointMappingsFromNetworkReply() {
if (line.startsWith("filename")) {
int filenameIndex = line.indexOf('=') + 1;
if (filenameIndex > 0) {
_skeletonFBXURL = _skeletonModelURL.resolved(QString(line.mid(filenameIndex).trimmed()));
_skeletonModelFilenameURL = _skeletonModelURL.resolved(QString(line.mid(filenameIndex).trimmed()));
}
}
if (!line.startsWith("jointIndex")) {
Expand Down
12 changes: 9 additions & 3 deletions assignment-client/src/avatars/ScriptableAvatar.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
#include <AvatarData.h>
#include <ScriptEngine.h>
#include <EntityItem.h>
#include "model-networking/ModelCache.h"
#include "Rig.h"

/*@jsdoc
* The <code>Avatar</code> API is used to manipulate scriptable avatars on the domain. This API is a subset of the
Expand Down Expand Up @@ -217,11 +219,15 @@ public slots:
AnimationPointer _animation;
AnimationDetails _animationDetails;
QStringList _maskedJoints;
AnimationPointer _bind; // a sleazy way to get the skeleton, given the various library/cmake dependencies
std::shared_ptr<AnimSkeleton> _animSkeleton;
GeometryResource::Pointer _geometryResource;
Rig _rig;
bool _isRigValid{false};
Rig _animationRig;
bool _isAnimationRigValid{false};
std::shared_ptr<AnimSkeleton> _avatarAnimSkeleton;
QHash<QString, int> _fstJointIndices; ///< 1-based, since zero is returned for missing keys
QStringList _fstJointNames; ///< in order of depth-first traversal
QUrl _skeletonFBXURL;
QUrl _skeletonModelFilenameURL; // This contains URL from filename field in fst file
mutable ScriptEnginePointer _scriptEngine;
std::map<QUuid, EntityItemPointer> _entities;

Expand Down
8 changes: 6 additions & 2 deletions libraries/model-networking/src/model-networking/ModelCache.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -355,8 +355,12 @@ void GeometryResource::deleter() {

void GeometryResource::setTextures() {
if (_hfmModel) {
for (const HFMMaterial& material : _hfmModel->materials) {
_materials.push_back(std::make_shared<NetworkMaterial>(material, _textureBaseURL));
if (DependencyManager::get<TextureCache>()) {
for (const HFMMaterial& material : _hfmModel->materials) {
_materials.push_back(std::make_shared<NetworkMaterial>(material, _textureBaseURL));
}
} else {
qDebug() << "GeometryResource::setTextures: TextureCache dependency not available, skipping textures";
}
}
}
Expand Down
8 changes: 7 additions & 1 deletion libraries/procedural/src/procedural/ProceduralMaterialCache.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -865,7 +865,13 @@ graphics::TextureMapPointer NetworkMaterial::fetchTextureMap(const QUrl& baseUrl
}

const auto url = getTextureUrl(baseUrl, hfmTexture);
const auto texture = DependencyManager::get<TextureCache>()->getTexture(url, type, hfmTexture.content, hfmTexture.maxNumPixels, hfmTexture.sourceChannel);
auto textureCache = DependencyManager::get<TextureCache>();
NetworkTexturePointer texture;
if (textureCache) {
textureCache->getTexture(url, type, hfmTexture.content, hfmTexture.maxNumPixels, hfmTexture.sourceChannel);
} else {
qDebug() << "GeometryResource::setTextures: TextureCache dependency not available, skipping textures";
}
_textures[channel] = Texture { hfmTexture.name, texture };

auto map = std::make_shared<graphics::TextureMap>();
Expand Down
1 change: 1 addition & 0 deletions libraries/ui/src/QmlWindowClass.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,7 @@ bool QmlWindowClass::isVisible() {
return quickItem->isVisible();
} else {
qDebug() << "QmlWindowClass::isVisible: asQuickItem() returned NULL";
return false;
}
}

Expand Down

0 comments on commit 7fdd4ae

Please sign in to comment.