the shadow passes get their own face culling mode

the new MaterialInstance::setCullingMode(color, shadow) API allows to
set a separate face culling mode for the color pass and the shadow 
passes.

FIXES[391679058]

android/filament-android/src/main/cpp/MaterialInstance.cpp

@@ -329,6 +329,18 @@ Java_com_google_android_filament_MaterialInstance_nSetCullingMode(JNIEnv*,
     instance->setCullingMode((MaterialInstance::CullingMode) cullingMode);
 }
 
+
+extern "C"
+JNIEXPORT void JNICALL
+Java_com_google_android_filament_MaterialInstance_nSetCullingModeSeparate(JNIEnv*, jclass,
+        jlong nativeMaterialInstance,
+        jlong colorPassCullingMode, jlong shadowPassCullingMode) {
+    MaterialInstance* instance = (MaterialInstance*) nativeMaterialInstance;
+    instance->setCullingMode(
+            (MaterialInstance::CullingMode) colorPassCullingMode,
+            (MaterialInstance::CullingMode) shadowPassCullingMode);
+}
+
 extern "C"
 JNIEXPORT void JNICALL
 Java_com_google_android_filament_MaterialInstance_nSetColorWrite(JNIEnv*,
@@ -505,6 +517,14 @@ Java_com_google_android_filament_MaterialInstance_nGetCullingMode(JNIEnv* env, j
     return (jint)instance->getCullingMode();
 }
 
+extern "C"
+JNIEXPORT jint JNICALL
+Java_com_google_android_filament_MaterialInstance_nGetShadowCullingMode(JNIEnv* env, jclass,
+        jlong nativeMaterialInstance) {
+    MaterialInstance* instance = (MaterialInstance*)nativeMaterialInstance;
+    return (jint)instance->getShadowCullingMode();
+}
+
 extern "C"
 JNIEXPORT jboolean JNICALL
 Java_com_google_android_filament_MaterialInstance_nIsColorWriteEnabled(JNIEnv* env, jclass clazz,

android/filament-android/src/main/java/com/google/android/filament/MaterialInstance.java

@@ -556,6 +556,20 @@ public void setCullingMode(@NonNull Material.CullingMode mode) {
         nSetCullingMode(getNativeObject(), mode.ordinal());
     }
 
+    /**
+     * Overrides the default triangle culling state that was set on the material separately for the
+     * color and shadow passes
+     *
+     * @see
+     * <a href="https://google.github.io/filament/Materials.html#materialdefinitions/materialblock/rasterization:culling">
+     * Rasterization: culling</a>
+     */
+    public void setCullingMode(@NonNull Material.CullingMode colorPassCullingMode,
+                               @NonNull Material.CullingMode shadowPassCullingMode) {
+        nSetCullingModeSeparate(getNativeObject(),
+            colorPassCullingMode.ordinal(), shadowPassCullingMode.ordinal());
+    }
+
     /**
      * Returns the face culling mode.
      */
@@ -564,6 +578,14 @@ public Material.CullingMode getCullingMode() {
         return sCullingModeValues[nGetCullingMode(getNativeObject())];
     }
 
+    /**
+     * Returns the face culling mode for the shadow passes.
+     */
+    @NonNull
+    public Material.CullingMode getShadowCullingMode() {
+        return sCullingModeValues[nGetShadowCullingMode(getNativeObject())];
+    }
+
     /**
      * Overrides the default color-buffer write state that was set on the material.
      *
@@ -920,6 +942,8 @@ private static native void nSetSpecularAntiAliasingThreshold(long nativeMaterial
 
     private static native void nSetDoubleSided(long nativeMaterialInstance, boolean doubleSided);
     private static native void nSetCullingMode(long nativeMaterialInstance, long mode);
+    private static native void nSetCullingModeSeparate(long nativeMaterialInstance,
+            long colorPassCullingMode, long shadowPassCullingMode);
     private static native void nSetColorWrite(long nativeMaterialInstance, boolean enable);
     private static native void nSetDepthWrite(long nativeMaterialInstance, boolean enable);
     private static native void nSetStencilWrite(long nativeMaterialInstance, boolean enable);
@@ -952,6 +976,7 @@ private static native void nSetStencilWriteMask(long nativeMaterialInstance, int
     private static native float nGetSpecularAntiAliasingThreshold(long nativeMaterialInstance);
     private static native boolean nIsDoubleSided(long nativeMaterialInstance);
     private static native int nGetCullingMode(long nativeMaterialInstance);
+    private static native int nGetShadowCullingMode(long nativeMaterialInstance);
     private static native boolean nIsColorWriteEnabled(long nativeMaterialInstance);
     private static native boolean nIsDepthWriteEnabled(long nativeMaterialInstance);
     private static native boolean nIsStencilWriteEnabled(long nativeMaterialInstance);

filament/include/filament/MaterialInstance.h

@@ -121,7 +121,7 @@ class UTILS_PUBLIC MaterialInstance : public FilamentAPI {
 
     /** inline helper to provide the name as a null-terminated string literal */
     template<typename T, typename = is_supported_parameter_t<T>>
-    void setParameter(StringLiteral name, T const& value) {
+    void setParameter(StringLiteral const name, T const& value) {
         setParameter<T>(name.data, name.size, value);
     }
 
@@ -148,14 +148,14 @@ class UTILS_PUBLIC MaterialInstance : public FilamentAPI {
 
     /** inline helper to provide the name as a null-terminated string literal */
     template<typename T, typename = is_supported_parameter_t<T>>
-    void setParameter(StringLiteral name, const T* UTILS_NONNULL values, size_t count) {
+    void setParameter(StringLiteral const name, const T* UTILS_NONNULL values, size_t const count) {
         setParameter<T>(name.data, name.size, values, count);
     }
 
     /** inline helper to provide the name as a null-terminated C string */
     template<typename T, typename = is_supported_parameter_t<T>>
     void setParameter(const char* UTILS_NONNULL name,
-                      const T* UTILS_NONNULL values, size_t count) {
+                      const T* UTILS_NONNULL values, size_t const count) {
         setParameter<T>(name, strlen(name), values, count);
     }
 
@@ -176,7 +176,7 @@ class UTILS_PUBLIC MaterialInstance : public FilamentAPI {
             Texture const* UTILS_NULLABLE texture, TextureSampler const& sampler);
 
     /** inline helper to provide the name as a null-terminated string literal */
-    void setParameter(StringLiteral name,
+    void setParameter(StringLiteral const name,
                       Texture const* UTILS_NULLABLE texture, TextureSampler const& sampler) {
         setParameter(name.data, name.size, texture, sampler);
     }
@@ -202,12 +202,12 @@ class UTILS_PUBLIC MaterialInstance : public FilamentAPI {
             RgbType type, math::float3 color);
 
     /** inline helper to provide the name as a null-terminated string literal */
-    void setParameter(StringLiteral name, RgbType type, math::float3 color) {
+    void setParameter(StringLiteral const name, RgbType const type, math::float3 const color) {
         setParameter(name.data, name.size, type, color);
     }
 
     /** inline helper to provide the name as a null-terminated C string */
-    void setParameter(const char* UTILS_NONNULL name, RgbType type, math::float3 color) {
+    void setParameter(const char* UTILS_NONNULL name, RgbType const type, math::float3 const color) {
         setParameter(name, strlen(name), type, color);
     }
 
@@ -226,12 +226,12 @@ class UTILS_PUBLIC MaterialInstance : public FilamentAPI {
             RgbaType type, math::float4 color);
 
     /** inline helper to provide the name as a null-terminated string literal */
-    void setParameter(StringLiteral name, RgbaType type, math::float4 color) {
+    void setParameter(StringLiteral const name, RgbaType const type, math::float4 const color) {
         setParameter(name.data, name.size, type, color);
     }
 
     /** inline helper to provide the name as a null-terminated C string */
-    void setParameter(const char* UTILS_NONNULL name, RgbaType type, math::float4 color) {
+    void setParameter(const char* UTILS_NONNULL name, RgbaType const type, math::float4 const color) {
         setParameter(name, strlen(name), type, color);
     }
 
@@ -251,7 +251,7 @@ class UTILS_PUBLIC MaterialInstance : public FilamentAPI {
 
     /** inline helper to provide the name as a null-terminated C string */
     template<typename T, typename = is_supported_parameter_t<T>>
-    T getParameter(StringLiteral name) const {
+    T getParameter(StringLiteral const name) const {
         return getParameter<T>(name.data, name.size);
     }
 
@@ -376,11 +376,22 @@ class UTILS_PUBLIC MaterialInstance : public FilamentAPI {
      */
     void setCullingMode(CullingMode culling) noexcept;
 
+    /**
+     * Overrides the default triangle culling state that was set on the material separately for the
+     * color and shadow passes
+     */
+    void setCullingMode(CullingMode colorPassCullingMode, CullingMode shadowPassCullingMode) noexcept;
+
     /**
      * Returns the face culling mode.
      */
     CullingMode getCullingMode() const noexcept;
 
+    /**
+     * Returns the face culling mode for the shadow passes.
+     */
+    CullingMode getShadowCullingMode() const noexcept;
+
     /**
      * Overrides the default color-buffer write state that was set on the material.
      */

filament/src/MaterialInstance.cpp

@@ -17,6 +17,26 @@
 #include <filament/MaterialInstance.h>
 
 #include "details/Material.h"
+#include "details/MaterialInstance.h"
+
+#include <filament/Color.h>
+#include <filament/MaterialEnums.h>
+
+#include <backend/DriverEnums.h>
+
+#include <utils/compiler.h>
+#include <utils/debug.h>
+
+#include <math/vec2.h>
+#include <math/vec3.h>
+#include <math/vec4.h>
+#include <math/mat3.h>
+
+#include <algorithm>
+#include <string_view>
+
+#include <stddef.h>
+#include <stdint.h>
 
 namespace filament {
 
@@ -85,7 +105,7 @@ void MaterialInstance::setParameter(const char* name, size_t nameLength, T const
 template<>
 UTILS_PUBLIC void MaterialInstance::setParameter(const char* name, size_t const nameLength, bool const& v) {
     // this kills tail-call optimization
-    setParameter(name, nameLength, (uint32_t)v);
+    setParameter(name, nameLength, uint32_t(v));
 }
 
 template<>
@@ -125,8 +145,8 @@ template UTILS_PUBLIC void MaterialInstance::setParameter<mat4f>   (const char*
 // ------------------------------------------------------------------------------------------------
 
 template <typename T, typename>
-void MaterialInstance::setParameter(const char* name, size_t nameLength, const T* value, size_t count) {
-    downcast(this)->setParameterImpl({ name, nameLength }, value, count);
+void MaterialInstance::setParameter(const char* name, size_t nameLength, const T* values, size_t count) {
+    downcast(this)->setParameterImpl({ name, nameLength }, values, count);
 }
 
 template<>
@@ -272,6 +292,11 @@ void MaterialInstance::setCullingMode(CullingMode const culling) noexcept {
     downcast(this)->setCullingMode(culling);
 }
 
+void MaterialInstance::setCullingMode(CullingMode const colorPassCullingMode,
+    CullingMode const shadowPassCullingMode) noexcept {
+    downcast(this)->setCullingMode(colorPassCullingMode, shadowPassCullingMode);
+}
+
 void MaterialInstance::setColorWrite(bool const enable) noexcept {
     downcast(this)->setColorWrite(enable);
 }
@@ -353,6 +378,10 @@ CullingMode MaterialInstance::getCullingMode() const noexcept {
     return downcast(this)->getCullingMode();
 }
 
+CullingMode MaterialInstance::getShadowCullingMode() const noexcept {
+    return downcast(this)->getShadowCullingMode();
+}
+
 bool MaterialInstance::isColorWriteEnabled() const noexcept {
     return downcast(this)->isColorWriteEnabled();
 }

filament/src/RenderPass.cpp

@@ -807,7 +807,13 @@ RenderPass::Command* RenderPass::generateCommandsImpl(CommandTypeFlags extraFlag
                     cmd.key &= ~Z_BUCKET_MASK;
                     cmd.key |= makeField(distanceBits >> 22u, Z_BUCKET_MASK, Z_BUCKET_SHIFT);
                 }
+
+                *curr = cmd;
+                // cancel command if both front and back faces are culled
+                curr->key |= select(mi->getCullingMode() == CullingMode::FRONT_AND_BACK);
+
             } else if constexpr (isDepthPass) {
+                const CullingMode cullingMode = hasShadowing ? mi->getShadowCullingMode() : mi->getCullingMode();
                 const RasterState rs = ma->getRasterState();
                 const TransparencyMode mode = mi->getTransparencyMode();
                 const BlendingMode blendingMode = ma->getBlendingMode();
@@ -816,7 +822,7 @@ RenderPass::Command* RenderPass::generateCommandsImpl(CommandTypeFlags extraFlag
                 const bool isPickingVariant = Variant::isPickingVariant(variant);
 
                 cmd.key |= mi->getSortingKey(); // already all set-up for direct or'ing
-                cmd.info.rasterState.culling = mi->getCullingMode();
+                cmd.info.rasterState.culling = cullingMode;
 
                 // FIXME: should writeDepthForShadowCasters take precedence over mi->getDepthWrite()?
                 cmd.info.rasterState.depthWrite = (1 // only keep bit 0
@@ -825,11 +831,12 @@ RenderPass::Command* RenderPass::generateCommandsImpl(CommandTypeFlags extraFlag
                                                    & !(filterTranslucentObjects & translucent)
                                                    & !(depthFilterAlphaMaskedObjects & rs.alphaToCoverage))
                                                   | writeDepthForShadowCasters;
+
+                *curr = cmd;
+                // cancel command if both front and back faces are culled
+                curr->key |= select(cullingMode == CullingMode::FRONT_AND_BACK);
             }
 
-            *curr = cmd;
-            // cancel command if both front and back faces are culled
-            curr->key |= select(mi->getCullingMode() == CullingMode::FRONT_AND_BACK);
             ++curr;
         }
     }

filament/src/details/MaterialInstance.cpp

@@ -196,10 +196,10 @@ void FMaterialInstance::commit(DriverApi& driver) const {
         for (auto const& [binding, p]: mTextureParameters) {
             assert_invariant(p.texture);
             // TODO: figure out a way to do this more efficiently (isValid() is a hashmap lookup)
-            FEngine& engine = mMaterial->getEngine();
+            FEngine const& engine = mMaterial->getEngine();
             FILAMENT_CHECK_PRECONDITION(engine.isValid(p.texture))
                     << "Invalid texture still bound to MaterialInstance: '" << getName() << "'\n";
-            Handle<HwTexture> handle = p.texture->getHwHandleForSampling();
+            Handle<HwTexture> const handle = p.texture->getHwHandleForSampling();
             assert_invariant(handle);
             mDescriptorSet.setSampler(binding, handle, p.params);
         }
@@ -216,7 +216,7 @@ void FMaterialInstance::commit(DriverApi& driver) const {
 
 void FMaterialInstance::setParameter(std::string_view const name,
         Handle<HwTexture> texture, SamplerParams const params) {
-    auto binding = mMaterial->getSamplerBinding(name);
+    auto const binding = mMaterial->getSamplerBinding(name);
     mDescriptorSet.setSampler(binding, texture, params);
 }
 
@@ -243,7 +243,7 @@ void FMaterialInstance::setParameterImpl(std::string_view const name,
     }
 #endif
 
-    auto binding = mMaterial->getSamplerBinding(name);
+    auto const binding = mMaterial->getSamplerBinding(name);
     if (texture && texture->textureHandleCanMutate()) {
         mTextureParameters[binding] = { texture, sampler.getSamplerParams() };
     } else {
@@ -328,13 +328,13 @@ const char* FMaterialInstance::getName() const noexcept {
 void FMaterialInstance::use(FEngine::DriverApi& driver) const {
 
     if (UTILS_UNLIKELY(mMissingSamplerDescriptors.any())) {
-        std::call_once(mMissingSamplersFlag, [this]() {
+        std::call_once(mMissingSamplersFlag, [this] {
             auto const& list = mMaterial->getSamplerInterfaceBlock().getSamplerInfoList();
             slog.w << "sampler parameters not set in MaterialInstance \""
                    << mName.c_str_safe() << "\" or Material \""
                    << mMaterial->getName().c_str_safe() << "\":\n";
             mMissingSamplerDescriptors.forEachSetBit([&list](descriptor_binding_t binding) {
-                auto pos = std::find_if(list.begin(), list.end(), [binding](const auto& item) {
+                auto const pos = std::find_if(list.begin(), list.end(), [binding](const auto& item) {
                     return item.binding == binding;
                 });
                 // just safety-check, should never fail
@@ -368,7 +368,7 @@ void FMaterialInstance::fixMissingSamplers() const {
         // here we need to set the samplers that are missing
         auto const& list = mMaterial->getSamplerInterfaceBlock().getSamplerInfoList();
         missingSamplerDescriptors.forEachSetBit([this, &list](descriptor_binding_t binding) {
-            auto pos = std::find_if(list.begin(), list.end(), [binding](const auto& item) {
+            auto const pos = std::find_if(list.begin(), list.end(), [binding](const auto& item) {
                 return item.binding == binding;
             });
 

filament/src/details/MaterialInstance.h

@@ -94,6 +94,8 @@ class FMaterialInstance : public MaterialInstance {
 
     backend::CullingMode getCullingMode() const noexcept { return mCulling; }
 
+    backend::CullingMode getShadowCullingMode() const noexcept { return mShadowCulling; }
+
     bool isColorWriteEnabled() const noexcept { return mColorWrite; }
 
     bool isDepthWriteEnabled() const noexcept { return mDepthWrite; }
@@ -137,6 +139,11 @@ class FMaterialInstance : public MaterialInstance {
 
     void setCullingMode(CullingMode const culling) noexcept { mCulling = culling; }
 
+    void setCullingMode(CullingMode const color, CullingMode const shadow) noexcept {
+        mCulling = color;
+        mShadowCulling = shadow;
+    }
+
     void setColorWrite(bool const enable) noexcept { mColorWrite = enable; }
 
     void setDepthWrite(bool const enable) noexcept { mDepthWrite = enable; }
@@ -271,7 +278,9 @@ class FMaterialInstance : public MaterialInstance {
     float mSpecularAntiAliasingThreshold = 0.0f;
 
     backend::CullingMode mCulling : 2;
+    backend::CullingMode mShadowCulling : 2;
     backend::RasterState::DepthFunc mDepthFunc : 3;
+
     bool mColorWrite : 1;
     bool mDepthWrite : 1;
     bool mHasScissor : 1;
@@ -283,8 +292,8 @@ class FMaterialInstance : public MaterialInstance {
 
     // Scissor rectangle is specified as: Left Bottom Width Height.
     backend::Viewport mScissorRect = { 0, 0,
-            (uint32_t)std::numeric_limits<int32_t>::max(),
-            (uint32_t)std::numeric_limits<int32_t>::max()
+            uint32_t(std::numeric_limits<int32_t>::max()),
+            uint32_t(std::numeric_limits<int32_t>::max())
     };
 
     utils::CString mName;