sg: Fix culling in layers

Pick-to: 6.8
Fixes: QTBUG-136611
Change-Id: If2a0a0365ca24360d850ffce98c0bec4a3961976
Reviewed-by: Jonas Karlsson <jonas.karlsson@qt.io>
(cherry picked from commit e32d335c8ff6a78d0601453b8cd805b097faa26f)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>
(cherry picked from commit c42721eb3baf3dc64362a1698b3bb8956dcde540)

11 files changed, 568 insertions, 7 deletions

diff --git a/examples/quick/scenegraph/customrendernode/customrender.cpp b/examples/quick/scenegraph/customrendernode/customrender.cpp
index ff5274c163..761c8d0778 100644
--- a/examples/quick/scenegraph/customrendernode/customrender.cpp
+++ b/examples/quick/scenegraph/customrendernode/customrender.cpp
@@ -130,10 +130,13 @@ void CustomRenderNode::prepare()
     if (!m_pipeline) {
         m_pipeline.reset(rhi->newGraphicsPipeline());
 
-        // If layer.enabled == true on our QQuickItem, the rendering face is flipped for
-        // backends with isYUpInFrameBuffer == true (OpenGL). This does not happen with
-        // RHI backends with isYUpInFrameBuffer == false. We swap the triangle winding
-        // order to work around this.
+        // If layer.enabled == true on our QQuickItem and layer.textureMirroring is the
+        // default MirrorVertically, the winding order needs to be inverted when
+        // isYUpInFrameBuffer == true (OpenGL). This does not happen with other backends,
+        // unless textureMirroring is changed so that the situation is inverted, meaning
+        // GL needs no adjustments while others need the opposite winding order. Here we
+        // choose the replicate, to a degree, what the scenegraph renderer does, but note that
+        // this is only correct as long as textureMirroring is not changed from the default.
         m_pipeline->setFrontFace(renderTarget()->resourceType() == QRhiResource::TextureRenderTarget
                                                  && rhi->isYUpInFramebuffer()
                                          ? QRhiGraphicsPipeline::CW
diff --git a/src/quick/scenegraph/coreapi/qsgabstractrenderer.cpp b/src/quick/scenegraph/coreapi/qsgabstractrenderer.cpp
index ffa5599231..54627b3db7 100644
--- a/src/quick/scenegraph/coreapi/qsgabstractrenderer.cpp
+++ b/src/quick/scenegraph/coreapi/qsgabstractrenderer.cpp
@@ -48,6 +48,7 @@ QT_BEGIN_NAMESPACE
 QSGAbstractRendererPrivate::QSGAbstractRendererPrivate()
     : m_root_node(nullptr)
     , m_clear_color(Qt::transparent)
+    , m_invertFrontFace(false)
 {
     m_projection_matrix.resize(1);
     m_projection_matrix_native_ndc.resize(1);
@@ -303,6 +304,24 @@ QMatrix4x4 QSGAbstractRenderer::projectionMatrixWithNativeNDC(int index) const
 }
 
 /*!
+    \internal
+ */
+void QSGAbstractRenderer::setInvertFrontFace(bool invert)
+{
+    Q_D(QSGAbstractRenderer);
+    d->m_invertFrontFace = invert;
+}
+
+/*!
+    \internal
+ */
+bool QSGAbstractRenderer::invertFrontFace() const
+{
+    Q_D(const QSGAbstractRenderer);
+    return d->m_invertFrontFace;
+}
+
+/*!
     Sets the \a color to clear the framebuffer.
 
     \sa clearColor()
diff --git a/src/quick/scenegraph/coreapi/qsgabstractrenderer_p.h b/src/quick/scenegraph/coreapi/qsgabstractrenderer_p.h
index 5d7a3b13b6..a4ed8547f3 100644
--- a/src/quick/scenegraph/coreapi/qsgabstractrenderer_p.h
+++ b/src/quick/scenegraph/coreapi/qsgabstractrenderer_p.h
@@ -60,6 +60,8 @@ public:
     QMatrix4x4 projectionMatrixWithNativeNDC(int index) const;
     int projectionMatrixCount() const;
     int projectionMatrixWithNativeNDCCount() const;
+    void setInvertFrontFace(bool invert);
+    bool invertFrontFace() const;
 
     void setClearColor(const QColor &color);
     QColor clearColor() const;
diff --git a/src/quick/scenegraph/coreapi/qsgabstractrenderer_p_p.h b/src/quick/scenegraph/coreapi/qsgabstractrenderer_p_p.h
index 3bc04247db..54206d2bc2 100644
--- a/src/quick/scenegraph/coreapi/qsgabstractrenderer_p_p.h
+++ b/src/quick/scenegraph/coreapi/qsgabstractrenderer_p_p.h
@@ -42,7 +42,7 @@ public:
 
     QVarLengthArray<QMatrix4x4, 1> m_projection_matrix;
     QVarLengthArray<QMatrix4x4, 1> m_projection_matrix_native_ndc;
-    uint m_mirrored : 1;
+    uint m_invertFrontFace : 1;
 };
 
 QT_END_NAMESPACE
diff --git a/src/quick/scenegraph/coreapi/qsgbatchrenderer.cpp b/src/quick/scenegraph/coreapi/qsgbatchrenderer.cpp
index 591d5d4df0..0cce21d833 100644
--- a/src/quick/scenegraph/coreapi/qsgbatchrenderer.cpp
+++ b/src/quick/scenegraph/coreapi/qsgbatchrenderer.cpp
@@ -2756,6 +2756,7 @@ bool Renderer::ensurePipelineState(Element *e, const ShaderManager::Shader *sms,
     ps->setFlags(flags);
     ps->setTopology(qsg_topology(m_gstate.drawMode, m_rhi));
     ps->setCullMode(m_gstate.cullMode);
+    ps->setFrontFace(invertFrontFace() ? QRhiGraphicsPipeline::CW : QRhiGraphicsPipeline::CCW);
     ps->setPolygonMode(m_gstate.polygonMode);
     ps->setMultiViewCount(m_gstate.multiViewCount);
 
diff --git a/src/quick/scenegraph/qsgrhilayer.cpp b/src/quick/scenegraph/qsgrhilayer.cpp
index 0baf3779b3..5c5d7abd27 100644
--- a/src/quick/scenegraph/qsgrhilayer.cpp
+++ b/src/quick/scenegraph/qsgrhilayer.cpp
@@ -410,22 +410,41 @@ void QSGRhiLayer::grab()
     m_renderer->setDevicePixelRatio(m_dpr);
     m_renderer->setDeviceRect(m_pixelSize);
     m_renderer->setViewportRect(m_pixelSize);
+
     QRectF mirrored; // in logical coordinates (no dpr) since this gets passed to setProjectionMatrixToRect()
-    if (m_rhi->isYUpInFramebuffer()) {
+
+    // In the unlikely event of back/front face culling used by a custom
+    // material or effect in the layer, the default front face setting may be
+    // wrong. Rather, it needs to invert based on what the vertex shader does,
+    // and so on the rect (and so matrix) generated here.
+    bool frontFaceSwap = false;
+
+    if (m_rhi->isYUpInFramebuffer()) { // basically OpenGL
         mirrored = QRectF(m_mirrorHorizontal ? m_logicalRect.right() : m_logicalRect.left(),
                           m_mirrorVertical ? m_logicalRect.bottom() : m_logicalRect.top(),
                           m_mirrorHorizontal ? -m_logicalRect.width() : m_logicalRect.width(),
                           m_mirrorVertical ? -m_logicalRect.height() : m_logicalRect.height());
-    } else {
+        if (m_mirrorHorizontal)
+            frontFaceSwap = !frontFaceSwap;
+        if (m_mirrorVertical)
+            frontFaceSwap = !frontFaceSwap;
+    } else { // APIs other than OpenGL
         mirrored = QRectF(m_mirrorHorizontal ? m_logicalRect.right() : m_logicalRect.left(),
                           m_mirrorVertical ? m_logicalRect.top() : m_logicalRect.bottom(),
                           m_mirrorHorizontal ? -m_logicalRect.width() : m_logicalRect.width(),
                           m_mirrorVertical ? m_logicalRect.height() : -m_logicalRect.height());
+        if (m_mirrorHorizontal)
+            frontFaceSwap = !frontFaceSwap;
+        if (!m_mirrorVertical)
+            frontFaceSwap = !frontFaceSwap;
     }
+
     QSGAbstractRenderer::MatrixTransformFlags matrixFlags;
     if (!m_rhi->isYUpInNDC())
         matrixFlags |= QSGAbstractRenderer::MatrixTransformFlipY;
+
     m_renderer->setProjectionMatrixToRect(mirrored, matrixFlags);
+    m_renderer->setInvertFrontFace(frontFaceSwap);
     m_renderer->setClearColor(Qt::transparent);
     m_renderer->setRenderTarget({ m_rt, m_rtRp, m_context->currentFrameCommandBuffer() });
 
diff --git a/tests/baseline/scenegraph/data/shaders/culling/culling_1_layer.qml b/tests/baseline/scenegraph/data/shaders/culling/culling_1_layer.qml
new file mode 100644
index 0000000000..784846b2fc
--- /dev/null
+++ b/tests/baseline/scenegraph/data/shaders/culling/culling_1_layer.qml
@@ -0,0 +1,100 @@
+import QtQuick 2.0
+
+Rectangle {
+    id: topLevel
+    width: 320
+    height: 480
+
+    // Make it a layer. Important because the winding order inverts based on the transforms the vertex shader does.
+    // The output should match culling_1 regardless.
+    layer.enabled: true
+
+    ShaderEffectSource {
+        id: front
+        visible: false
+        smooth: true
+        sourceItem: Rectangle {
+            width: 256
+            height: 64
+            color: "cornflowerblue"
+            radius: 8
+            Text {
+                anchors.centerIn: parent
+                text: "Front"
+                font.pixelSize: 48
+                color: "white"
+            }
+        }
+    }
+    ShaderEffectSource {
+        id: back
+        visible: false
+        smooth: true
+        sourceItem: Rectangle {
+            width: 256
+            height: 64
+            color: "firebrick"
+            radius: 8
+            Text {
+                anchors.centerIn: parent
+                text: "Back"
+                font.pixelSize: 48
+                color: "white"
+            }
+        }
+    }
+    Column {
+        anchors.fill: parent
+        Repeater {
+            model: ListModel {
+                ListElement {
+                    foo: "No culling"
+                    bar: ShaderEffect.NoCulling
+                    turned: false
+                }
+                ListElement {
+                    foo: "Back-face culling"
+                    bar: ShaderEffect.BackFaceCulling
+                    turned: false
+                }
+                ListElement {
+                    foo: "Front-face culling"
+                    bar: ShaderEffect.FrontFaceCulling
+                    turned: false
+                }
+            }
+
+            Item{
+                id: item_0000
+                width: 320
+                height: 120
+                ShaderEffect {
+                    anchors.right: parent.right
+                    anchors.verticalCenter: parent.verticalCenter
+                    anchors.margins: 10
+                    width: 200
+                    height: 100
+                    cullMode: model.bar
+                    property variant frontSource: front
+                    property variant backSource: back
+                    fragmentShader: "qrc:shaders/culling.frag.qsb"
+                    transform: Rotation {
+                        origin.x: 200
+                        origin.y: 180 - 120 * index
+                        axis { x: 0; y: 1; z: 0 }
+                        angle: (turned == true) ? 180 : 0
+
+                    }
+                }
+                Text {
+                    font.pointSize: 10
+                    anchors.verticalCenter: parent.verticalCenter
+                    anchors.left: parent.left
+                    anchors.margins: 5
+                    text: foo
+                }
+            }
+        }
+    }
+}
+
diff --git a/tests/baseline/scenegraph/data/shaders/culling/culling_1_layer_both.qml b/tests/baseline/scenegraph/data/shaders/culling/culling_1_layer_both.qml
new file mode 100644
index 0000000000..fe90a47992
--- /dev/null
+++ b/tests/baseline/scenegraph/data/shaders/culling/culling_1_layer_both.qml
@@ -0,0 +1,106 @@
+import QtQuick 2.0
+
+Rectangle {
+    id: topLevel
+    width: 320
+    height: 480
+
+    // Make it a layer. Important because the winding order inverts based on the transforms the vertex shader does.
+    // The output should match culling_1 regardless.
+    layer.enabled: true
+
+    // layer.textureMirroring by default is MirrorVertically. It has no effect
+    // on the visual result in this example, but it has a huge effect on the
+    // transformations internally. Change it to something to see the output is
+    // still the same.
+    layer.textureMirroring: ShaderEffectSource.MirrorHorizontally | ShaderEffectSource.MirrorVertically
+
+    ShaderEffectSource {
+        id: front
+        visible: false
+        smooth: true
+        sourceItem: Rectangle {
+            width: 256
+            height: 64
+            color: "cornflowerblue"
+            radius: 8
+            Text {
+                anchors.centerIn: parent
+                text: "Front"
+                font.pixelSize: 48
+                color: "white"
+            }
+        }
+    }
+    ShaderEffectSource {
+        id: back
+        visible: false
+        smooth: true
+        sourceItem: Rectangle {
+            width: 256
+            height: 64
+            color: "firebrick"
+            radius: 8
+            Text {
+                anchors.centerIn: parent
+                text: "Back"
+                font.pixelSize: 48
+                color: "white"
+            }
+        }
+    }
+    Column {
+        anchors.fill: parent
+        Repeater {
+            model: ListModel {
+                ListElement {
+                    foo: "No culling"
+                    bar: ShaderEffect.NoCulling
+                    turned: false
+                }
+                ListElement {
+                    foo: "Back-face culling"
+                    bar: ShaderEffect.BackFaceCulling
+                    turned: false
+                }
+                ListElement {
+                    foo: "Front-face culling"
+                    bar: ShaderEffect.FrontFaceCulling
+                    turned: false
+                }
+            }
+
+            Item{
+                id: item_0000
+                width: 320
+                height: 120
+                ShaderEffect {
+                    anchors.right: parent.right
+                    anchors.verticalCenter: parent.verticalCenter
+                    anchors.margins: 10
+                    width: 200
+                    height: 100
+                    cullMode: model.bar
+                    property variant frontSource: front
+                    property variant backSource: back
+                    fragmentShader: "qrc:shaders/culling.frag.qsb"
+                    transform: Rotation {
+                        origin.x: 200
+                        origin.y: 180 - 120 * index
+                        axis { x: 0; y: 1; z: 0 }
+                        angle: (turned == true) ? 180 : 0
+
+                    }
+                }
+                Text {
+                    font.pointSize: 10
+                    anchors.verticalCenter: parent.verticalCenter
+                    anchors.left: parent.left
+                    anchors.margins: 5
+                    text: foo
+                }
+            }
+        }
+    }
+}
+
diff --git a/tests/baseline/scenegraph/data/shaders/culling/culling_1_layer_horiz.qml b/tests/baseline/scenegraph/data/shaders/culling/culling_1_layer_horiz.qml
new file mode 100644
index 0000000000..c973c1e668
--- /dev/null
+++ b/tests/baseline/scenegraph/data/shaders/culling/culling_1_layer_horiz.qml
@@ -0,0 +1,106 @@
+import QtQuick 2.0
+
+Rectangle {
+    id: topLevel
+    width: 320
+    height: 480
+
+    // Make it a layer. Important because the winding order inverts based on the transforms the vertex shader does.
+    // The output should match culling_1 regardless.
+    layer.enabled: true
+
+    // layer.textureMirroring by default is MirrorVertically. It has no effect
+    // on the visual result in this example, but it has a huge effect on the
+    // transformations internally. Change it to something to see the output is
+    // still the same.
+    layer.textureMirroring: ShaderEffectSource.MirrorHorizontally
+
+    ShaderEffectSource {
+        id: front
+        visible: false
+        smooth: true
+        sourceItem: Rectangle {
+            width: 256
+            height: 64
+            color: "cornflowerblue"
+            radius: 8
+            Text {
+                anchors.centerIn: parent
+                text: "Front"
+                font.pixelSize: 48
+                color: "white"
+            }
+        }
+    }
+    ShaderEffectSource {
+        id: back
+        visible: false
+        smooth: true
+        sourceItem: Rectangle {
+            width: 256
+            height: 64
+            color: "firebrick"
+            radius: 8
+            Text {
+                anchors.centerIn: parent
+                text: "Back"
+                font.pixelSize: 48
+                color: "white"
+            }
+        }
+    }
+    Column {
+        anchors.fill: parent
+        Repeater {
+            model: ListModel {
+                ListElement {
+                    foo: "No culling"
+                    bar: ShaderEffect.NoCulling
+                    turned: false
+                }
+                ListElement {
+                    foo: "Back-face culling"
+                    bar: ShaderEffect.BackFaceCulling
+                    turned: false
+                }
+                ListElement {
+                    foo: "Front-face culling"
+                    bar: ShaderEffect.FrontFaceCulling
+                    turned: false
+                }
+            }
+
+            Item{
+                id: item_0000
+                width: 320
+                height: 120
+                ShaderEffect {
+                    anchors.right: parent.right
+                    anchors.verticalCenter: parent.verticalCenter
+                    anchors.margins: 10
+                    width: 200
+                    height: 100
+                    cullMode: model.bar
+                    property variant frontSource: front
+                    property variant backSource: back
+                    fragmentShader: "qrc:shaders/culling.frag.qsb"
+                    transform: Rotation {
+                        origin.x: 200
+                        origin.y: 180 - 120 * index
+                        axis { x: 0; y: 1; z: 0 }
+                        angle: (turned == true) ? 180 : 0
+
+                    }
+                }
+                Text {
+                    font.pointSize: 10
+                    anchors.verticalCenter: parent.verticalCenter
+                    anchors.left: parent.left
+                    anchors.margins: 5
+                    text: foo
+                }
+            }
+        }
+    }
+}
+
diff --git a/tests/baseline/scenegraph/data/shaders/culling/culling_1_layer_no.qml b/tests/baseline/scenegraph/data/shaders/culling/culling_1_layer_no.qml
new file mode 100644
index 0000000000..99a43041ee
--- /dev/null
+++ b/tests/baseline/scenegraph/data/shaders/culling/culling_1_layer_no.qml
@@ -0,0 +1,106 @@
+import QtQuick 2.0
+
+Rectangle {
+    id: topLevel
+    width: 320
+    height: 480
+
+    // Make it a layer. Important because the winding order inverts based on the transforms the vertex shader does.
+    // The output should match culling_1 regardless.
+    layer.enabled: true
+
+    // layer.textureMirroring by default is MirrorVertically. It has no effect
+    // on the visual result in this example, but it has a huge effect on the
+    // transformations internally. Change it to something to see the output is
+    // still the same.
+    layer.textureMirroring: ShaderEffectSource.NoMirroring
+
+    ShaderEffectSource {
+        id: front
+        visible: false
+        smooth: true
+        sourceItem: Rectangle {
+            width: 256
+            height: 64
+            color: "cornflowerblue"
+            radius: 8
+            Text {
+                anchors.centerIn: parent
+                text: "Front"
+                font.pixelSize: 48
+                color: "white"
+            }
+        }
+    }
+    ShaderEffectSource {
+        id: back
+        visible: false
+        smooth: true
+        sourceItem: Rectangle {
+            width: 256
+            height: 64
+            color: "firebrick"
+            radius: 8
+            Text {
+                anchors.centerIn: parent
+                text: "Back"
+                font.pixelSize: 48
+                color: "white"
+            }
+        }
+    }
+    Column {
+        anchors.fill: parent
+        Repeater {
+            model: ListModel {
+                ListElement {
+                    foo: "No culling"
+                    bar: ShaderEffect.NoCulling
+                    turned: false
+                }
+                ListElement {
+                    foo: "Back-face culling"
+                    bar: ShaderEffect.BackFaceCulling
+                    turned: false
+                }
+                ListElement {
+                    foo: "Front-face culling"
+                    bar: ShaderEffect.FrontFaceCulling
+                    turned: false
+                }
+            }
+
+            Item{
+                id: item_0000
+                width: 320
+                height: 120
+                ShaderEffect {
+                    anchors.right: parent.right
+                    anchors.verticalCenter: parent.verticalCenter
+                    anchors.margins: 10
+                    width: 200
+                    height: 100
+                    cullMode: model.bar
+                    property variant frontSource: front
+                    property variant backSource: back
+                    fragmentShader: "qrc:shaders/culling.frag.qsb"
+                    transform: Rotation {
+                        origin.x: 200
+                        origin.y: 180 - 120 * index
+                        axis { x: 0; y: 1; z: 0 }
+                        angle: (turned == true) ? 180 : 0
+
+                    }
+                }
+                Text {
+                    font.pointSize: 10
+                    anchors.verticalCenter: parent.verticalCenter
+                    anchors.left: parent.left
+                    anchors.margins: 5
+                    text: foo
+                }
+            }
+        }
+    }
+}
+
diff --git a/tests/baseline/scenegraph/data/shaders/culling/culling_2_layer.qml b/tests/baseline/scenegraph/data/shaders/culling/culling_2_layer.qml
new file mode 100644
index 0000000000..41147b714c
--- /dev/null
+++ b/tests/baseline/scenegraph/data/shaders/culling/culling_2_layer.qml
@@ -0,0 +1,99 @@
+import QtQuick 2.0
+
+Rectangle {
+    id: topLevel
+    width: 320
+    height: 480
+
+    // Make it a layer. Important because the winding order inverts based on the transforms the vertex shader does.
+    // The output should match culling_2 regardless.
+    layer.enabled: true
+
+    ShaderEffectSource {
+        id: front
+        visible: false
+        smooth: true
+        sourceItem: Rectangle {
+            width: 256
+            height: 64
+            color: "cornflowerblue"
+            radius: 8
+            Text {
+                anchors.centerIn: parent
+                text: "Front"
+                font.pixelSize: 48
+                color: "white"
+            }
+        }
+    }
+    ShaderEffectSource {
+        id: back
+        visible: false
+        smooth: true
+        sourceItem: Rectangle {
+            width: 256
+            height: 64
+            color: "firebrick"
+            radius: 8
+            Text {
+                anchors.centerIn: parent
+                text: "Back"
+                font.pixelSize: 48
+                color: "white"
+            }
+        }
+    }
+    Column {
+        anchors.fill: parent
+        Repeater {
+            model: ListModel {
+                ListElement {
+                    foo: "No culling"
+                    bar: ShaderEffect.NoCulling
+                    turned: true
+                }
+                ListElement {
+                    foo: "Back-face culling"
+                    bar: ShaderEffect.BackFaceCulling
+                    turned: true
+                }
+                ListElement {
+                    foo: "Front-face culling"
+                    bar: ShaderEffect.FrontFaceCulling
+                    turned: true
+                }
+            }
+
+            Item{
+                id: item_0000
+                width: 320
+                height: 120
+                ShaderEffect{
+                    anchors.right: parent.right
+                    anchors.verticalCenter: parent.verticalCenter
+                    anchors.margins: 10
+                    width: 200
+                    height: 100
+                    cullMode: model.bar
+                    property variant frontSource: front
+                    property variant backSource: back
+                    fragmentShader: "qrc:shaders/culling.frag.qsb"
+                    transform: Rotation {
+                        origin.x: 100
+                        origin.y: 180 - 120 * index
+                        axis { x: 0; y: 1; z: 0 }
+                        angle: (turned == true) ? 180 : 0
+
+                    }
+                }
+                Text {
+                    font.pointSize: 10
+                    anchors.verticalCenter: parent.verticalCenter
+                    anchors.left: parent.left
+                    anchors.margins: 5
+                    text: foo
+                }
+            }
+        }
+    }
+}