wayland: Reset surface on QWindow type changeHEADdev

Changing the role between toplevel and popups is illegal on wayland even
if the window is not visible. We need to reset to the wl_surface.

This meant we cannot call setWindowFlags in the constructor.

QWindow::flags / QWindow::windowType changes after
QPlatformWindow::setFlags has finished, so we need to use our own flags
rather in XdgShell rather than pulling them from the QWindow.

Task-number: QTBUG-136110
Change-Id: I8b54b7ea8a768a539178395e53cc63a64fd80232
Reviewed-by: David Edmundson <[email protected]>

4 files changed, 66 insertions, 6 deletions

diff --git a/src/plugins/platforms/wayland/plugins/shellintegration/xdg-shell/qwaylandxdgshell.cpp b/src/plugins/platforms/wayland/plugins/shellintegration/xdg-shell/qwaylandxdgshell.cpp
index 8ac5f2d7bf4..1356d93abab 100644
--- a/src/plugins/platforms/wayland/plugins/shellintegration/xdg-shell/qwaylandxdgshell.cpp
+++ b/src/plugins/platforms/wayland/plugins/shellintegration/xdg-shell/qwaylandxdgshell.cpp
@@ -311,7 +311,7 @@ QWaylandXdgSurface::QWaylandXdgSurface(QWaylandXdgShell *shell, ::xdg_surface *s
     , m_window(window)
 {
     QWaylandDisplay *display = window->display();
-    Qt::WindowType type = window->window()->type();
+    Qt::WindowType type =  static_cast<Qt::WindowType>(int(window->windowFlags() & Qt::WindowType_Mask));
     auto *transientParent = window->transientParent();
 
     if (type == Qt::ToolTip) {
diff --git a/src/plugins/platforms/wayland/qwaylandwindow.cpp b/src/plugins/platforms/wayland/qwaylandwindow.cpp
index ad43fa8f124..e79bf230f50 100644
--- a/src/plugins/platforms/wayland/qwaylandwindow.cpp
+++ b/src/plugins/platforms/wayland/qwaylandwindow.cpp
@@ -62,6 +62,7 @@ QWaylandWindow::QWaylandWindow(QWindow *window, QWaylandDisplay *display)
     }
 
     initializeWlSurface();
+    mFlags = window->flags();
 
     setWindowIcon(window->icon());
 
@@ -180,6 +181,9 @@ void QWaylandWindow::initWindow()
         }
     }
 
+    createDecoration();
+    updateInputRegion();
+
     // Enable high-dpi rendering. Scale() returns the screen scale factor and will
     // typically be integer 1 (normal-dpi) or 2 (high-dpi). Call set_buffer_scale()
     // to inform the compositor that high-resolution buffers will be provided.
@@ -188,7 +192,6 @@ void QWaylandWindow::initWindow()
     else if (mSurface->version() >= 3)
         mSurface->set_buffer_scale(std::ceil(scale()));
 
-    setWindowFlags(window()->flags());
     QRect geometry = windowGeometry();
     QRect defaultGeometry = this->defaultGeometry();
     if (geometry.width() <= 0)
@@ -198,10 +201,11 @@ void QWaylandWindow::initWindow()
 
     setGeometry_helper(geometry);
     setMask(window()->mask());
-    if (mShellSurface)
+    if (mShellSurface) {
         mShellSurface->requestWindowStates(window()->windowStates());
+        mShellSurface->setWindowFlags(mFlags);
+    }
     handleContentOrientationChange(window()->contentOrientation());
-    mFlags = window()->flags();
 
     if (mShellSurface && mShellSurface->commitSurfaceRole())
         mSurface->commit();
@@ -1046,16 +1050,33 @@ void QWaylandWindow::setWindowState(Qt::WindowStates states)
 
 void QWaylandWindow::setWindowFlags(Qt::WindowFlags flags)
 {
-    if (mShellSurface)
-        mShellSurface->setWindowFlags(flags);
+    const bool wasPopup = mFlags.testFlag(Qt::Popup);
+    const bool isPopup = flags.testFlag(Qt::Popup);
 
     mFlags = flags;
+    // changing role is not allowed on XdgShell on the same wl_surface
+    if (wasPopup != isPopup) {
+        reset();
+        initializeWlSurface();
+        if (window()->isVisible()) {
+            initWindow();
+        }
+    } else {
+        if (mShellSurface)
+            mShellSurface->setWindowFlags(flags);
+    }
+
     createDecoration();
 
     QReadLocker locker(&mSurfaceLock);
     updateInputRegion();
 }
 
+Qt::WindowFlags QWaylandWindow::windowFlags() const
+{
+    return mFlags;
+}
+
 bool QWaylandWindow::createDecoration()
 {
     Q_ASSERT_X(QThread::currentThreadId() == QThreadData::get2(thread())->threadId.loadRelaxed(),
diff --git a/src/plugins/platforms/wayland/qwaylandwindow_p.h b/src/plugins/platforms/wayland/qwaylandwindow_p.h
index 800cc817a09..5d0f93b406e 100644
--- a/src/plugins/platforms/wayland/qwaylandwindow_p.h
+++ b/src/plugins/platforms/wayland/qwaylandwindow_p.h
@@ -156,6 +156,7 @@ public:
     Qt::WindowStates windowStates() const;
     void setWindowState(Qt::WindowStates states) override;
     void setWindowFlags(Qt::WindowFlags flags) override;
+    Qt::WindowFlags windowFlags() const;
     void handleWindowStatesChanged(Qt::WindowStates states);
 
     void raise() override;
diff --git a/tests/auto/wayland/xdgshell/tst_xdgshell.cpp b/tests/auto/wayland/xdgshell/tst_xdgshell.cpp
index e23cc83f5b6..3a7e001e1d9 100644
--- a/tests/auto/wayland/xdgshell/tst_xdgshell.cpp
+++ b/tests/auto/wayland/xdgshell/tst_xdgshell.cpp
@@ -24,6 +24,7 @@ private slots:
     void popup();
     void tooltipOnPopup();
     void tooltipAndSiblingPopup();
+    void windowTypeChanges();
     void switchPopups();
     void hidePopupParent();
     void popupsWithoutParent();
@@ -475,6 +476,43 @@ void tst_xdgshell::tooltipAndSiblingPopup()
     QCOMPOSITOR_TRY_COMPARE(xdgPopup(0), nullptr);
 }
 
+void tst_xdgshell::windowTypeChanges()
+{
+    QRasterWindow parentWindow;
+    parentWindow.resize(200, 200);
+    parentWindow.show();
+
+    QCOMPOSITOR_TRY_VERIFY(xdgToplevel());
+    exec([&] { xdgToplevel()->sendCompleteConfigure(); });
+    QCOMPOSITOR_TRY_VERIFY(xdgToplevel()->m_xdgSurface->m_committedConfigureSerial);
+
+    // show a toplevel
+    QRasterWindow window;
+    window.resize(100, 100);
+    window.setTransientParent(&parentWindow);
+    window.show();
+    QCOMPOSITOR_TRY_VERIFY(xdgToplevel(1));
+    exec([&] { xdgToplevel(1)->sendCompleteConfigure(); });
+    QCOMPOSITOR_TRY_VERIFY(xdgToplevel(1)->m_xdgSurface->m_committedConfigureSerial);
+
+    // now change it to a popup
+    window.setFlag(Qt::ToolTip, true);
+    QCOMPOSITOR_TRY_VERIFY(!xdgToplevel(1));
+    QCOMPOSITOR_TRY_VERIFY(xdgPopup());
+    exec([&] { xdgPopup()->sendCompleteConfigure(QRect(100, 100, 100, 100)); });
+    QCOMPOSITOR_TRY_VERIFY(xdgPopup()->m_xdgSurface->m_committedConfigureSerial);
+
+    window.hide();
+    QCOMPOSITOR_TRY_VERIFY(!xdgPopup());
+
+    // change to a toplevel again this time whilst hidden
+    window.setFlag(Qt::ToolTip, false);
+    window.show();
+    QCOMPOSITOR_TRY_VERIFY(xdgToplevel(1));
+    exec([&] { xdgToplevel(1)->sendCompleteConfigure(); });
+    QCOMPOSITOR_TRY_VERIFY(xdgToplevel(1)->m_xdgSurface->m_committedConfigureSerial);
+}
+
 // QTBUG-65680
 void tst_xdgshell::switchPopups()
 {