// Copyright (C) 2024 The Qt Company Ltd. // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only #include "qquick3dxrmanager_visionos_p.h" #include "qquick3dxrorigin_p.h" #include "qquick3dxrmanager_p.h" #include "qquick3dxrinputmanager_visionos_p.h" #include "qquick3dxranchormanager_visionos_p.h" #include "../qquick3dxrinputmanager_p.h" #include "../qquick3dxranimationdriver_p.h" #include #include #include #include #include #include #include #include #include #include QT_BEGIN_NAMESPACE Q_DECLARE_LOGGING_CATEGORY(lcQuick3DXr); static const char s_renderThreadName[] = "QQuick3DXrRenderThread"; // NOTE: This only affects visionOS static bool qssgDisableFoveation() { static bool foveationDisabled = qEnvironmentVariableIntValue("QT_QUICK3D_XR_DISABLE_FOVEATION") != 0; return foveationDisabled; } class CompositorLayer : public QObject, public QNativeInterface::QVisionOSApplication::ImmersiveSpaceCompositorLayer { Q_OBJECT public: CompositorLayer() = default; ~CompositorLayer() final { m_xrManager = nullptr; [m_layerRenderer release]; m_layerRenderer = nullptr; m_worldTrackingProvider = nullptr; m_arSession = nullptr; m_initialized = false; } using EventT = std::underlying_type_t; enum Event : EventT { Render = QEvent::User + 1, Pause, Stop, Pulse }; static constexpr QEvent::Type asQEvent(CompositorLayer::Event event) { return static_cast(event); } static bool supportsLayoutType(cp_layer_renderer_capabilities_t capabilities, cp_layer_renderer_layout layout) { const size_t layoutCount = cp_layer_renderer_capabilities_supported_layouts_count(capabilities, cp_supported_layouts_options_none); bool found = false; for (size_t i = 0; i < layoutCount && !found; ++i) found = (layout == cp_layer_renderer_capabilities_supported_layout(capabilities, cp_supported_layouts_options_none, i)); return found; } void configure(cp_layer_renderer_capabilities_t capabilities, cp_layer_renderer_configuration_t configuration) const override { QMutexLocker locker(&m_compositorLayerMtx); m_multiviewEnabled = !QQuick3DXrManager::isMultiviewRenderingDisabled(); // If multiview isn't disabled we need to check if the target supports it. // NOTE: We're only doing layered or dedicated rendering, so shared is not supported (untested). m_multiviewSupported = supportsLayoutType(capabilities, cp_layer_renderer_layout_layered); m_multiviewEnabled = m_multiviewEnabled && m_multiviewSupported; // NOTE: If foviation is supported we enable it. m_foveationSupported = cp_layer_renderer_capabilities_supports_foveation(capabilities); // ... but we can disable it with an environment variable. m_foveationEnabled = m_foveationSupported && !qssgDisableFoveation(); // NOTE: We're only doing layered or dedicated rendering, so shared is not supported, even though we technically // could support it. Since the only target we currently have (visionOS on the Vision Pro) support all these // modes anyways we only need to care about these two (dedicated is required for the emulator). cp_layer_renderer_layout textureLayout = m_multiviewEnabled ? cp_layer_renderer_layout_layered : cp_layer_renderer_layout_dedicated; cp_layer_renderer_configuration_set_layout(configuration, textureLayout); cp_layer_renderer_configuration_set_foveation_enabled(configuration, m_foveationEnabled); cp_layer_renderer_configuration_set_color_format(configuration, MTLPixelFormatRGBA16Float); simd_float2 depthRange = cp_layer_renderer_configuration_get_default_depth_range(configuration); // NOTE: The depth range is inverted for VisionOS (x = far, y = near) m_depthRange[0] = depthRange.y; m_depthRange[1] = depthRange.x; qCDebug(lcQuick3DXr) << "Configuring with the following settings:" << "\n\tMultiview supported: " << m_multiviewSupported << "\n\tMultiview enabled: " << m_multiviewEnabled << "\n\tFoveation supported: " << m_foveationSupported << "\n\tFoveation enabled: " << m_foveationEnabled << "\n\tDepth Range: " << m_depthRange[0] << " to: " << m_depthRange[1]; } void render(cp_layer_renderer_t renderer) override { if (m_layerRenderer != renderer) { m_layerRenderer = renderer; emit layerRendererChanged(); } if (m_layerRenderer) { QMutexLocker locker(&m_mutex); // NOTE: The layer renderer is our handle to the compositor layer, so we need to ensure that // we need to ensure that it stays alive long enough for us to know if we should tear down // the compositor layer or not. [m_layerRenderer retain]; checkRenderState(); emit layerRendererReady(); } } // Listen for spatial events (these are native gestures like pinch click/drag coming from SwiftUI) void handleSpatialEvents(const QJsonObject &events) override { if (m_xrManager) m_xrManager->processSpatialEvents(events); } bool isInitialized() const { return m_initialized; } bool isMultiviewSupported() const { return m_multiviewSupported; } bool isMultiviewEnabled() const { return m_multiviewEnabled; } bool isFoveationSupported() const { return m_foveationSupported; } bool isFoveationEnabled() const { return m_foveationEnabled; } cp_layer_renderer_t layerRenderer() const { return m_layerRenderer; } void getDefaultDepthRange(float &near, float &far) const { near = m_depthRange[0]; far = m_depthRange[1]; } // Must be called from the GUI thread void init(QQuick3DXrManagerPrivate *xrManager) { Q_ASSERT(qApp->thread() == QThread::currentThread()); QSSG_ASSERT(!m_initialized, return); m_xrManager = xrManager; runWorldTrackingARSession(); m_initialized = true; } // Called from the gui thread while holding the render lock. // For now we only let the thread know that it should stop rendering, // as we're going to tear down the compositor layer on the render thread. void teardown() { m_teardown = true; } void stopArSession() { Q_ASSERT(qApp->thread() == QThread::currentThread()); QMutexLocker locker(&m_arSessionMtx); if (m_arSession) { qCDebug(lcQuick3DXr, "Stopping AR session"); // NOTE: We're stopping the AR session, so we need to ensure that the AR session // doesn't get posted at some later point, so we change the dispatch queue to // use the main queue (which is the GUI thread) before calling stop. ar_session_set_data_provider_state_change_handler_f(m_arSession, dispatch_get_main_queue(), this, &onArStateChanged); ar_session_stop(m_arSession); m_arSession = nullptr; m_arTrackingState = QQuick3DXrManagerPrivate::ArTrackingState::Uninitialized; } } QMutex &renderLock() { return m_mutex; } bool waitForSyncToComplete() { return m_waitCondition.wait(&m_mutex); } Q_INVOKABLE static void destroy(QQuick3DXrManagerPrivate *mngr, QQuickWindow *window, CompositorLayer *compositorLayer) { QSSG_ASSERT(mngr != nullptr, return); qCDebug(lcQuick3DXr) << "Destroying compositor layer"; if (auto *visionOSApplicaton = qGuiApp->nativeInterface()) visionOSApplicaton->setImmersiveSpaceCompositorLayer(nullptr); // NOTE: This is a bit of a hack, but we need to cleanup the nodes // on the render thread while the GUI thread is blocked (as documented.) // Since we cannot destruct the window on the render thread, and we cannot // cleanup all nodes on the GUI thread due to dependencies on the render // thread and resources created on it. Instead we just invalidate the render // control and cleanup the nodes on the render thread. This is not ideal, // but it's what we have for now... if (window) { // If we never got past the setup we don't have a window. auto *d = QQuickWindowPrivate::get(window); d->cleanupNodesOnShutdown(); mngr->releaseResources(); if (auto *rc = d->renderControl) rc->invalidate(); } // We're holding onto the lock, so defer the delete until we're done here. // All reasources should be released at this point anyways. compositorLayer->deleteLater(); } Q_SIGNALS: void layerRendererReady(); void layerRendererChanged(); void renderStateChanged(QQuick3DXrManagerPrivate::RenderState); void arStateChanged(QQuick3DXrManagerPrivate::ArTrackingState); protected: bool event(QEvent *event) override { { // NOTE: Intentionally scoped to avoid locking the mutex for events we're not handling // and that has side-effects (e.g. cleanup due to a deferred delete). QMutexLocker locker(&m_mutex); if (m_teardown) { qCDebug(lcQuick3DXr) << "Teardown in progress, skipping event handling"; // If we're tearing down we're about to be destroyed, so we don't want to // ensure that the mutex is unlocked, as we're going to be destroyed anyways. if (event->type() == QEvent::DeferredDelete) locker.unlock(); return QObject::event(event); } switch (static_cast(event->type())) { case Event::Render: { const bool success = renderFrame(locker); // Check if we successfully rendered the frame, if not the GUI thread // is likely waiting for the render thread to complete rendering, so we // need to wake it up. if (!success) { qCDebug(lcQuick3DXr) << "Waking up the GUI thread, rendering returned early..."; m_xrManager->m_syncDone = false; m_waitCondition.wakeOne(); } } return true; case Event::Stop: cleanup(); return true; case Event::Pause: pause(); return true; case Event::Pulse: checkRenderState(); return true; } } return QObject::event(event); } private: friend bool QQuick3DXrManagerPrivate::renderFrameImpl(QMutexLocker &locker, QWaitCondition &waitCondition); friend void QQuick3DXrManagerPrivate::update(); static void onArStateChanged(void *context, ar_data_providers_t data_providers, ar_data_provider_state_t new_state, ar_error_t error, ar_data_provider_t failed_data_provider) { Q_UNUSED(context); Q_UNUSED(data_providers); Q_UNUSED(error); Q_UNUSED(failed_data_provider); auto *that = reinterpret_cast(context); QMutexLocker lock(&that->m_arSessionMtx); // If the ar session is null we're called after we tore down the session. QSSG_ASSERT_X(that->m_arSession != nullptr, "AR session is not running, skipping state change!", return); const auto oldState = that->m_arTrackingState; switch (new_state) { case ar_data_provider_state_initialized: that->m_arTrackingState = QQuick3DXrManagerPrivate::ArTrackingState::Initialized; break; case ar_data_provider_state_running: that->m_arTrackingState = QQuick3DXrManagerPrivate::ArTrackingState::Running; break; case ar_data_provider_state_paused: that->m_arTrackingState = QQuick3DXrManagerPrivate::ArTrackingState::Paused; break; case ar_data_provider_state_stopped: that->m_arTrackingState = QQuick3DXrManagerPrivate::ArTrackingState::Stopped; break; } if (oldState != that->m_arTrackingState) { QMutexLocker renderLocker(&that->m_mutex); emit that->arStateChanged(that->m_arTrackingState); } } enum class RenderStateAction { CheckOnly, WaitUntilRunning, }; template void checkRenderState() { QSSG_ASSERT(m_layerRenderer != nullptr, return); const auto oldState = m_renderState; switch (cp_layer_renderer_get_state(m_layerRenderer)) { case cp_layer_renderer_state_paused: m_renderState = QQuick3DXrManagerPrivate::RenderState::Paused; break; case cp_layer_renderer_state_running: m_renderState = QQuick3DXrManagerPrivate::RenderState::Running; break; case cp_layer_renderer_state_invalidated: m_renderState = QQuick3DXrManagerPrivate::RenderState::Invalidated; break; } if (oldState != m_renderState) emit renderStateChanged(m_renderState); if (m_renderState == QQuick3DXrManagerPrivate::RenderState::Paused) { if constexpr (Action == RenderStateAction::WaitUntilRunning) { qCDebug(lcQuick3DXr) << "Waiting for rendering to resume..."; cp_layer_renderer_wait_until_running(m_layerRenderer); QCoreApplication::postEvent(this, new QEvent(asQEvent(CompositorLayer::Event::Pulse))); } } } ar_device_anchor_t createPoseForTiming(cp_frame_timing_t timing) { QSSG_ASSERT(m_worldTrackingProvider != nullptr, return nullptr); ar_device_anchor_t outAnchor = ar_device_anchor_create(); cp_time_t presentationTime = cp_frame_timing_get_presentation_time(timing); CFTimeInterval queryTime = cp_time_to_cf_time_interval(presentationTime); ar_device_anchor_query_status_t status = ar_world_tracking_provider_query_device_anchor_at_timestamp(m_worldTrackingProvider, queryTime, outAnchor); if (status != ar_device_anchor_query_status_success) { NSLog(@"Failed to get estimated pose from world tracking provider for presentation timestamp %0.3f", queryTime); } return outAnchor; } void runWorldTrackingARSession() { QMutexLocker locker(&m_arSessionMtx); ar_world_tracking_configuration_t worldTrackingConfiguration = ar_world_tracking_configuration_create(); m_worldTrackingProvider = ar_world_tracking_provider_create(worldTrackingConfiguration); ar_data_providers_t dataProviders = ar_data_providers_create(); ar_data_providers_add_data_provider(dataProviders, m_worldTrackingProvider); QQuick3DXrInputManager *inputManager = QQuick3DXrInputManager::instance(); QQuick3DXrAnchorManager *anchorManager = QQuick3DXrAnchorManager::instance(); // 1. prepare QQuick3DXrInputManagerPrivate *pim = nullptr; if (QSSG_GUARD_X(inputManager != nullptr, "No InputManager available!")) { pim = QQuick3DXrInputManagerPrivate::get(inputManager); if (QSSG_GUARD(pim != nullptr)) pim->prepareHandtracking(dataProviders); } if (QSSG_GUARD_X(anchorManager != nullptr, "No AnchorManager available!")) QQuick3DXrManagerPrivate::prepareAnchorManager(anchorManager, dataProviders); m_arSession = ar_session_create(); ar_session_set_data_provider_state_change_handler_f(m_arSession, dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), this, &onArStateChanged); ar_session_run(m_arSession, dataProviders); // 2. initialize QQuick3DXrManagerPrivate::initInputManager(inputManager); QQuick3DXrManagerPrivate::initAnchorManager(anchorManager); } void pause() { qCDebug(lcQuick3DXr, "Pausing rendering"); } void cleanup() { qCDebug(lcQuick3DXr, "Cleaning up"); if (m_xrManager) m_xrManager->releaseResources(); } [[nodiscard]] bool renderFrame(QMutexLocker &locker) { Q_ASSERT(locker.isLocked()); checkRenderState(); if (m_renderState != QQuick3DXrManagerPrivate::RenderState::Running) { qCDebug(lcQuick3DXr, "Rendering is not running, skipping frame rendering!"); return false; } // Lock the AR session mutex and render the frame. We hold onto the lock until we're done rendering // to ensure that the AR session isn't stopped while we're rendering (this can happen!) QMutexLocker arLocker(&m_arSessionMtx); if (m_arTrackingState != QQuick3DXrManagerPrivate::ArTrackingState::Running) { qCDebug(lcQuick3DXr, "AR tracking is not running, skipping frame rendering!"); return false; } // Lock the compositor layer mutex and render the frame. We hold onto the lock until we're done rendering // as the configure call comes from the GUI thread and we need to ensure that the configuration is not // changed while we're rendering. QMutexLocker compositorLayer(&m_compositorLayerMtx); return m_xrManager->renderFrameImpl(locker, m_waitCondition); } ar_world_tracking_provider_t worldTrackingProvider() const { return m_worldTrackingProvider; } // Rendering mutable QMutex m_mutex; QWaitCondition m_waitCondition; mutable QMutex m_compositorLayerMtx; // AR Session mutable QMutex m_arSessionMtx; QQuick3DXrManagerPrivate *m_xrManager = nullptr; cp_layer_renderer_t m_layerRenderer = nullptr; ar_world_tracking_provider_t m_worldTrackingProvider = nullptr; ar_session_t m_arSession = nullptr; mutable float m_depthRange[2] {1.0f, 10000.0f}; // NOTE: Near, Far QQuick3DXrManagerPrivate::RenderState m_renderState = QQuick3DXrManagerPrivate::RenderState::Uninitialized; QQuick3DXrManagerPrivate::ArTrackingState m_arTrackingState = QQuick3DXrManagerPrivate::ArTrackingState::Uninitialized; bool m_initialized = false; bool m_teardown = false; mutable bool m_multiviewSupported = true; mutable bool m_multiviewEnabled = true; mutable bool m_foveationSupported = true; mutable bool m_foveationEnabled = true; }; struct QSSGCompositionLayerInstance { QPointer instance; }; Q_GLOBAL_STATIC(QSSGCompositionLayerInstance, s_compositorLayer) // FIXME: Maybe unify with the openxr implementation?! void QQuick3DXrManagerPrivate::updateCameraImp(simd_float4x4 headTransform, cp_drawable_t drawable, QQuick3DXrOrigin *xrOrigin, int i) { cp_view_t view = cp_drawable_get_view(drawable, i); simd_float2 depth_range = cp_drawable_get_depth_range(drawable); const float clipNear = depth_range[1]; const float clipFar = depth_range[0]; xrOrigin->eyeCamera(i)->setClipNear(clipNear); xrOrigin->eyeCamera(i)->setClipFar(clipFar); simd_float4x4 projection = cp_drawable_compute_projection(drawable, cp_axis_direction_convention_right_up_forward, i); QMatrix4x4 proj{projection.columns[0].x, projection.columns[1].x, projection.columns[2].x, projection.columns[3].x, projection.columns[0].y, projection.columns[1].y, projection.columns[2].y, projection.columns[3].y, projection.columns[0].z, projection.columns[1].z, projection.columns[2].z, projection.columns[3].z, projection.columns[0].w, projection.columns[1].w, projection.columns[2].w, projection.columns[3].w}; xrOrigin->eyeCamera(i)->setProjection(proj); simd_float4x4 localEyeTransform = cp_view_get_transform(view); simd_float4x4 eyeCameraTransform = simd_mul(headTransform, localEyeTransform); // NOTE: We need to convert from meters to centimeters here QMatrix4x4 transform{eyeCameraTransform.columns[0].x, eyeCameraTransform.columns[1].x, eyeCameraTransform.columns[2].x, eyeCameraTransform.columns[3].x * 100, eyeCameraTransform.columns[0].y, eyeCameraTransform.columns[1].y, eyeCameraTransform.columns[2].y, eyeCameraTransform.columns[3].y * 100, eyeCameraTransform.columns[0].z, eyeCameraTransform.columns[1].z, eyeCameraTransform.columns[2].z, eyeCameraTransform.columns[3].z * 100, 0.0f, 0.0f, 0.0f, 1.0f}; QQuick3DNodePrivate::get(xrOrigin->eyeCamera(i))->setLocalTransform(transform); } void QQuick3DXrManagerPrivate::updateCamera(QQuick3DViewport *xrViewport, simd_float4x4 headTransform, cp_drawable_t drawable, QQuick3DXrOrigin *xrOrigin, int i) { updateCameraImp(headTransform, drawable, xrOrigin, i); xrViewport->setCamera(xrOrigin->eyeCamera(i)); } void QQuick3DXrManagerPrivate::updateCameraMultiview(QQuick3DViewport *xrViewport, simd_float4x4 headTransform, cp_drawable_t drawable, QQuick3DXrOrigin *xrOrigin) { QQuick3DCamera *cameras[2] {xrOrigin->eyeCamera(0), xrOrigin->eyeCamera(1)}; for (int i = 0; i < 2; ++i) updateCameraImp(headTransform, drawable, xrOrigin, i); xrViewport->setMultiViewCameras(cameras); } QQuick3DXrManagerPrivate::QQuick3DXrManagerPrivate(QQuick3DXrManager &manager) : q_ptr(&manager) { } QQuick3DXrManagerPrivate::~QQuick3DXrManagerPrivate() { } QQuick3DXrManagerPrivate *QQuick3DXrManagerPrivate::get(QQuick3DXrManager *manager) { QSSG_ASSERT(manager != nullptr, return nullptr); return manager->d_func(); } bool QQuick3DXrManagerPrivate::initialize() { Q_Q(QQuick3DXrManager); // NOTE: Check if there's a global instance of the compositor layer // already created, if not create one. Should probably move this over // to the native interface. if (!m_compositorLayer) { m_compositorLayer = s_compositorLayer->instance; if (!m_compositorLayer) { m_compositorLayer = new CompositorLayer; s_compositorLayer->instance = m_compositorLayer; } } if (!m_inputManager) m_inputManager = QQuick3DXrInputManager::instance(); if (!m_anchorManager) m_anchorManager = QQuick3DXrAnchorManager::instance(); // NOTE: Check if the compository layer proxy is already active. if (!m_compositorLayer->isInitialized()) { if (auto *visionOSApplicaton = qGuiApp->nativeInterface()) { visionOSApplicaton->setImmersiveSpaceCompositorLayer(m_compositorLayer); m_compositorLayer->init(this); // FIXME: We don't actually handle the case where the rendere changes or we get multiple calls should do something. QObject::connect(m_compositorLayer, &CompositorLayer::layerRendererReady, q, &QQuick3DXrManager::initialized, Qt::ConnectionType(Qt::SingleShotConnection | Qt::QueuedConnection)); QObject::connect(m_compositorLayer, &CompositorLayer::renderStateChanged, q, [q](QQuick3DXrManagerPrivate::RenderState state) { switch (state) { case QQuick3DXrManagerPrivate::RenderState::Uninitialized: qCDebug(lcQuick3DXr, "Render state: Uninitialized"); QQuick3DXrManagerPrivate::get(q)->m_running = false; break; case QQuick3DXrManagerPrivate::RenderState::Running: qCDebug(lcQuick3DXr, "Render state: Running"); QQuick3DXrManagerPrivate::get(q)->m_running = true; QCoreApplication::postEvent(q, new QEvent(QEvent::UpdateRequest)); break; case QQuick3DXrManagerPrivate::RenderState::Invalidated: qCDebug(lcQuick3DXr, "Render state: Invalidated"); QQuick3DXrManagerPrivate::get(q)->m_running = false; emit q->sessionEnded(); break; case QQuick3DXrManagerPrivate::RenderState::Paused: QQuick3DXrManagerPrivate::get(q)->m_running = false; qCDebug(lcQuick3DXr, "Render state: Paused"); break; } }, Qt::DirectConnection); QObject::connect(m_compositorLayer, &CompositorLayer::arStateChanged, q, [q](QQuick3DXrManagerPrivate::ArTrackingState state) { switch (state) { case QQuick3DXrManagerPrivate::ArTrackingState::Uninitialized: qCDebug(lcQuick3DXr, "AR state: Uninitialized"); QQuick3DXrManagerPrivate::get(q)->m_arRunning = false; break; case QQuick3DXrManagerPrivate::ArTrackingState::Initialized: qCDebug(lcQuick3DXr, "AR state: Initialized"); QQuick3DXrManagerPrivate::get(q)->m_arRunning = false; break; case QQuick3DXrManagerPrivate::ArTrackingState::Running: qCDebug(lcQuick3DXr, "AR state: Running"); QQuick3DXrManagerPrivate::get(q)->m_arRunning = true; QCoreApplication::postEvent(q, new QEvent(QEvent::UpdateRequest)); break; case QQuick3DXrManagerPrivate::ArTrackingState::Paused: qCDebug(lcQuick3DXr, "AR state: Paused"); QQuick3DXrManagerPrivate::get(q)->m_arRunning = false; break; case QQuick3DXrManagerPrivate::ArTrackingState::Stopped: qCDebug(lcQuick3DXr, "AR state: Stopped"); QQuick3DXrManagerPrivate::get(q)->m_arRunning = false; break; } }, Qt::DirectConnection); } return false; } return true; } void QQuick3DXrManagerPrivate::setupWindow(QQuickWindow *window) { if (!window) { qWarning("QQuick3DXrManagerPrivate: Window is null!"); return; } QSSG_ASSERT_X(m_compositorLayer != nullptr, "No composition layer!", return); cp_layer_renderer_t renderer = m_compositorLayer->layerRenderer(); if (!renderer) { qWarning("QQuick3DXrManagerPrivate: Layer renderer is not available."); return; } auto device = cp_layer_renderer_get_device(renderer); auto commandQueue = [device newCommandQueue]; auto qqGraphicsDevice = QQuickGraphicsDevice::fromDeviceAndCommandQueue(static_cast(device), static_cast(commandQueue)); window->setGraphicsDevice(qqGraphicsDevice); } bool QQuick3DXrManagerPrivate::finalizeGraphics(QRhi *rhi) { Q_UNUSED(rhi); m_isGraphicsInitialized = true; return m_isGraphicsInitialized; } bool QQuick3DXrManagerPrivate::isReady() const { return m_compositorLayer && (m_compositorLayer->layerRenderer() != nullptr); } bool QQuick3DXrManagerPrivate::isGraphicsInitialized() const { return m_isGraphicsInitialized; } bool QQuick3DXrManagerPrivate::setupGraphics(QQuickWindow *window) { QSSG_ASSERT(window != nullptr && m_compositorLayer != nullptr, return false); if (m_compositorLayer->isMultiviewEnabled()) { if (!m_renderThread) { m_renderThread = new QThread; m_renderThread->setObjectName(QLatin1StringView(s_renderThreadName)); m_renderThread->start(); } if (m_compositorLayer->thread() != m_renderThread) m_compositorLayer->moveToThread(m_renderThread); Q_ASSERT(m_renderThread != nullptr); QQuickWindowPrivate::get(window)->renderControl->prepareThread(m_renderThread); } // NOTE: See QQuick3DXrManager::setupGraphics() for emission of the value changed signal m_multiviewRenderingEnabled = m_compositorLayer->isMultiviewEnabled(); return true; } void QQuick3DXrManagerPrivate::getDefaultClipDistances(float &nearClip, float &farClip) const { m_compositorLayer->getDefaultDepthRange(nearClip, farClip); } void QQuick3DXrManagerPrivate::teardown() { Q_Q(QQuick3DXrManager); qCDebug(lcQuick3DXr) << "Tearing down XR session"; QMutexLocker locker { nullptr }; if (m_compositorLayer) locker = QMutexLocker{ &m_compositorLayer->renderLock() }; m_running = false; if (m_inputManager) QQuick3DXrInputManagerPrivate::get(m_inputManager)->teardown(); if (m_anchorManager) m_anchorManager->teardown(); if (m_compositorLayer) { m_compositorLayer->stopArSession(); m_compositorLayer->teardown(); // NOTE: Unlock the render mutex. The compositor layer will be destroyed on the render thread now! // Also, be aware of the race condition here once the lock is released (see: teardownStarted())... locker.unlock(); Qt::ConnectionType connection = (m_compositorLayer->thread() == QThread::currentThread()) ? Qt::DirectConnection : Qt::BlockingQueuedConnection; QMetaObject::invokeMethod(m_compositorLayer, &CompositorLayer::destroy, connection, this, q->m_quickWindow, m_compositorLayer); m_compositorLayer = nullptr; } if (m_renderThread) { m_renderThread->quit(); m_renderThread->wait(); delete m_renderThread; m_renderThread = nullptr; } } void QQuick3DXrManagerPrivate::setMultiViewRenderingEnabled(bool enable) { Q_UNUSED(enable); qCWarning(lcQuick3DXr) << "Changing multiview rendering is not supported at runtime on visionOS!"; } bool QQuick3DXrManagerPrivate::isMultiViewRenderingEnabled() const { return m_multiviewRenderingEnabled; } void QQuick3DXrManagerPrivate::setPassthroughEnabled(bool enable) { Q_UNUSED(enable); qCWarning(lcQuick3DXr) << "Changing passthrough is not supported at runtime on visionOS!"; } QtQuick3DXr::ReferenceSpace QQuick3DXrManagerPrivate::getReferenceSpace() const { // FIXME: Not sure exactly what reference space is default or what is supported etc. return QtQuick3DXr::ReferenceSpace::ReferenceSpaceLocalFloor; } void QQuick3DXrManagerPrivate::setReferenceSpace(QtQuick3DXr::ReferenceSpace newReferenceSpace) { Q_UNUSED(newReferenceSpace); qCWarning(lcQuick3DXr) << "Changing reference space is not supported at runtime on visionOS!"; } void QQuick3DXrManagerPrivate::setDepthSubmissionEnabled(bool enable) { if (!enable) qCWarning(lcQuick3DXr, "Depth submission is required on visionOS"); } void QQuick3DXrManagerPrivate::update() { Q_Q(QQuick3DXrManager); // Lock the render mutex and request a new frame to be rendered. QMutexLocker locker { &m_compositorLayer->renderLock() }; if (!m_running || !m_arRunning || !m_isGraphicsInitialized) { qCDebug(lcQuick3DXr, "Not running, skipping update"); return; } if (m_compositorLayer->isMultiviewEnabled()) { // polish (GUI thread) q->m_renderControl->polishItems(); m_syncDone = false; QCoreApplication::postEvent(m_compositorLayer, new QEvent(CompositorLayer::asQEvent(CompositorLayer::Event::Render))); // Wait for the sync to complete. bool waitCompleted = m_compositorLayer->waitForSyncToComplete(); // The gui thread can now continue. QQuick3DXrAnimationDriver *animationDriver = q->m_animationDriver; if (Q_LIKELY(waitCompleted && m_syncDone && animationDriver)) { animationDriver->setStep(m_nextStepSize); animationDriver->advance(); } else { qCDebug(lcQuick3DXr) << "Failed to wait for sync to complete" << "\nWaitCompleted:" << waitCompleted << "\nsyncDone" << m_syncDone; } } else { [[maybe_unused]] bool ret = m_compositorLayer->renderFrame(locker); } QCoreApplication::postEvent(q, new QEvent(QEvent::UpdateRequest)); } void QQuick3DXrManagerPrivate::processXrEvents() { // NOTE: This is not used on visionOS } void QQuick3DXrManagerPrivate::processSpatialEvents(const QJsonObject &events) { Q_Q(QQuick3DXrManager); QSSG_ASSERT(q->m_vrViewport != nullptr, return); QQuick3DXrInputManagerPrivate::processSpatialEvents(*q->m_vrViewport, events); } void QQuick3DXrManagerPrivate::prepareAnchorManager(QQuick3DXrAnchorManager *anchorManager, ar_data_providers_t dataProviders) { anchorManager->prepareAnchorManager(dataProviders); } void QQuick3DXrManagerPrivate::initAnchorManager(QQuick3DXrAnchorManager *anchorManager) { anchorManager->initAnchorManager(); } void QQuick3DXrManagerPrivate::initInputManager(QQuick3DXrInputManager *im) { QQuick3DXrInputManagerPrivate::get(im)->initHandtracking(); } void QQuick3DXrManagerPrivate::setSamples(int samples) { Q_UNUSED(samples); qCWarning(lcQuick3DXr) << "Changing sample count is not supported on visionOS!"; } QString QQuick3DXrManagerPrivate::runtimeName() const { return QStringLiteral("visionOS"); } QVersionNumber QQuick3DXrManagerPrivate::runtimeVersion() const { static const auto versionNumber = QOperatingSystemVersion::current().version(); return versionNumber; } QString QQuick3DXrManagerPrivate::errorString() const { return QString(); } void QQuick3DXrManagerPrivate::setupShadingRateMap(QQuickWindow *window, QRhiShadingRateMap *srm) { if (QRhiSwapChain *swapchain = window->swapChain()) { swapchain->setShadingRateMap(srm); if (!m_srmRenderPassDesc) { qCDebug(lcQuick3DXr) << "Creating render pass descriptor suitable for shading rate map use"; m_srmRenderPassDesc = swapchain->newCompatibleRenderPassDescriptor(); swapchain->setRenderPassDescriptor(m_srmRenderPassDesc); } } else { QSGRendererInterface *rif = window->rendererInterface(); QRhiTextureRenderTarget *rt = static_cast(rif->getResource(window, QSGRendererInterface::RhiRedirectRenderTarget)); if (QSSG_GUARD_X(rt, "No render target!")) { QRhiTextureRenderTargetDescription desc = rt->description(); desc.setShadingRateMap(srm); rt->setDescription(desc); if (!m_srmRenderPassDesc) { qCDebug(lcQuick3DXr) << "Creating render pass descriptor suitable for shading rate map use"; QRhiRenderPassDescriptor *rpd = rt->renderPassDescriptor(); m_srmRenderPassDesc = rpd->newCompatibleRenderPassDescriptor(); rt->setRenderPassDescriptor(m_srmRenderPassDesc); rt->create(); } } } } void QQuick3DXrManagerPrivate::releaseResources() { qCDebug(lcQuick3DXr) << "Releasing resources"; Q_ASSERT((m_renderThread == nullptr) || QThread::currentThread() == m_renderThread); delete m_srmRenderPassDesc; m_srmRenderPassDesc = nullptr; delete m_rhiDepthTexture; m_rhiDepthTexture = nullptr; for (size_t i = 0, end = std::size(m_srm); i < end; ++i) { delete m_srm[i]; m_srm[i] = nullptr; } } bool QQuick3DXrManagerPrivate::renderFrameImpl(QMutexLocker &locker, QWaitCondition &waitCondition) { Q_Q(QQuick3DXrManager); // NOTE: The GUI thread is locked at this point QQuickWindow *window = q->m_quickWindow; QQuickRenderControl *renderControl = q->m_renderControl; QQuick3DXrOrigin *xrOrigin = q->m_xrOrigin; QQuick3DViewport *xrViewport = q->m_vrViewport; QQuick3DXrAnimationDriver *animationDriver = q->m_animationDriver; QSSG_ASSERT_X(window && renderControl && xrViewport && xrOrigin && animationDriver, "Invalid state, rendering aborted", return false); const bool multiviewRenderingEnabled = isMultiViewRenderingEnabled(); Q_ASSERT(!multiviewRenderingEnabled || QThread::currentThread() != window->thread()); auto layerRenderer = m_compositorLayer->layerRenderer(); cp_frame_t frame = cp_layer_renderer_query_next_frame(layerRenderer); if (Q_UNLIKELY(frame == nullptr)) { qWarning("Failed to get next frame"); return false; } cp_frame_timing_t timing = cp_frame_predict_timing(frame); if (Q_UNLIKELY(timing == nullptr)) { qWarning("Failed to get timing for frame"); return false; } cp_frame_start_update(frame); cp_frame_end_update(frame); cp_time_t optimalInputTime = cp_frame_timing_get_optimal_input_time(timing); cp_time_wait_until(optimalInputTime); cp_frame_start_submission(frame); cp_drawable_t drawable = cp_frame_query_drawable(frame); if (Q_UNLIKELY(drawable == nullptr)) { qWarning("Failed to get drawable for frame"); return false; } cp_frame_timing_t actualTiming = cp_drawable_get_frame_timing(drawable); ar_device_anchor_t anchor = m_compositorLayer->createPoseForTiming(actualTiming); cp_drawable_set_device_anchor(drawable, anchor); // Get the pose transform from the anchor simd_float4x4 headTransform = ar_anchor_get_origin_from_anchor_transform(anchor); // NOTE: We need to convert from meters to centimeters here QMatrix4x4 qtHeadTransform{headTransform.columns[0].x, headTransform.columns[1].x, headTransform.columns[2].x, headTransform.columns[3].x * 100, headTransform.columns[0].y, headTransform.columns[1].y, headTransform.columns[2].y, headTransform.columns[3].y * 100, headTransform.columns[0].z, headTransform.columns[1].z, headTransform.columns[2].z, headTransform.columns[3].z * 100, 0.0f, 0.0f, 0.0f, 1.0f}; xrOrigin->updateTrackedCamera(qtHeadTransform); // Update the hands if (QSSG_GUARD(m_inputManager != nullptr)) QQuick3DXrInputManagerPrivate::get(m_inputManager)->updateHandtracking(); // Animation driver // Convert the cp_frame_timing_t ticks to milliseconds enum : size_t { DisplayPeriod = 0, DisplayDelta }; qint64 stepSizes[2] {0, 0}; auto &[displayPeriodMS, displayDeltaMS] = stepSizes; displayPeriodMS = qint64(cp_time_to_cf_time_interval(optimalInputTime) * 1000.0); displayDeltaMS = ((qint64(cp_time_to_cf_time_interval(cp_frame_timing_get_optimal_input_time(actualTiming)) * 1000.0)) - m_previousTime); const size_t selector = ((m_previousTime == 0) || (displayDeltaMS > displayPeriodMS)) ? DisplayPeriod : DisplayDelta; m_nextStepSize = stepSizes[selector]; m_previousTime = displayPeriodMS; if (!multiviewRenderingEnabled) { animationDriver->setStep(m_nextStepSize); animationDriver->advance(); } QRhi *rhi = renderControl->rhi(); const auto viewCount = cp_drawable_get_view_count(drawable); const auto textureCount = cp_drawable_get_texture_count(drawable); const auto renderCalls = textureCount; // To make it less confusing... bool foveationEnabled = m_compositorLayer->isFoveationEnabled(); Q_ASSERT(!foveationEnabled || rhi->isFeatureSupported(QRhi::VariableRateShadingMap)); // We still should make sure we have a rate map! if (foveationEnabled) { const size_t rrmapcount = cp_drawable_get_rasterization_rate_map_count(drawable); const bool validMapCount = (rrmapcount == 1 || rrmapcount == 2); QSSG_GUARD_X(validMapCount, "Invalid rate map count!"); foveationEnabled = validMapCount; if (foveationEnabled) { for (size_t i = 0; i < rrmapcount; ++i) { if (!m_srm[i]) m_srm[i] = rhi->newShadingRateMap(); id rrm = cp_drawable_get_rasterization_rate_map(drawable, i); const bool wasCreated = m_srm[i]->createFrom({ quint64(rrm) }); QSSG_GUARD_X(wasCreated, "Failed to create shading rate map!"); foveationEnabled = (foveationEnabled && wasCreated); } } } // NOTE: Expectation is that when multiview rendering is enabled we get a multiple drawables with a single texture array, // each view/eye is then rendered to a slice in the texture array. If multiview rendering is _not_ enabled we get // two drawables with two textures, one for each eye. In this case we render each eye to its own texture. // // And just to keep things interesting the emulator will use one view and one texture, so even if we hit the // "dedicated"/non-multiview case we need to handle that as well. NOTE: We don't do threaded rendering when // multiview rendering is disabled! for (size_t i = 0, end = renderCalls; i != end ; ++i) { // Setup the RenderTarget based on the current drawable id colorMetalTexture = cp_drawable_get_color_texture(drawable, i); auto textureSize = QSize([colorMetalTexture width], [colorMetalTexture height]); QQuickRenderTarget renderTarget; if (multiviewRenderingEnabled) renderTarget = QQuickRenderTarget::fromMetalTexture(static_cast(colorMetalTexture), [colorMetalTexture pixelFormat], [colorMetalTexture pixelFormat]/*viewFormat*/, textureSize, 1 /*sampleCount*/, viewCount, {}); else renderTarget = QQuickRenderTarget::fromMetalTexture(static_cast(colorMetalTexture), [colorMetalTexture pixelFormat], textureSize); auto depthMetalTexture = cp_drawable_get_depth_texture(drawable, i); auto depthTextureSize = QSize([depthMetalTexture width], [depthMetalTexture height]); MTLPixelFormat depthTextureFormat = [depthMetalTexture pixelFormat]; static const auto convertFormat = [](MTLPixelFormat format) -> QRhiTexture::Format { switch (format) { case MTLPixelFormatDepth16Unorm: return QRhiTexture::D16; case MTLPixelFormatDepth32Float: return QRhiTexture::D32F; default: qWarning("Unsupported depth texture format"); return QRhiTexture::UnknownFormat; } }; auto depthFormat = convertFormat(depthTextureFormat); if (depthFormat != QRhiTexture::UnknownFormat) { if (m_rhiDepthTexture && (m_rhiDepthTexture->format() != depthFormat || m_rhiDepthTexture->pixelSize() != depthTextureSize)) { delete m_rhiDepthTexture; m_rhiDepthTexture = nullptr; } if (!m_rhiDepthTexture) { if (multiviewRenderingEnabled) m_rhiDepthTexture = rhi->newTextureArray(depthFormat, viewCount, depthTextureSize, 1, QRhiTexture::RenderTarget); else m_rhiDepthTexture = rhi->newTexture(depthFormat, depthTextureSize, 1, QRhiTexture::RenderTarget); } m_rhiDepthTexture->createFrom({ quint64(static_cast(depthMetalTexture)), 0}); renderTarget.setDepthTexture(m_rhiDepthTexture); } window->setRenderTarget(renderTarget); // Initial window size is the size of the texture, but we also need to check the viewport size // as this can be different. If the viewport size is different we need to adjust the window size. // This is the case when we render to a smaller texture and then scale it up to the window (i.e. foveated rendering). QSize renderSize = textureSize; if (foveationEnabled) { cp_view_t view = cp_drawable_get_view(drawable, i); cp_view_texture_map_t texture_map = cp_view_get_view_texture_map(view); auto vp = cp_view_texture_map_get_viewport(texture_map); renderSize = QSize(vp.width, vp.height); } window->setGeometry(0, 0, renderSize.width(), renderSize.height()); window->contentItem()->setSize(renderSize); // NOTE: We set the size here as we need to make sure the window size is correct before // the sync, or the xrView's size will be incorrect. This happens because we're on the // render thread (GUI thread is blocked) and any updates to the contentItem size will // not be updated (see: QQuick3DXrView::init)... xrViewport->setSize(renderSize); // Update the camera pose if (QSSG_GUARD(xrOrigin)) { if (multiviewRenderingEnabled) updateCameraMultiview(xrViewport, headTransform, drawable, xrOrigin); else updateCamera(xrViewport, headTransform, drawable, xrOrigin, i); } // We only do this if we are rendering on a separate thread and with multiview rendering enabled, // or else the beginFrame and sync will be done on the GUI thread after both eyes have been // rendered... if (multiviewRenderingEnabled) { // Marks the start of the frame. renderControl->beginFrame(); // Synchronization happens here on the render thread (with the GUI thread locked) m_syncDone = renderControl->sync(); if (Q_UNLIKELY(!m_syncDone)) return false; if (foveationEnabled) setupShadingRateMap(window, m_srm[i]); // Signal the GUI thread that the sync is done, so it can continue. waitCondition.wakeOne(); locker.unlock(); // Render the frame renderControl->render(); // Marks the end of the frame. renderControl->endFrame(); } else { renderControl->polishItems(); renderControl->beginFrame(); renderControl->sync(); if (foveationEnabled) setupShadingRateMap(window, m_srm[i]); renderControl->render(); renderControl->endFrame(); } #if TARGET_OS_SIMULATOR == 1 // With multiview this indicates that the frame with both eyes is ready from // the 3D APIs perspective. Without multiview this is done - and so the // signal is emitted - multiple times (twice) per "frame" (eye). QRhiRenderTarget *rt = QQuickWindowPrivate::get(window)->activeCustomRhiRenderTarget(); if (rt->resourceType() == QRhiResource::TextureRenderTarget && static_cast(rt)->description().colorAttachmentAt(0)->texture()) emit q->frameReady(); #endif // TARGET_OS_SIMULATOR } id commandBuffer = [static_cast(renderControl->rhi()->nativeHandles())->cmdQueue commandBuffer]; cp_drawable_encode_present(drawable, commandBuffer); [commandBuffer commit]; cp_frame_end_submission(frame); return true; } QT_END_NAMESPACE #include "qquick3dxrmanager_visionos.moc"