QSoundEffect: use QAudioPlaybackEngine to implement new QSoundEffectHEADdev

QAudioPlaybackEngine can be reused among instances and allows us to
implement QSoundEffect in a more efficient and robust manner:
* we share a single QAudioPlaybackEngine
* we allow multiple sounds to play with the same instance

Task-number: QTBUG-127560
Task-number: QTBUG-131912
Task-number: QTBUG-133307
Pick-to: 6.10
Change-Id: I4785d42356f93ca8b3be6a330a32580b51b5ad64
Reviewed-by: Timur Pocheptsov <[email protected]>

5 files changed, 571 insertions, 2 deletions

diff --git a/src/multimedia/CMakeLists.txt b/src/multimedia/CMakeLists.txt
index 4fcc118bf..bce3a6e21 100644
--- a/src/multimedia/CMakeLists.txt
+++ b/src/multimedia/CMakeLists.txt
@@ -37,6 +37,7 @@ qt_internal_add_module(Multimedia
         audio/qaudio_rtsan_support_p.h
         audio/qsamplecache_p.cpp audio/qsamplecache_p.h
         audio/qsoundeffect.cpp audio/qsoundeffect.h
+        audio/qsoundeffectwithplayer.cpp audio/qsoundeffectwithplayer_p.h
         audio/qsoundeffectsynchronous.cpp audio/qsoundeffectsynchronous_p.h
         audio/qsoundeffect_p.h
         audio/qrtaudioengine.cpp audio/qrtaudioengine_p.h
diff --git a/src/multimedia/audio/qsoundeffect.cpp b/src/multimedia/audio/qsoundeffect.cpp
index 2184620a1..d92120932 100644
--- a/src/multimedia/audio/qsoundeffect.cpp
+++ b/src/multimedia/audio/qsoundeffect.cpp
@@ -10,8 +10,10 @@
 #include <QtMultimedia/qaudiodevice.h>
 #include <QtMultimedia/qaudiosink.h>
 #include <QtMultimedia/qmediadevices.h>
+#include <QtMultimedia/private/qaudiosystem_p.h>
 #include <QtMultimedia/private/qsamplecache_p.h>
 #include <QtMultimedia/private/qsoundeffectsynchronous_p.h>
+#include <QtMultimedia/private/qsoundeffectwithplayer_p.h>
 #include <QtMultimedia/private/qtmultimediaglobal_p.h>
 
 QT_BEGIN_NAMESPACE
@@ -19,6 +21,25 @@ QT_BEGIN_NAMESPACE
 Q_APPLICATION_STATIC(QSampleCache, sampleCache)
 Q_LOGGING_CATEGORY(qLcSoundEffect, "qt.multimedia.soundeffect")
 
+namespace {
+
+QSoundEffectPrivate *makeSoundEffectPrivate(QSoundEffect *fx, const QAudioDevice &audioDevice)
+{
+#if defined(Q_OS_MACOS) && defined(Q_OS_WIN)
+    return new QtMultimediaPrivate::QSoundEffectPrivateWithPlayer(fx, audioDevice);
+#endif
+
+    QAudioSink dummySink(audioDevice.isNull() ? QMediaDevices::defaultAudioOutput() : audioDevice);
+    auto platformSink = QPlatformAudioSink::get(dummySink);
+
+    if (platformSink && platformSink->hasCallbackAPI())
+        return new QtMultimediaPrivate::QSoundEffectPrivateWithPlayer(fx, audioDevice);
+    else
+        return new QSoundEffectPrivateSynchronous(fx, audioDevice);
+}
+
+} // namespace
+
 /*!
     \class QSoundEffect
     \brief The QSoundEffect class provides a way to play low latency sound effects.
@@ -94,7 +115,7 @@ QSoundEffect::QSoundEffect(QObject *parent)
     Creates a QSoundEffect with the given \a audioDevice and \a parent.
 */
 QSoundEffect::QSoundEffect(const QAudioDevice &audioDevice, QObject *parent)
-    : QObject(*new QSoundEffectPrivateSynchronous(this, audioDevice), parent)
+    : QObject(*makeSoundEffectPrivate(this, audioDevice), parent)
 {
 }
 
diff --git a/src/multimedia/audio/qsoundeffectsynchronous_p.h b/src/multimedia/audio/qsoundeffectsynchronous_p.h
index e2e68203a..fd6d253a1 100644
--- a/src/multimedia/audio/qsoundeffectsynchronous_p.h
+++ b/src/multimedia/audio/qsoundeffectsynchronous_p.h
@@ -22,7 +22,8 @@
 
 QT_BEGIN_NAMESPACE
 
-class QSoundEffectPrivateSynchronous : public QIODevice, public QSoundEffectPrivate
+class Q_MULTIMEDIA_EXPORT QSoundEffectPrivateSynchronous : public QIODevice,
+                                                           public QSoundEffectPrivate
 {
     struct AudioSinkDeleter
     {
diff --git a/src/multimedia/audio/qsoundeffectwithplayer.cpp b/src/multimedia/audio/qsoundeffectwithplayer.cpp
new file mode 100644
index 000000000..1a9ec7a4a
--- /dev/null
+++ b/src/multimedia/audio/qsoundeffectwithplayer.cpp
@@ -0,0 +1,421 @@
+// Copyright (C) 2025 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#include "qsoundeffectwithplayer_p.h"
+
+#include <QtCore/qmutex.h>
+#include <QtCore/q20map.h>
+
+#include <utility>
+
+QT_BEGIN_NAMESPACE
+
+namespace QtMultimediaPrivate {
+
+namespace {
+
+QSpan<const float> toFloatSpan(QSpan<const char> byteArray)
+{
+    return QSpan{
+        reinterpret_cast<const float *>(byteArray.data()),
+        qsizetype(byteArray.size_bytes() / sizeof(float)),
+    };
+}
+
+} // namespace
+
+///////////////////////////////////////////////////////////////////////////////////////////////////
+
+QSoundEffectVoice::QSoundEffectVoice(VoiceId voiceId, std::shared_ptr<const QSample> sample,
+                                     float volume, bool muted, int totalLoopCount)
+    : QRtAudioEngineVoice{ voiceId },
+      m_sample{ std::move(sample) },
+      m_volume{ volume },
+      m_muted{ muted },
+      m_loopsRemaining{ totalLoopCount }
+{
+}
+
+VoicePlayResult QSoundEffectVoice::play(QSpan<float> outputBuffer) noexcept QT_MM_NONBLOCKING
+{
+    const QAudioFormat &format = m_sample->format();
+    int totalSamples = m_totalFrames * format.channelCount();
+    int currentSample = format.channelCount() * m_currentFrame;
+
+    const QSpan fullSample = toFloatSpan(m_sample->data());
+    QSpan playbackRange = take(drop(fullSample, currentSample), totalSamples);
+
+    Q_ASSERT(!playbackRange.empty());
+
+    // later: (auto)vectorize?
+    qsizetype samplesToPlay = std::min(playbackRange.size(), outputBuffer.size());
+    if (m_muted || m_volume == 0.f) {
+        auto outputRange = take(outputBuffer, samplesToPlay);
+        std::fill(outputRange.begin(), outputRange.end(), 0.f);
+    } else if (m_volume == 1.f) {
+        for (qsizetype i = 0; i != samplesToPlay; ++i)
+            outputBuffer[i] += playbackRange[i];
+    } else {
+        for (qsizetype i = 0; i != samplesToPlay; ++i)
+            outputBuffer[i] += playbackRange[i] * m_volume;
+    }
+
+    m_currentFrame += samplesToPlay / format.channelCount();
+
+    if (m_currentFrame == m_totalFrames) {
+        const bool isInfiniteLoop = loopsRemaining() == QSoundEffect::Infinite;
+        bool continuePlaying = isInfiniteLoop;
+
+        if (!isInfiniteLoop)
+            continuePlaying = m_loopsRemaining.fetch_sub(1, std::memory_order_relaxed) > 1;
+
+        if (continuePlaying) {
+            if (!isInfiniteLoop)
+                m_currentLoopChanged.set();
+            m_currentFrame = 0;
+            QSpan remainingOutputBuffer = drop(outputBuffer, samplesToPlay);
+            return play(remainingOutputBuffer);
+        }
+        return VoicePlayResult::Finished;
+    }
+    return VoicePlayResult::Playing;
+}
+
+bool QSoundEffectVoice::isActive() noexcept QT_MM_NONBLOCKING
+{
+    if (m_currentFrame != m_totalFrames)
+        return true;
+
+    return loopsRemaining() != 0;
+}
+
+std::shared_ptr<QSoundEffectVoice> QSoundEffectVoice::clone() const
+{
+    auto clone = std::make_shared<QSoundEffectVoice>(QRtAudioEngine::allocateVoiceId(), m_sample,
+                                                     m_volume, m_muted, loopsRemaining());
+
+    // caveat: reading frame is not atomic, so we may have a race here ... is is rare, though,
+    // not sure if we really care
+    clone->m_currentFrame = m_currentFrame;
+    return clone;
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////////////
+
+QSoundEffectPrivateWithPlayer::QSoundEffectPrivateWithPlayer(QSoundEffect *q,
+                                                             QAudioDevice audioDevice)
+    : q_ptr{ q }, m_audioDevice{ std::move(audioDevice) }
+{
+    resolveAudioDevice();
+
+    QObject::connect(&m_mediaDevices, &QMediaDevices::audioOutputsChanged, this, [this] {
+        QAudioDevice defaultAudioDevice = QMediaDevices::defaultAudioOutput();
+        if (defaultAudioDevice == m_defaultAudioDevice)
+            return;
+
+        m_defaultAudioDevice = QMediaDevices::defaultAudioOutput();
+        if (m_audioDevice.isNull())
+            setResolvedAudioDevice(m_defaultAudioDevice);
+    });
+}
+
+QSoundEffectPrivateWithPlayer::~QSoundEffectPrivateWithPlayer()
+{
+    stop();
+}
+
+bool QSoundEffectPrivateWithPlayer::setAudioDevice(QAudioDevice device)
+{
+    if (device == m_audioDevice)
+        return false;
+
+    m_audioDevice = std::move(device);
+    resolveAudioDevice();
+    return true;
+}
+
+void QSoundEffectPrivateWithPlayer::setResolvedAudioDevice(QAudioDevice device)
+{
+    if (m_resolvedAudioDevice == device)
+        return;
+
+    m_resolvedAudioDevice = std::move(device);
+
+    if (!m_player)
+        return;
+
+    for (const auto &voice : m_voices)
+        m_player->stop(voice);
+
+    std::vector<std::shared_ptr<QSoundEffectVoice>> voices{
+        std::make_move_iterator(m_voices.begin()), std::make_move_iterator(m_voices.end())
+    };
+    m_voices.clear();
+
+    bool hasPlayer = updatePlayer();
+    if (!hasPlayer)
+        return;
+
+    for (const auto &voice : voices)
+        // we re-allocate a new voice ID and play on the new player
+        play(voice->clone());
+}
+
+void QSoundEffectPrivateWithPlayer::resolveAudioDevice()
+{
+    if (m_audioDevice.isNull())
+        m_defaultAudioDevice = QMediaDevices::defaultAudioOutput();
+    setResolvedAudioDevice(m_audioDevice.isNull() ? m_defaultAudioDevice : m_audioDevice);
+}
+
+QAudioDevice QSoundEffectPrivateWithPlayer::audioDevice() const
+{
+    return m_audioDevice;
+}
+
+bool QSoundEffectPrivateWithPlayer::setSource(const QUrl &url, QSampleCache &sampleCache)
+{
+    if (m_sampleLoadFuture.isValid())
+        m_sampleLoadFuture.cancel();
+
+    m_url = url;
+    m_sample = {};
+
+    if (url.isEmpty()) {
+        setStatus(QSoundEffect::Null);
+        return false;
+    }
+
+    if (!url.isValid()) {
+        setStatus(QSoundEffect::Error);
+        return false;
+    }
+
+    setStatus(QSoundEffect::Loading);
+
+    m_sampleLoadFuture =
+            sampleCache.requestSampleFuture(url).then(this, [this](SharedSamplePtr result) {
+        if (result) {
+            if (!formatIsSupported(result->format())) {
+                qWarning("QSoundEffect: QSoundEffect only supports mono or stereo files");
+                setStatus(QSoundEffect::Error);
+                return;
+            }
+
+            m_sample = std::move(result);
+            setStatus(QSoundEffect::Ready);
+            bool hasPlayer = updatePlayer();
+            if (std::exchange(m_playPending, false)) {
+                if (hasPlayer)
+                    play();
+            }
+        } else {
+            qWarning("QSoundEffect: Error decoding source %ls", qUtf16Printable(m_url.toString()));
+            setStatus(QSoundEffect::Error);
+        }
+    });
+
+    return true;
+}
+
+QUrl QSoundEffectPrivateWithPlayer::url() const
+{
+    return m_url;
+}
+
+void QSoundEffectPrivateWithPlayer::setStatus(QSoundEffect::Status status)
+{
+    if (status == m_status)
+        return;
+    m_status = status;
+    emit q_ptr->statusChanged();
+}
+
+QSoundEffect::Status QSoundEffectPrivateWithPlayer::status() const
+{
+    return m_status;
+}
+
+int QSoundEffectPrivateWithPlayer::loopCount() const
+{
+    return m_loopCount;
+}
+
+bool QSoundEffectPrivateWithPlayer::setLoopCount(int loopCount)
+{
+    if (loopCount == 0)
+        loopCount = 1;
+
+    if (loopCount == m_loopCount)
+        return false;
+
+    m_loopCount = loopCount;
+
+    if (m_voices.empty())
+        return true;
+
+    const std::shared_ptr<QSoundEffectVoice> &voice = *m_voices.rbegin();
+    voice->m_loopsRemaining.store(loopCount, std::memory_order_relaxed);
+
+    setLoopsRemaining(loopCount);
+
+    return true;
+}
+
+int QSoundEffectPrivateWithPlayer::loopsRemaining() const
+{
+    if (m_voices.empty())
+        return 0;
+
+    return m_loopsRemaining;
+}
+
+float QSoundEffectPrivateWithPlayer::volume() const
+{
+    return m_volume;
+}
+
+bool QSoundEffectPrivateWithPlayer::setVolume(float volume)
+{
+    if (m_volume == volume)
+        return false;
+
+    m_volume = volume;
+    for (const auto &voice : m_voices) {
+        m_player->visitVoiceRt(voice, [volume](QSoundEffectVoice &voice) {
+            voice.m_volume = volume;
+        });
+    }
+    return true;
+}
+
+bool QSoundEffectPrivateWithPlayer::muted() const
+{
+    return m_muted;
+}
+
+bool QSoundEffectPrivateWithPlayer::setMuted(bool muted)
+{
+    if (m_muted == muted)
+        return false;
+
+    m_muted = muted;
+    for (const auto &voice : m_voices) {
+        m_player->visitVoiceRt(voice, [muted](QSoundEffectVoice &voice) {
+            voice.m_muted = muted;
+        });
+    }
+    return true;
+}
+
+void QSoundEffectPrivateWithPlayer::play()
+{
+    if (!m_sample) {
+        m_playPending = true;
+        return;
+    }
+
+    // each `play` will start a new voice
+    Q_ASSERT(m_player);
+
+    auto voice = std::make_shared<QSoundEffectVoice>(QRtAudioEngine::allocateVoiceId(), m_sample,
+                                                     m_volume, m_muted, m_loopCount);
+
+    play(std::move(voice));
+}
+
+void QSoundEffectPrivateWithPlayer::stop()
+{
+    size_t activeVoices = m_voices.size();
+    for (const auto &voice : m_voices)
+        m_player->stop(voice->voiceId());
+    setLoopsRemaining(0);
+
+    m_voices.clear();
+    m_playPending = false;
+    if (activeVoices)
+        emit q_ptr->playingChanged();
+}
+
+bool QSoundEffectPrivateWithPlayer::playing() const
+{
+    return !m_voices.empty();
+}
+
+void QSoundEffectPrivateWithPlayer::play(std::shared_ptr<QSoundEffectVoice> voice)
+{
+    QObject::connect(&voice->m_currentLoopChanged, &QAutoResetEvent::activated, this,
+                     [this, voiceId = voice->voiceId()] {
+        auto foundVoice = m_voices.find(voiceId);
+        if (foundVoice == m_voices.end())
+            return;
+
+        if (voiceId != activeVoice())
+            return;
+
+        setLoopsRemaining((*foundVoice)->loopsRemaining());
+    });
+
+    m_player->play(voice);
+    m_voices.insert(std::move(voice));
+    setLoopsRemaining(m_loopCount);
+    if (m_voices.size() == 1)
+        emit q_ptr->playingChanged();
+}
+
+bool QSoundEffectPrivateWithPlayer::updatePlayer()
+{
+    Q_ASSERT(m_voices.empty());
+    QObject::disconnect(m_voiceFinishedConnection);
+
+    m_player = {};
+    if (m_resolvedAudioDevice.isNull())
+        return false;
+
+    auto player = QRtAudioEngine::getEngineFor(m_resolvedAudioDevice, m_sample->format());
+    m_player = player;
+
+    m_voiceFinishedConnection = QObject::connect(m_player.get(), &QRtAudioEngine::voiceFinished,
+                                                 this, [this](VoiceId voiceId) {
+        if (voiceId == activeVoice())
+            setLoopsRemaining(0);
+
+        auto found = m_voices.find(voiceId);
+        if (found != m_voices.end()) {
+            m_voices.erase(found);
+            if (m_voices.empty())
+                emit q_ptr->playingChanged();
+        }
+    });
+    return true;
+}
+
+std::optional<VoiceId> QSoundEffectPrivateWithPlayer::activeVoice() const
+{
+    if (m_voices.empty())
+        return std::nullopt;
+    return (*m_voices.rbegin())->voiceId();
+}
+
+bool QSoundEffectPrivateWithPlayer::formatIsSupported(const QAudioFormat &fmt)
+{
+    switch (fmt.channelCount()) {
+    case 1:
+    case 2:
+        return true;
+    default:
+        return false;
+    }
+}
+
+void QSoundEffectPrivateWithPlayer::setLoopsRemaining(int loopsRemaining)
+{
+    if (loopsRemaining == m_loopsRemaining)
+        return;
+    m_loopsRemaining = loopsRemaining;
+    emit q_ptr->loopsRemainingChanged();
+}
+
+} // namespace QtMultimediaPrivate
+
+QT_END_NAMESPACE
diff --git a/src/multimedia/audio/qsoundeffectwithplayer_p.h b/src/multimedia/audio/qsoundeffectwithplayer_p.h
new file mode 100644
index 000000000..577df8e52
--- /dev/null
+++ b/src/multimedia/audio/qsoundeffectwithplayer_p.h
@@ -0,0 +1,125 @@
+// Copyright (C) 2025 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#ifndef QSOUNDEFFECTWITHPLAYER_P_H
+#define QSOUNDEFFECTWITHPLAYER_P_H
+
+//
+//  W A R N I N G
+//  -------------
+//
+// This file is not part of the Qt API. It exists purely as an
+// implementation detail. This header file may change from version to
+// version without notice, or even be removed.
+//
+// We mean it.
+//
+
+#include <QtMultimedia/qaudiosink.h>
+#include <QtMultimedia/qmediadevices.h>
+#include <QtMultimedia/private/qaudiosystem_p.h>
+#include <QtMultimedia/private/qautoresetevent_p.h>
+#include <QtMultimedia/private/qrtaudioengine_p.h>
+#include <QtMultimedia/private/qsoundeffect_p.h>
+
+QT_BEGIN_NAMESPACE
+
+namespace QtMultimediaPrivate {
+
+using QtPrivate::QAutoResetEvent;
+
+class QSoundEffectVoice final : public QRtAudioEngineVoice
+{
+public:
+    QSoundEffectVoice(VoiceId voiceId, std::shared_ptr<const QSample> sample, float volume,
+                      bool muted, int totalLoopCount);
+
+    VoicePlayResult play(QSpan<float>) noexcept QT_MM_NONBLOCKING override;
+    bool isActive() noexcept QT_MM_NONBLOCKING override;
+    const QAudioFormat &format() noexcept override { return m_sample->format(); }
+
+    int loopsRemaining() const { return m_loopsRemaining.load(std::memory_order_relaxed); }
+    std::shared_ptr<QSoundEffectVoice> clone() const;
+
+    const std::shared_ptr<const QSample> m_sample;
+    const int m_totalFrames{
+        m_sample->format().framesForBytes(m_sample->data().size()),
+    };
+
+    float m_volume{};
+    bool m_muted{};
+
+    std::atomic_int m_loopsRemaining;
+    int m_currentFrame{};
+
+    QAutoResetEvent m_currentLoopChanged;
+};
+
+// Design notes
+// * each play() will start a new voice
+// * stop() will stop all voices
+// * the most recently created voice can be obtained via `activeVoice()`
+// * mute/volume will affect all voices
+// * loopCount/loopRemaining will be obtained from most recently created voice
+class QSoundEffectPrivateWithPlayer final : public QObject, public QSoundEffectPrivate
+{
+public:
+    QSoundEffectPrivateWithPlayer(QSoundEffect *q, QAudioDevice audioDevice);
+    Q_DISABLE_COPY_MOVE(QSoundEffectPrivateWithPlayer)
+    ~QSoundEffectPrivateWithPlayer() override;
+
+    // QSoundEffectPrivate interface
+    bool setAudioDevice(QAudioDevice device) override;
+    QAudioDevice audioDevice() const override;
+    bool setSource(const QUrl &, QSampleCache &) override;
+    QUrl url() const override;
+    QSoundEffect::Status status() const override;
+    int loopCount() const override;
+    bool setLoopCount(int) override;
+    int loopsRemaining() const override;
+    float volume() const override;
+    bool setVolume(float) override;
+    bool muted() const override;
+    bool setMuted(bool) override;
+    void play() override;
+    void stop() override;
+    bool playing() const override;
+
+private:
+    void play(std::shared_ptr<QSoundEffectVoice>);
+    void setStatus(QSoundEffect::Status status);
+    [[nodiscard]] bool updatePlayer();
+    std::optional<VoiceId> activeVoice() const;
+    static bool formatIsSupported(const QAudioFormat &);
+    void setResolvedAudioDevice(QAudioDevice device);
+    void resolveAudioDevice();
+
+    QSoundEffect *q_ptr{};
+    QAudioDevice m_audioDevice;
+    QAudioDevice m_resolvedAudioDevice;
+    std::shared_ptr<QRtAudioEngine> m_player;
+    QMetaObject::Connection m_voiceFinishedConnection;
+    std::set<std::shared_ptr<QSoundEffectVoice>, QRtAudioEngineVoiceCompare> m_voices;
+
+    void setLoopsRemaining(int);
+    int m_loopsRemaining{ 0 };
+
+    QFuture<void> m_sampleLoadFuture;
+    QUrl m_url;
+    SharedSamplePtr m_sample;
+    float m_volume = 1.f;
+    bool m_muted = false;
+    int m_loopCount = 1;
+    bool m_playPending = false;
+
+    QSoundEffect::Status m_status{};
+
+    QMediaDevices m_mediaDevices;
+    QAudioDevice m_defaultAudioDevice;
+};
+
+} // namespace QtMultimediaPrivate
+
+QT_END_NAMESPACE
+
+#endif // QSOUNDEFFECTWITHPLAYER_P_H