Client Hints: Improve version lists

There are several issues affecting Sec-CH-UA and -Full-Version-List,
because the general algorithm which extends and shuffles these lists
was not used in WebEngine.

At the current version of Chromium we can't use the corresponding user
agent utils functions, because they only allow one extra brand/version
pair besides the two generated (Chromium and Not-A-Brand) ones.
This change implements a simplified method to extend and shuffle the
version lists to follow the behavior of Chromium and support more than
three elements.

QTBUG-133799: The order of brands and versions are still not
customizable by the API user, but it is permutated in each major
version of Chromium to comply with the standards. To allow
customization of the order seems to be impossible without changing our
public API.

QTBUG-133708: The missing values of Sec-CH-UA are generated from the
full version list, not customizable individually. This way it keeps
the same values and the same algorithmically generated order as the
full version list.

Task-number: QTBUG-133711
Task-number: QTBUG-133708
Task-number: QTBUG-133777
Task-number: QTBUG-133799
Change-Id: I96f214ce54190666a34779130a04b56f600abef7
Reviewed-by: Peter Varga <[email protected]>
Reviewed-by: Moss Heim <[email protected]>

Author: davidsz

src/core/api/qwebengineclienthints.cpp

@@ -132,10 +132,14 @@ QString QWebEngineClientHints::bitness() const
 
 /*!
     \property QWebEngineClientHints::fullVersionList
-    The value of the \c{Sec-CH-UA-Full-Version-List} HTTP header and \c{fullVersionList} member of NavigatorUAData in JavaScript.
+    The value of the \c{Sec-CH-UA-Full-Version-List} HTTP header and \c{fullVersionList} member of
+   NavigatorUAData in JavaScript.
 
-    It holds brand name and version number pairs in a QVariantMap. The provided values will be automatically extended by the currently used version
-    of Chromium and a semi-random brand.
+    The value of Sec-CH-UA header will also be generated from this by truncating the version
+   numbers.
+
+    It holds brand name and version number pairs in a QVariantMap. The provided values will be
+   automatically extended by the currently used version of Chromium and a semi-random brand.
 */
 QVariantMap QWebEngineClientHints::fullVersionList() const
 {

src/core/profile_adapter.cpp

@@ -6,6 +6,7 @@
 #include "base/files/file_util.h"
 #include "base/task/cancelable_task_tracker.h"
 #include "base/threading/thread_restrictions.h"
+#include "base/version_info/version_info.h"
 #include "components/embedder_support/user_agent_utils.h"
 #include "components/favicon/core/favicon_service.h"
 #include "components/history/content/browser/history_database_helper.h"
@@ -53,6 +54,36 @@ inline QString buildLocationFromStandardPath(const QString &standardPath, const
     location += "/QtWebEngine/"_L1 % name;
     return location;
 }
+
+void PopulateBrandVersionLists(const QJsonObject &fullVersionList,
+                               blink::UserAgentMetadata &userAgentMetadata)
+{
+    userAgentMetadata.brand_version_list.clear();
+    userAgentMetadata.brand_full_version_list.clear();
+
+    for (const QString &key : fullVersionList.keys()) {
+        std::string version = fullVersionList.value(key).toString().toStdString();
+        userAgentMetadata.brand_full_version_list.push_back({ key.toStdString(), version });
+        version = version.substr(0, version.find('.'));
+        userAgentMetadata.brand_version_list.push_back({ key.toStdString(), version });
+    }
+
+    // Shuffle the lists
+    int permutations = 1;
+    for (int i = 2; i <= fullVersionList.size(); i++)
+        permutations *= i;
+    // We keep the brand lists identical throughout the lifetime of each major version of Chromium.
+    permutations = version_info::GetMajorVersionNumberAsInt() % permutations;
+    auto compare = [](blink::UserAgentBrandVersion &a, blink::UserAgentBrandVersion &b) {
+        return a.brand + a.version < b.brand + b.version;
+    };
+    for (int i = 0; i < permutations; i++) {
+        std::next_permutation(userAgentMetadata.brand_version_list.begin(),
+                              userAgentMetadata.brand_version_list.end(), compare);
+        std::next_permutation(userAgentMetadata.brand_full_version_list.begin(),
+                              userAgentMetadata.brand_full_version_list.end(), compare);
+    }
+}
 }
 
 namespace QtWebEngineCore {
@@ -713,13 +744,7 @@ void ProfileAdapter::setClientHint(ClientHint clientHint, const QVariant &value)
         userAgentMetadata.bitness = value.toString().toStdString();
         break;
     case ProfileAdapter::UAFullVersionList: {
-        userAgentMetadata.brand_full_version_list.clear();
-        QJsonObject fullVersionList = value.toJsonObject();
-        for (const QString &key : fullVersionList.keys())
-            userAgentMetadata.brand_full_version_list.push_back({
-                key.toStdString(),
-                fullVersionList.value(key).toString().toStdString()
-            });
+        PopulateBrandVersionLists(value.toJsonObject(), userAgentMetadata);
         break;
     }
     case ProfileAdapter::UAWOW64:
@@ -758,7 +783,7 @@ void ProfileAdapter::setClientHintsEnabled(bool enabled)
 
 void ProfileAdapter::resetClientHints()
 {
-    m_profile->m_userAgentMetadata = embedder_support::GetUserAgentMetadata();
+    m_profile->initUserAgentMetadata();
     std::vector<content::WebContentsImpl *> list = content::WebContentsImpl::GetAllWebContents();
     for (content::WebContentsImpl *web_contents : list) {
         if (web_contents->GetBrowserContext() == m_profile.data()) {

src/core/profile_adapter.h

@@ -200,7 +200,6 @@ class Q_WEBENGINECORE_EXPORT ProfileAdapter : public QObject
     void setClientHintsEnabled(bool enabled);
     void resetClientHints();
 
-
     void clearHttpCache();
 
 #if QT_CONFIG(ssl)

src/core/profile_qt.cpp

@@ -20,6 +20,7 @@
 #include "base/path_service.h"
 #include "base/files/file_util.h"
 #include "base/task/thread_pool.h"
+#include "base/version_info/version_info.h"
 #include "components/keyed_service/content/browser_context_dependency_manager.h"
 #include "components/prefs/pref_service.h"
 #include "components/user_prefs/user_prefs.h"
@@ -48,9 +49,7 @@ enum {
 };
 
 ProfileQt::ProfileQt(ProfileAdapter *profileAdapter)
-    : m_profileIOData(new ProfileIODataQt(this))
-    , m_profileAdapter(profileAdapter)
-    , m_userAgentMetadata(embedder_support::GetUserAgentMetadata())
+    : m_profileIOData(new ProfileIODataQt(this)), m_profileAdapter(profileAdapter)
 {
     profile_metrics::SetBrowserProfileType(this, IsOffTheRecord()
         ? profile_metrics::BrowserProfileType::kIncognito
@@ -68,6 +67,8 @@ ProfileQt::ProfileQt(ProfileAdapter *profileAdapter)
 #if BUILDFLAG(ENABLE_EXTENSIONS)
     static_cast<extensions::ExtensionSystemQt*>(extensions::ExtensionSystem::Get(this))->InitForRegularProfile(true);
 #endif
+
+    initUserAgentMetadata();
 }
 
 ProfileQt::~ProfileQt()
@@ -300,6 +301,35 @@ const PrefServiceAdapter &ProfileQt::prefServiceAdapter() const
     return m_prefServiceAdapter;
 }
 
+void ProfileQt::initUserAgentMetadata()
+{
+    m_userAgentMetadata = embedder_support::GetUserAgentMetadata();
+    m_userAgentMetadata.brand_version_list.clear();
+    m_userAgentMetadata.brand_full_version_list.clear();
+
+    // Chromium version
+    m_userAgentMetadata.brand_version_list.emplace_back(
+            blink::UserAgentBrandVersion("Chromium", version_info::GetMajorVersionNumber()));
+    m_userAgentMetadata.brand_full_version_list.emplace_back(blink::UserAgentBrandVersion(
+            "Chromium", std::string(version_info::GetVersionNumber())));
+
+    // We keep the brand lists identical throughout the lifetime of each major version of Chromium.
+    int seed = version_info::GetMajorVersionNumberAsInt();
+    // Generate a greasey version
+    const std::vector<std::vector<int>> orders{ { 0, 1, 2 }, { 0, 2, 1 }, { 1, 0, 2 },
+                                                { 1, 2, 0 }, { 2, 0, 1 }, { 2, 1, 0 } };
+    const std::vector<int> order = orders[seed % 6];
+    m_userAgentMetadata.brand_version_list.emplace_back(
+            embedder_support::GetGreasedUserAgentBrandVersion(
+                    order, seed, std::nullopt, std::nullopt, true,
+                    blink::UserAgentBrandVersionType::kMajorVersion));
+
+    m_userAgentMetadata.brand_full_version_list.emplace_back(
+            embedder_support::GetGreasedUserAgentBrandVersion(
+                    order, seed, std::nullopt, std::nullopt, true,
+                    blink::UserAgentBrandVersionType::kFullVersion));
+}
+
 const blink::UserAgentMetadata &ProfileQt::userAgentMetadata()
 {
     return m_userAgentMetadata;

src/core/profile_qt.h

@@ -75,6 +75,7 @@ class ProfileQt : public Profile
     PrefServiceAdapter &prefServiceAdapter();
     const PrefServiceAdapter &prefServiceAdapter() const;
 
+    void initUserAgentMetadata();
     const blink::UserAgentMetadata &userAgentMetadata();
 
 private:

tests/auto/widgets/qwebenginepage/tst_qwebenginepage.cpp

@@ -5482,12 +5482,16 @@ void tst_QWebEnginePage::clientHints()
     QSignalSpy loadSpy(&page, SIGNAL(loadFinished(bool)));
 
     QWebEngineClientHints *clientHints = page.profile()->clientHints();
+    // fullVersionList is extended by Chromium and a greased brand by default
+    QCOMPARE(clientHints->fullVersionList().size(), 2);
+    QVERIFY(clientHints->fullVersionList().contains("Chromium"));
     clientHints->setAllClientHintsEnabled(clientHintsEnabled);
 
     HttpServer server;
     int requestCount = 0;
-    connect(&server, &HttpServer::newRequest, [&] (HttpReqRep *r) {
-        // Platform and Mobile hints are always sent and can't be disabled with this API
+    connect(&server, &HttpServer::newRequest, [&](HttpReqRep *r) {
+        // UA, Platform and Mobile hints are always sent and can't be disabled with this API
+        QVERIFY(r->hasRequestHeader("Sec-CH-UA"));
         QVERIFY(r->hasRequestHeader("Sec-CH-UA-Platform"));
         QVERIFY(r->hasRequestHeader("Sec-CH-UA-Mobile"));
         if (!clientHintsEnabled) {
@@ -5512,9 +5516,10 @@ void tst_QWebEnginePage::clientHints()
             QCOMPARE(QString(r->requestHeader("Sec-CH-UA-Platform-Version")).remove("\""), platformVersion.toLower());
             QCOMPARE(QString(r->requestHeader("Sec-CH-UA-Bitness")).remove("\""), bitness.toLower());
             QCOMPARE(QString(r->requestHeader("Sec-CH-UA-Wow64")).remove("\""), isWOW64 ? "?1" : "?0");
-            for (auto i = fullVersionList.cbegin(), end = fullVersionList.cend(); i != end; ++i)
+            for (auto i = fullVersionList.cbegin(), end = fullVersionList.cend(); i != end; ++i) {
+                QVERIFY(QString(r->requestHeader("Sec-CH-UA")).contains(i.key().toLower()));
                 QVERIFY(QString(r->requestHeader("Sec-CH-UA-Full-Version-List")).contains(i.key().toLower()));
-
+            }
             for (auto formFactor : formFactors)
                 QVERIFY(QString(r->requestHeader("Sec-CH-UA-Form-Factors")).contains(formFactor.toLower()));
         }
@@ -5549,8 +5554,10 @@ void tst_QWebEnginePage::clientHints()
     QCOMPARE(clientHints->bitness(), bitness);
     QCOMPARE(clientHints->isWow64(), isWOW64);
     QCOMPARE(clientHints->formFactors(), formFactors);
+    QCOMPARE(clientHints->fullVersionList().size(), fullVersionList.size());
     for (auto i = fullVersionList.cbegin(), end = fullVersionList.cend(); i != end; ++i)
         QCOMPARE(clientHints->fullVersionList()[i.key()], i.value());
+    QVERIFY(!clientHints->fullVersionList().contains("Chromium"));
 
     // A new user agent string should not override/disable client hints
     page.profile()->setHttpUserAgent(QStringLiteral("Custom user agent"));
@@ -5571,6 +5578,7 @@ void tst_QWebEnginePage::clientHints()
     QCOMPARE_NE(clientHints->platformVersion(), platformVersion);
     QCOMPARE_NE(clientHints->bitness(), bitness);
     QCOMPARE_NE(clientHints->formFactors(), formFactors);
+    QCOMPARE(clientHints->fullVersionList().size(), 2);
     for (auto i = fullVersionList.cbegin(), end = fullVersionList.cend(); i != end; ++i)
         QVERIFY(!clientHints->fullVersionList().contains(i.key()));
     QVERIFY(clientHints->fullVersionList().contains("Chromium"));