Merge pull request seleniumbase#3405 from seleniumbase/cdp-mode-patch-23

CDP Mode - Patch 23

Author: mdmintz

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2014-2024 Michael Mintz
+Copyright (c) 2014-2025 Michael Mintz
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

examples/cdp_mode/ReadMe.md

@@ -6,11 +6,16 @@
 
 --------
 
-<!-- YouTube View --><a href="https://www.youtube.com/watch?v=Mr90iQmNsKM"><img src="http://img.youtube.com/vi/Mr90iQmNsKM/0.jpg" title="SeleniumBase on YouTube" width="366" /></a>
+<!-- YouTube View --><a href="https://www.youtube.com/watch?v=Mr90iQmNsKM"><img src="https://github.com/user-attachments/assets/91e7ff7b-d155-4ba9-b17b-b097825fcf42" title="SeleniumBase on YouTube" width="350" /></a>
 <p>(<b><a href="https://www.youtube.com/watch?v=Mr90iQmNsKM">Watch the CDP Mode tutorial on YouTube! ▶️</a></b>)</p>
 
 --------
 
+<!-- YouTube View --><a href="https://www.youtube.com/watch?v=vt2zsdiNh3U"><img src="https://github.com/user-attachments/assets/82ab2715-727e-4d09-9314-b8905795dc43" title="SeleniumBase on YouTube" width="350" /></a>
+<p>(<b><a href="https://www.youtube.com/watch?v=vt2zsdiNh3U">Watch "Hacking websites with CDP" on YouTube! ▶️</a></b>)</p>
+
+--------
+
 👤 <b translate="no">UC Mode</b> avoids bot-detection by first disconnecting WebDriver from the browser at strategic times, calling special <code>PyAutoGUI</code> methods to bypass CAPTCHAs (as needed), and finally reconnecting the <code>driver</code> afterwards so that WebDriver actions can be performed again. Although this approach works for bypassing simple CAPTCHAs, more flexibility is needed for bypassing bot-detection on websites with advanced protection. (That's where <b translate="no">CDP Mode</b> comes in.)
 
 🐙 <b translate="no">CDP Mode</b> is based on <a href="https://github.com/HyperionGray/python-chrome-devtools-protocol" translate="no">python-cdp</a>, <a href="https://github.com/HyperionGray/trio-chrome-devtools-protocol" translate="no">trio-cdp</a>, and <a href="https://github.com/ultrafunkamsterdam/nodriver" translate="no">nodriver</a>. <code>trio-cdp</code> is an early implementation of <code>python-cdp</code>, and <code>nodriver</code> is a modern implementation of <code>python-cdp</code>. (Refactored <code>Python-CDP</code> code is imported from <a href="https://github.com/mdmintz/MyCDP" translate="no">MyCDP</a>.)

examples/presenter/hacking_with_cdp.py

@@ -0,0 +1,146 @@
+# https://www.youtube.com/watch?v=vt2zsdiNh3U
+from seleniumbase import BaseCase
+BaseCase.main(__name__, __file__)
+
+
+class UCPresentationClass(BaseCase):
+    def test_hacking_with_cdp(self):
+        self.open("data:,")
+        self.set_window_position(4, 40)
+        self._output_file_saves = False
+        self.create_presentation(theme="serif", transition="none")
+        self.add_slide("<h2>Press SPACE to begin!</h2>\n")
+        self.add_slide(
+            "<p><h3><mk-0>Coming up on the Hacker Show...</mk-0></h3></p>\n"
+            "<hr /><ul>\n"
+            '<img src="https://seleniumbase.io/other/hackers_at_comp.jpg"'
+            ' width="100%">'
+            "</ul>",
+        )
+        self.add_slide(
+            "<p><b>Coming up on the Hacker Show...</b></p>\n"
+            "<hr /><br /><ul>\n"
+            "<li><mk-0>Intercepting requests/responses/XHR with CDP."
+            "</mk-0></li><br />\n"
+            "<br /><br />\n"
+            "<br /><br />\n"
+            "<br /><br />\n"
+            "<br /><br />\n"
+            "</ul>",
+        )
+        self.add_slide(
+            "<p><b>Coming up on the Hacker Show...</b></p>\n"
+            "<hr /><br /><ul>\n"
+            "<li>Intercepting requests/responses/XHR with CDP."
+            "</li><br />\n"
+            "<li><mk-0>Modifying requests: CDP.Fetch.continueRequest."
+            "</mk-0></li><br />\n"
+            "<br /><br />\n"
+            "<br /><br />\n"
+            "<br /><br />\n"
+            "</ul>",
+        )
+        self.add_slide(
+            "<p><b>Coming up on the Hacker Show...</b></p>\n"
+            "<hr /><br /><ul>\n"
+            "<li>Intercepting requests/responses/XHR with CDP."
+            "</li><br />\n"
+            "<li>Modifying requests: CDP.Fetch.continueRequest."
+            "</li><br />\n"
+            "<li><mk-0>Controlling browsers via remote-debugging-port"
+            "</mk-0></li><br />\n"
+            "<br /><br />\n"
+            "<br /><br />\n"
+            "</ul>",
+        )
+        self.add_slide(
+            "<p><b>Coming up on the Hacker Show...</b></p>\n"
+            "<hr /><br /><ul>\n"
+            "<li>Intercepting requests/responses/XHR with CDP."
+            "</li><br />\n"
+            "<li>Modifying requests: CDP.Fetch.continueRequest."
+            "</li><br />\n"
+            "<li>Controlling browsers via remote-debugging-port"
+            "</li><br />\n"
+            "<li><mk-0>Bypassing CAPTCHAs & anti-bot defenses."
+            "</mk-0></li><br />\n"
+            "<br /><br />\n"
+            "</ul>",
+        )
+        self.add_slide(
+            "<p><b>Coming up on the Hacker Show...</b></p>\n"
+            "<hr /><br /><ul>\n"
+            "<li>Intercepting requests/responses/XHR with CDP."
+            "</li><br />\n"
+            "<li>Modifying requests: CDP.Fetch.continueRequest."
+            "</li><br />\n"
+            "<li>Controlling browsers via remote-debugging-port"
+            "</li><br />\n"
+            "<li>Bypassing CAPTCHAs & anti-bot defenses."
+            "</li><br />\n"
+            "<li><mk-0>And live demos of all the above... with Python!"
+            "</mk-0></li><br />\n"
+            "</ul>",
+        )
+        self.add_slide(
+            "<h2>Get ready for some<br />serious hacking!</h2>"
+        )
+        self.add_slide(
+            '<img src="https://seleniumbase.io/other/hacking_with_cdp.jpg"'
+            ' width="100%">'
+        )
+        self.add_slide(
+            '<img src="https://seleniumbase.io/other/cdp.jpg"'
+            ' width="100%">'
+        )
+        self.add_slide(
+            '<img src="https://seleniumbase.io/other/ms_edp.jpg"'
+            ' width="100%">'
+        )
+        self.add_slide(
+            '<img src="https://seleniumbase.io/other/vid4_on_yt.jpg"'
+            ' width="100%">'
+        )
+        self.add_slide(
+            '<img src="https://seleniumbase.io/other/cdp_in_sb.jpg"'
+            ' width="100%">'
+        )
+        self.add_slide(
+            '<img src="https://seleniumbase.io/other/hacker_news.png"'
+            ' width="100%">'
+        )
+        self.add_slide(
+            '<img src="https://seleniumbase.io/other/sb_star_history_3.png"'
+            ' width="100%">'
+        )
+        self.add_slide(
+            '<img src="https://seleniumbase.io/other/top_trending_month.png"'
+            ' width="100%">'
+        )
+        self.add_slide(
+            '<img src="https://seleniumbase.io/other/cdp_con_req.jpg"'
+            ' width="100%">'
+        )
+        self.add_slide(
+            '<img src="https://seleniumbase.io/other/mycdp_con_req.jpg"'
+            ' width="100%">'
+        )
+        self.add_slide(
+            '<img src="https://seleniumbase.io/other/sb_con_req.jpg"'
+            ' width="100%">'
+        )
+        self.add_slide(
+            '<img src="https://seleniumbase.io/other/xhr_info.jpg"'
+            ' width="100%">'
+        )
+        self.add_slide(
+            '<h3>The <code>remote-debugging-port</code></h3>'
+            '<img src="https://seleniumbase.io/other/rd_port.jpg"'
+            ' width="100%">'
+        )
+        self.add_slide(
+            "<h3>Let's get to the fun part...</h3>"
+            '<img src="https://seleniumbase.io/other/hackers_at_comp.jpg"'
+            ' width="80%">'
+        )
+        self.begin_presentation(filename="uc_presentation.html")

examples/raw_skype_mobile.py

@@ -0,0 +1,15 @@
+"""Mobile emulation test for Skype."""
+from seleniumbase import SB
+
+with SB(mobile=True, test=True) as sb:
+    sb.open("https://www.skype.com/en/get-skype/")
+    sb.assert_element('[aria-label="Microsoft"]')
+    sb.assert_text("Download Skype", "h1")
+    sb.highlight("div.appBannerContent")
+    sb.highlight("h1")
+    sb.assert_text("Skype for Mobile", "h2")
+    sb.highlight("h2")
+    sb.highlight("#get-skype-0")
+    sb.highlight_click("span[data-dropdown-icon]")
+    sb.highlight("#get-skype-0_android-download")
+    sb.highlight('[data-bi-id*="ios"]')

mkdocs_build/requirements.txt

@@ -2,7 +2,7 @@
 # Minimum Python version: 3.9 (for generating docs only)
 
 regex>=2024.11.6
-pymdown-extensions>=10.13
+pymdown-extensions>=10.14
 pipdeptree>=2.24.0
 python-dateutil>=2.8.2
 Markdown==3.7

requirements.txt

@@ -1,7 +1,7 @@
 pip>=24.3.1
 packaging>=24.2
 setuptools~=70.2;python_version<"3.10"
-setuptools>=75.6.0;python_version>="3.10"
+setuptools>=75.8.0;python_version>="3.10"
 wheel>=0.45.1
 attrs>=24.3.0
 certifi>=2024.12.14
@@ -23,7 +23,7 @@ parse>=1.20.2
 parse-type>=0.6.4
 colorama>=0.4.6
 pyyaml>=6.0.2
-pygments>=2.18.0
+pygments>=2.19.1
 pyreadline3>=3.5.3;platform_system=="Windows"
 tabcompleter>=1.4.0
 pdbp>=1.6.1
@@ -36,7 +36,8 @@ requests==2.32.3
 sniffio==1.3.1
 h11==0.14.0
 outcome==1.3.0.post0
-trio==0.27.0
+trio==0.27.0;python_version<"3.9"
+trio==0.28.0;python_version>="3.9"
 trio-websocket==0.11.1
 wsproto==1.2.0
 websocket-client==1.8.0
@@ -67,7 +68,7 @@ rich==13.9.4
 # ("pip install -r requirements.txt" also installs this, but "pip install -e ." won't.)
 
 coverage>=7.6.1;python_version<"3.9"
-coverage>=7.6.9;python_version>="3.9"
+coverage>=7.6.10;python_version>="3.9"
 pytest-cov>=5.0.0;python_version<"3.9"
 pytest-cov>=6.0.0;python_version>="3.9"
 flake8==5.0.4;python_version<"3.9"

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.33.12"
+__version__ = "4.33.13"

seleniumbase/core/browser_launcher.py

@@ -533,10 +533,26 @@ def uc_open_with_cdp_mode(driver, url=None):
     if url_protocol not in ["about", "data", "chrome"]:
         safe_url = False
 
+    headless = False
+    headed = None
+    xvfb = None
+    if hasattr(sb_config, "headless"):
+        headless = sb_config.headless
+    if hasattr(sb_config, "headed"):
+        headed = sb_config.headed
+    if hasattr(sb_config, "xvfb"):
+        xvfb = sb_config.xvfb
+
     loop = asyncio.new_event_loop()
     asyncio.set_event_loop(loop)
     driver.cdp_base = loop.run_until_complete(
-        cdp_util.start(host=cdp_host, port=cdp_port)
+        cdp_util.start(
+            host=cdp_host,
+            port=cdp_port,
+            headless=headless,
+            headed=headed,
+            xvfb=xvfb,
+        )
     )
     loop.run_until_complete(driver.cdp_base.wait(0))
 
@@ -863,13 +879,15 @@ def __install_pyautogui_if_missing():
                         xvfb_height = 768
                     sb_config._xvfb_height = xvfb_height
                 with suppress(Exception):
-                    xvfb_display = Display(
+                    _xvfb_display = Display(
                         visible=True,
                         size=(xvfb_width, xvfb_height),
                         backend="xvfb",
                         use_xauth=True,
                     )
-                    xvfb_display.start()
+                    _xvfb_display.start()
+                    sb_config._virtual_display = _xvfb_display
+                    sb_config.headless_active = True
 
 
 def install_pyautogui_if_missing(driver):

seleniumbase/fixtures/base_case.py

@@ -13795,7 +13795,8 @@ def __switch_to_newest_window_if_not_blank(self):
             if self.get_current_url() == "about:blank":
                 self.switch_to_window(current_window)
         except Exception:
-            self.switch_to_window(current_window)
+            with suppress(Exception):
+                self.switch_to_window(current_window)
 
     def __needs_minimum_wait(self):
         if (
@@ -14004,9 +14005,10 @@ def __activate_standard_virtual_display(self):
                 visible=0, size=(width, height)
             )
             self._xvfb_display.start()
-            sb_config._virtual_display = self._xvfb_display
             self.headless_active = True
-            sb_config.headless_active = True
+            if not self.undetectable:
+                sb_config._virtual_display = self._xvfb_display
+                sb_config.headless_active = True
 
     def __activate_virtual_display(self):
         if self.undetectable and not (self.headless or self.headless2):
@@ -14029,6 +14031,8 @@ def __activate_virtual_display(self):
                         "\nX11 display failed! Will use regular xvfb!"
                     )
                     self.__activate_standard_virtual_display()
+                else:
+                    self.headless_active = True
             except Exception as e:
                 if hasattr(e, "msg"):
                     print("\n" + str(e.msg))
@@ -16601,6 +16605,7 @@ def tearDown(self):
             self.__quit_all_drivers()
         # Resume tearDown() for all test runners, (Pytest / Pynose / Behave)
         if hasattr(self, "_xvfb_display") and self._xvfb_display:
+            # Stop the Xvfb virtual display launched from BaseCase
             try:
                 if hasattr(self._xvfb_display, "stop"):
                     self._xvfb_display.stop()
@@ -16610,6 +16615,20 @@ def tearDown(self):
                 pass
             except Exception:
                 pass
+        if (
+            hasattr(sb_config, "_virtual_display")
+            and sb_config._virtual_display
+            and hasattr(sb_config._virtual_display, "stop")
+        ):
+            # CDP Mode may launch a 2nd Xvfb virtual display
+            try:
+                sb_config._virtual_display.stop()
+                sb_config._virtual_display = None
+                sb_config.headless_active = False
+            except AttributeError:
+                pass
+            except Exception:
+                pass
         if self.__visual_baseline_copies:
             sb_config._visual_baseline_copies = True
             if has_exception:

seleniumbase/plugins/sb_manager.py

@@ -1256,6 +1256,19 @@ def SB(
             print(traceback.format_exc().strip())
             if test and not test_passed:
                 print("********** ERROR: The test AND the tearDown() FAILED!")
+        if (
+            hasattr(sb_config, "_virtual_display")
+            and sb_config._virtual_display
+            and hasattr(sb_config._virtual_display, "stop")
+        ):
+            try:
+                sb_config._virtual_display.stop()
+                sb_config._virtual_display = None
+                sb_config.headless_active = False
+            except AttributeError:
+                pass
+            except Exception:
+                pass
         end_time = time.time()
         run_time = end_time - start_time
         sb_config = sb_config_backup

seleniumbase/undetected/cdp_driver/cdp_util.py

@@ -84,6 +84,9 @@ def __activate_virtual_display_as_needed(
                             "\nX11 display failed! Will use regular xvfb!"
                         )
                         __activate_standard_virtual_display()
+                    else:
+                        sb_config._virtual_display = _xvfb_display
+                        sb_config.headless_active = True
                 except Exception as e:
                     if hasattr(e, "msg"):
                         print("\n" + str(e.msg))