From 031f4fb98771f6e9366d823127fcefcfa8cf6782 Mon Sep 17 00:00:00 2001 From: Mats Lindh Date: Tue, 15 Mar 2022 12:51:31 +0100 Subject: [PATCH 01/10] feat: allow xpath for test regression against element + Windows support Warning: This patch does two things, which I sadly didn't have time to split out into separate PRs. 1) Allow a user to give an optional `xpath` qualifier when calling `screenshot_regression`. The regression will then be tested against this single element instead of the whole page. 2) Make pytest-image-diff with splinter work properly on Windows (see issue #1). Since Windows requires us to close the temp file first, we make the temp file persistent, close it, and then tells splinter to write to it. Afterwards we clean up the temporary file before returning from the function. --- pytest_image_diff/splinter.py | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/pytest_image_diff/splinter.py b/pytest_image_diff/splinter.py index 22b5abb..d3fdce7 100644 --- a/pytest_image_diff/splinter.py +++ b/pytest_image_diff/splinter.py @@ -1,3 +1,5 @@ +import os +import pathlib import pytest from tempfile import NamedTemporaryFile from typing import Optional, Generator @@ -19,6 +21,7 @@ def __call__( browser: Optional[Browser] = None, threshold: Optional[float] = None, suffix: Optional[str] = None, + xpath: Optional[str] = "", ) -> bool: pass @@ -36,6 +39,7 @@ def screenshot_regression( :param browser: optional, by default from `browser` fixture :param threshold: float, by default from `image_diff_threshold` :param suffix: str, need for multiple checks by one test + :param xpath: str, optional xpath expression to select an element to screenshot instead of page """ default_browser = browser @@ -43,15 +47,38 @@ def _factory( browser: Optional[Browser] = None, threshold: Optional[float] = None, suffix: Optional[str] = "", + xpath: Optional[str] = "", ) -> bool: if browser is None: browser = default_browser if threshold is None: threshold = image_diff_threshold - tf = NamedTemporaryFile(suffix=".png") - image = tf.name - browser.driver.save_screenshot(image) - return image_regression(image, threshold, suffix) + + with NamedTemporaryFile(suffix=".png", delete=False) as tf: + temp_image_path = pathlib.Path(tf.name) + + try: + screenshot_path = os.fspath(temp_image_path) + + if xpath: + # `unique_file=False` since we already have a temporary file + # + # Since an xpath screenshot composes its own file name, we need to give it the prefix and + # suffix as separate parameters. `:-4` for the path without extension, then suffix given manually. + browser.find_by_xpath(xpath).first.screenshot(screenshot_path[:-4], + suffix=screenshot_path[-4:], + unique_file=False, + full=True, + ) + else: + browser.driver.save_screenshot(screenshot_path) + + result = image_regression(screenshot_path, threshold, suffix) + finally: + temp_image_path.unlink(missing_ok=True) + + return result + yield _factory From 68fcfb8d91d17778fbfc64b2fcce3f5cd52bcba1 Mon Sep 17 00:00:00 2001 From: apkawa Date: Wed, 16 Mar 2022 13:19:13 +0300 Subject: [PATCH 02/10] add windows os --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97f083b..605a35c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,11 +9,10 @@ on: jobs: test: - runs-on: ubuntu-20.04 - name: "python ${{ matrix.python-version }} ${{ matrix.toxenv }}" strategy: fail-fast: false matrix: + os: ["ubuntu-20.04", "windows-latest"] python-version: [ 3.7, 3.8, 3.9, "3.10"] toxenv: [""] experimental: [ false ] @@ -32,6 +31,8 @@ jobs: - experimental: true python-version: "pypy-3.7" + runs-on: ${{ matrix.os }} + name: "python ${{ matrix.python-version }} ${{ matrix.toxenv }}" continue-on-error: ${{ matrix.experimental }} env: From 674c06a591ef1fb269a095413593552fd232e5a9 Mon Sep 17 00:00:00 2001 From: apkawa Date: Wed, 16 Mar 2022 13:27:48 +0300 Subject: [PATCH 03/10] try fix ci --- .github/workflows/ci.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 605a35c..69ad606 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-20.04", "windows-latest"] + os: [ubuntu-20.04] python-version: [ 3.7, 3.8, 3.9, "3.10"] toxenv: [""] experimental: [ false ] @@ -20,19 +20,31 @@ jobs: - toxenv: qa python-version: 3.7 experimental: false + os: ubuntu-20.04 - toxenv: type python-version: 3.7 experimental: false + os: ubuntu-20.04 - toxenv: splinter experimental: false python-version: 3.7 + os: ubuntu-20.04 + - toxenv: splinter + experimental: false + python-version: 3.7 + os: windows-latest + - experimental: false + python-version: "3.9" + os: windows-latest - experimental: true python-version: "3.11.0-alpha - 3.11" + os: ubuntu-20.04 - experimental: true python-version: "pypy-3.7" + os: ubuntu-20.04 runs-on: ${{ matrix.os }} - name: "python ${{ matrix.python-version }} ${{ matrix.toxenv }}" + name: "python ${{ matrix.python-version }} ${{ matrix.os }} ${{ matrix.toxenv }}" continue-on-error: ${{ matrix.experimental }} env: From 244fe3093850f805c5d99422b32d965e0c0fbb59 Mon Sep 17 00:00:00 2001 From: apkawa Date: Wed, 16 Mar 2022 14:52:25 +0300 Subject: [PATCH 04/10] Fix compatibles for windows, latest chrome; add test for xpath --- pytest_image_diff/_types.py | 3 ++- pytest_image_diff/helpers.py | 20 +++++++++++++++++-- pytest_image_diff/plugin.py | 16 +++++++-------- pytest_image_diff/splinter.py | 37 ++++++++++++++++------------------- setup.cfg | 2 +- setup.py | 4 +--- tests/test_splinter.py | 11 +++++++++++ 7 files changed, 58 insertions(+), 35 deletions(-) diff --git a/pytest_image_diff/_types.py b/pytest_image_diff/_types.py index a66105c..a5621c5 100644 --- a/pytest_image_diff/_types.py +++ b/pytest_image_diff/_types.py @@ -1,9 +1,10 @@ +from pathlib import Path from typing import BinaryIO, Union, Tuple, Optional from PIL.Image import Image from typing_extensions import Literal, Protocol -PathOrFileType = Union[str, bytes, BinaryIO] +PathOrFileType = Union[str, bytes, Path, BinaryIO] ImageFileType = Union[Image, PathOrFileType] ImageSize = Tuple[int, int] diff --git a/pytest_image_diff/helpers.py b/pytest_image_diff/helpers.py index c51f98f..432fd7c 100644 --- a/pytest_image_diff/helpers.py +++ b/pytest_image_diff/helpers.py @@ -1,6 +1,9 @@ import os +import pathlib import shutil import typing +from contextlib import contextmanager +from tempfile import NamedTemporaryFile from PIL.Image import Image from _pytest import junitxml @@ -27,6 +30,19 @@ def build_filename( ) +@contextmanager +def temp_file(suffix: typing.Optional[str] = None) -> typing.Iterator[pathlib.Path]: + with NamedTemporaryFile(suffix=suffix, delete=False) as tf: + temp_image_path = pathlib.Path(tf.name) + try: + yield temp_image_path + finally: + try: + temp_image_path.unlink() + except FileNotFoundError: + pass + + class TestInfo: test_name: str class_name: str @@ -58,13 +74,13 @@ def get_test_info( return TestInfo.get_test_info(request, suffix, prefix) -def image_save(image: ImageFileType, path: str) -> None: +def image_save(image: ImageFileType, path: typing.Union[str, pathlib.Path]) -> None: if isinstance(image, Image): image.save(path) elif isinstance(image, str): if not os.path.exists(image): raise ValueError("Image maybe path. Path not exists!") - shutil.copyfile(image, path) + shutil.copyfile(image, str(path)) elif hasattr(image, "read"): with open(path, "wb") as f: image = typing.cast(typing.BinaryIO, image) diff --git a/pytest_image_diff/plugin.py b/pytest_image_diff/plugin.py index fef88d6..6b2d5b3 100644 --- a/pytest_image_diff/plugin.py +++ b/pytest_image_diff/plugin.py @@ -1,5 +1,4 @@ import os -from tempfile import NamedTemporaryFile from typing import Optional, NamedTuple, cast, Callable, Generator import pytest @@ -8,7 +7,7 @@ from _pytest.runner import CallInfo from ._types import ImageFileType, ImageRegressionCallableType, ImageDiffCallableType -from .helpers import get_test_info, build_filename, image_save, ensure_dirs +from .helpers import get_test_info, build_filename, image_save, ensure_dirs, temp_file from .image_diff import _diff try: @@ -18,7 +17,7 @@ @pytest.hookimpl(tryfirst=True, hookwrapper=True) # type: ignore -def pytest_runtest_makereport(item: Function, call: CallInfo) -> None: +def pytest_runtest_makereport(item: Function, call: CallInfo): # type: ignore # execute all other hooks to obtain the report object outcome = yield rep = outcome.get_result() @@ -180,8 +179,6 @@ def _factory( ) -> bool: if threshold is None: threshold = image_diff_threshold - image_temp_file = NamedTemporaryFile(suffix=".jpg") - image_2_temp_file = NamedTemporaryFile(suffix=".jpg") _info = _image_diff_info(image, suffix) diff_path = cast(str, _info.diff_name) @@ -198,9 +195,12 @@ def _cleanup() -> None: request.addfinalizer(_cleanup) - image_save(image, path=image_temp_file.name) - image_save(image_2, path=image_2_temp_file.name) - diff_ratio = _diff(image_temp_file.name, image_2_temp_file.name, diff_path) + with temp_file(suffix=".jpg") as image_temp_file, temp_file( + suffix=".jpg" + ) as image_2_temp_file: + image_save(image, path=image_temp_file) + image_save(image_2, path=image_2_temp_file) + diff_ratio = _diff(image_temp_file, image_2_temp_file, diff_path) assert diff_ratio <= threshold, "Image not equals! See %s" % diff_path # noqa return True diff --git a/pytest_image_diff/splinter.py b/pytest_image_diff/splinter.py index d3fdce7..86678da 100644 --- a/pytest_image_diff/splinter.py +++ b/pytest_image_diff/splinter.py @@ -1,10 +1,10 @@ import os -import pathlib -import pytest -from tempfile import NamedTemporaryFile from typing import Optional, Generator + +import pytest from typing_extensions import Protocol +from .helpers import temp_file try: # Check pytest-splinter @@ -39,7 +39,8 @@ def screenshot_regression( :param browser: optional, by default from `browser` fixture :param threshold: float, by default from `image_diff_threshold` :param suffix: str, need for multiple checks by one test - :param xpath: str, optional xpath expression to select an element to screenshot instead of page + :param xpath: str, optional xpath expression to select an element to + screenshot instead of page """ default_browser = browser @@ -55,30 +56,26 @@ def _factory( if threshold is None: threshold = image_diff_threshold - with NamedTemporaryFile(suffix=".png", delete=False) as tf: - temp_image_path = pathlib.Path(tf.name) - - try: + with temp_file(suffix=".png") as temp_image_path: screenshot_path = os.fspath(temp_image_path) if xpath: # `unique_file=False` since we already have a temporary file # - # Since an xpath screenshot composes its own file name, we need to give it the prefix and - # suffix as separate parameters. `:-4` for the path without extension, then suffix given manually. - browser.find_by_xpath(xpath).first.screenshot(screenshot_path[:-4], - suffix=screenshot_path[-4:], - unique_file=False, - full=True, - ) + # Since an xpath screenshot composes its own file name, + # we need to give it the prefix and + # suffix as separate parameters. `:-4` for the path without extension, + # then suffix given manually. + browser.find_by_xpath(xpath).first.screenshot( + screenshot_path[:-4], + suffix=screenshot_path[-4:], + unique_file=False, + full=True, + ) else: browser.driver.save_screenshot(screenshot_path) result = image_regression(screenshot_path, threshold, suffix) - finally: - temp_image_path.unlink(missing_ok=True) - - return result - + return result yield _factory diff --git a/setup.cfg b/setup.cfg index 150dbf0..c74d206 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,7 +20,7 @@ universal = 1 [flake8] ignore = D203 -exclude = +exclude = .git/, .tox/, docs/, diff --git a/setup.py b/setup.py index 76c7162..f03b1e2 100644 --- a/setup.py +++ b/setup.py @@ -54,9 +54,7 @@ def read(fname): python_requires=">=3.6, <4", entry_points={"pytest11": ["image_diff = pytest_image_diff.plugin"]}, install_requires=["pytest", "typing_extensions", "diffimg", "imgdiff"], - extras_require={ - "splinter": ["pytest-splinter>=2.1.0", "chromedriver-binary==2.40.1"] - }, + extras_require={"splinter": ["pytest-splinter>=2.1.0", "chromedriver-binary-auto"]}, zip_safe=False, include_package_data=True, keywords=["pytest"], diff --git a/tests/test_splinter.py b/tests/test_splinter.py index 3d70a8b..4759861 100644 --- a/tests/test_splinter.py +++ b/tests/test_splinter.py @@ -50,3 +50,14 @@ def test_splinter(browser, screenshot_regression): screenshot_regression() browser.driver.set_window_size(800, 600) screenshot_regression(suffix="small_window") + + +@with_splinter +def test_splinter_with_xpath(browser, screenshot_regression): + tf = tempfile.NamedTemporaryFile(suffix=".html") + tf.write(b"

Example

") + tf.flush() + + browser.driver.set_window_size(1280, 1024) + browser.visit("file://" + tf.name) + screenshot_regression(xpath="//h1") From 75fe4e04009f2086d875c8e3905dcd5edc5aead4 Mon Sep 17 00:00:00 2001 From: apkawa Date: Thu, 17 Mar 2022 12:13:40 +0300 Subject: [PATCH 05/10] try fix splinter windows test; drop chromedriver-binary from [splinter] --- .github/workflows/ci.yml | 2 +- setup.py | 2 +- tox.ini | 8 ++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69ad606..caeda20 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,7 @@ jobs: os: ubuntu-20.04 runs-on: ${{ matrix.os }} - name: "python ${{ matrix.python-version }} ${{ matrix.os }} ${{ matrix.toxenv }}" + name: "${{ matrix.os }} python ${{ matrix.python-version }} ${{ matrix.toxenv }}" continue-on-error: ${{ matrix.experimental }} env: diff --git a/setup.py b/setup.py index f03b1e2..fec3000 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ def read(fname): python_requires=">=3.6, <4", entry_points={"pytest11": ["image_diff = pytest_image_diff.plugin"]}, install_requires=["pytest", "typing_extensions", "diffimg", "imgdiff"], - extras_require={"splinter": ["pytest-splinter>=2.1.0", "chromedriver-binary-auto"]}, + extras_require={"splinter": ["pytest-splinter>=2.1.0"]}, zip_safe=False, include_package_data=True, keywords=["pytest"], diff --git a/tox.ini b/tox.ini index 0fb03b7..a8d8293 100644 --- a/tox.ini +++ b/tox.ini @@ -5,10 +5,18 @@ envlist = py3{6,7,8,9,10,11} [testenv] +# environment will be skipped if regular expression does not match against the sys.platform string +platform = linux: linux + macos: darwin + windows: win32 + changedir = {toxinidir} deps = -r{toxinidir}/requirements-dev.txt splinter: -e .[splinter] + splinter-linux: chromedriver-binary-auto + splinter-windows: chromedriver-binary==2.40.1 + setenv = PYTHONPATH = {toxinidir} passenv = From 35070f2efbb110c169047f880ce0c1f30eea406a Mon Sep 17 00:00:00 2001 From: apkawa Date: Thu, 17 Mar 2022 12:29:39 +0300 Subject: [PATCH 06/10] fix ci --- .github/workflows/ci.yml | 4 ++-- tox.ini | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index caeda20..851652b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,11 +25,11 @@ jobs: python-version: 3.7 experimental: false os: ubuntu-20.04 - - toxenv: splinter + - toxenv: splinter-linux experimental: false python-version: 3.7 os: ubuntu-20.04 - - toxenv: splinter + - toxenv: splinter-windows experimental: false python-version: 3.7 os: windows-latest diff --git a/tox.ini b/tox.ini index a8d8293..013cdba 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ minversion = 2.3 skip_missing_interpreters = true envlist = - py3{6,7,8,9,10,11} + py3{6,7,8,9,10,11}-{linux,windows} [testenv] # environment will be skipped if regular expression does not match against the sys.platform string From fc10ba55e308de2a04cfbaa24ad4159f5f558326 Mon Sep 17 00:00:00 2001 From: apkawa Date: Thu, 17 Mar 2022 12:43:10 +0300 Subject: [PATCH 07/10] pin chrome and chromedriver versions --- .github/workflows/ci.yml | 2 ++ tox.ini | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 851652b..92f2d1e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,8 @@ jobs: steps: # chrome headless - uses: browser-actions/setup-chrome@latest + with: + chrome-version: 957103 - uses: actions/checkout@v2 - name: Set up python ${{ matrix.python-version}} uses: actions/setup-python@v2 diff --git a/tox.ini b/tox.ini index 013cdba..62c67a7 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ deps = -r{toxinidir}/requirements-dev.txt splinter: -e .[splinter] splinter-linux: chromedriver-binary-auto - splinter-windows: chromedriver-binary==2.40.1 + splinter-windows: chromedriver-binary==95.0.4638.69.0 setenv = PYTHONPATH = {toxinidir} From 3856c3fcff95b8765053c09aed180ac0703335d7 Mon Sep 17 00:00:00 2001 From: apkawa Date: Thu, 17 Mar 2022 13:00:34 +0300 Subject: [PATCH 08/10] try pin chrome --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92f2d1e..50978f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,7 @@ jobs: python-version: [ 3.7, 3.8, 3.9, "3.10"] toxenv: [""] experimental: [ false ] + chrome: ["stable"] include: - toxenv: qa python-version: 3.7 @@ -33,6 +34,7 @@ jobs: experimental: false python-version: 3.7 os: windows-latest + chrome: "959969" - experimental: false python-version: "3.9" os: windows-latest @@ -53,9 +55,9 @@ jobs: # chrome headless - uses: browser-actions/setup-chrome@latest with: - chrome-version: 957103 + chrome-version: ${{ matrix.chrome }} - uses: actions/checkout@v2 - - name: Set up python ${{ matrix.python-version}} + - name: Set up python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} From 5e6848cf90d37c5cf667a87792899c9f9e5de598 Mon Sep 17 00:00:00 2001 From: apkawa Date: Thu, 17 Mar 2022 13:03:56 +0300 Subject: [PATCH 09/10] fix chrome win version number --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50978f1..1ed4b83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: experimental: false python-version: 3.7 os: windows-latest - chrome: "959969" + chrome: "959905" - experimental: false python-version: "3.9" os: windows-latest From 7f6e37b163671cbb864280e57b4cccd164327aa7 Mon Sep 17 00:00:00 2001 From: apkawa Date: Thu, 17 Mar 2022 13:11:28 +0300 Subject: [PATCH 10/10] disable support selenium-windows, bruh --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ed4b83..f2148d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,11 +30,14 @@ jobs: experimental: false python-version: 3.7 os: ubuntu-20.04 + - toxenv: splinter-windows - experimental: false + # Do not support selenium-windows + experimental: true python-version: 3.7 os: windows-latest chrome: "959905" + - experimental: false python-version: "3.9" os: windows-latest