Implement throw_exception fixture, up readme and docs #4 (#5)

* Implement no throw_exception fixture, up readme and docs #4
* Fix windows compatibly tests

Author: Apkawa

.github/workflows/ci.yml

@@ -3,20 +3,17 @@ name: ci
 on:
   push:
   pull_request:
-  schedule:
-    # Weekly
-    - cron: "0 0 * * 0"
 
 jobs:
   test:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04]
-        python-version: [ 3.7, 3.8, 3.9, "3.10"]
-        toxenv: [""]
+        os: [ ubuntu-20.04 ]
+        python-version: [ 3.7, 3.8, 3.9, "3.10" ]
+        toxenv: [ "" ]
         experimental: [ false ]
-        chrome: ["stable"]
+        chrome: [ "stable" ]
         include:
           - toxenv: qa
             python-version: 3.7
@@ -36,7 +33,7 @@ jobs:
             experimental: true
             python-version: 3.7
             os: windows-latest
-            chrome: "959905"
+#            chrome: "959905"
 
           - experimental: false
             python-version: "3.9"

README.md

@@ -40,6 +40,45 @@ def test_regression(image_regression):
     image_regression(image, threshold=0.5)
 ```
 
+Also use with assert
+
+```python
+import pytest
+
+from typing import Union
+from PIL import Image
+
+@pytest.fixture(scope="session")
+def image_diff_throw_exception() -> bool:
+    """
+    Set default throw exception. By default - True
+    """
+    return False
+
+def test_compare(image_diff):
+    image: Image or str or bytes = Image.new()
+    image2: Image or str or bytes = '/path/to/image.jpeg'
+    assert image_diff(image, image2)
+    assert image_diff(image, image2, threshold=0.5)
+    # Also can check threshold in compare, ie
+    assert image_diff(image, image2) < 0.5
+    # For different checks in one test
+    assert image_diff(image, image2, threshold=0.5, suffix="one")
+    # Or without fixture image_diff_throw_exception
+    assert image_diff(image, image2, threshold=0.5, throw_exception=False)
+
+
+def test_regression(image_regression):
+    image: Union[Image, str, bytes] = Image.new()
+    assert image_regression(image, threshold=0.5)
+    # Also can check threshold in compare, ie
+    assert image_regression(image) < 0.5
+    # For different checks in one test
+    assert image_regression(image, threshold=0.5, suffix="foo")
+    # Or without fixture image_diff_throw_exception
+    assert image_regression(image, threshold=0.5, throw_exception=False)
+```
+
 First run creates reference images
 
 ## pytest-splinter

docs/source/code.rst

@@ -10,6 +10,8 @@ Config fixtures
 .. autofunction:: pytest_image_diff.plugin.image_diff_dir
 .. autofunction:: pytest_image_diff.plugin.image_diff_reference_dir
 
+.. autofunction:: pytest_image_diff.plugin.image_diff_throw_exception
+
 
 Fixtures
 --------

pytest_image_diff/__init__.py

@@ -1,3 +1,6 @@
 # -*- coding: utf-8 -*-
+import pytest
+
+pytest.register_assert_rewrite("pytest_image_diff.helpers")
 
 __version__ = "0.0.8"

pytest_image_diff/_types.py

@@ -1,8 +1,11 @@
 from pathlib import Path
-from typing import BinaryIO, Union, Tuple, Optional
+from typing import BinaryIO, Union, Tuple, Optional, TYPE_CHECKING
+from typing_extensions import Literal, Protocol
 
 from PIL.Image import Image
-from typing_extensions import Literal, Protocol
+
+if TYPE_CHECKING:
+    from .plugin import DiffCompareResult
 
 PathOrFileType = Union[str, bytes, Path, BinaryIO]
 ImageFileType = Union[Image, PathOrFileType]
@@ -15,7 +18,8 @@ def __call__(
         image: ImageFileType,
         threshold: Optional[float] = None,
         suffix: Optional[str] = None,
-    ) -> bool:
+        throw_exception: Optional[bool] = None,
+    ) -> "DiffCompareResult":
         pass
 
 
@@ -26,7 +30,8 @@ def __call__(
         image_2: ImageFileType,
         threshold: Optional[float] = None,
         suffix: Optional[str] = None,
-    ) -> bool:
+        throw_exception: Optional[bool] = None,
+    ) -> "DiffCompareResult":
         pass
 
 

pytest_image_diff/helpers.py

@@ -38,9 +38,13 @@ def temp_file(suffix: typing.Optional[str] = None) -> typing.Iterator[pathlib.Pa
         yield temp_image_path
     finally:
         try:
+            tf.close()
             temp_image_path.unlink()
         except FileNotFoundError:
             pass
+        except PermissionError:
+            # Opps, windows issue
+            pass
 
 
 class TestInfo:
@@ -96,3 +100,8 @@ def ensure_dirs(filepath: str) -> None:
         file_dir = os.path.dirname(filepath)
         if not os.path.exists(file_dir):
             os.makedirs(file_dir)
+
+
+# def pytest_assertrepr_compare(config, op, left, right):
+#     print(op, left, right)
+#     pass

pytest_image_diff/plugin.py

@@ -1,5 +1,5 @@
 import os
-from typing import Optional, NamedTuple, cast, Callable, Generator
+from typing import Optional, NamedTuple, cast, Callable, Generator, Union
 
 import pytest
 from _pytest.fixtures import FixtureRequest
@@ -36,12 +36,20 @@ def image_diff_threshold() -> float:
     return 0.001
 
 
+@pytest.fixture(scope="session")
+def image_diff_throw_exception() -> bool:
+    """
+    Set default throw exception. By default - True
+    """
+    return True
+
+
 @pytest.fixture(scope="session")
 def image_diff_root(request: FixtureRequest) -> str:
     """
     Root path for storing diff images. By default - `request.config.rootdir`
     """
-    return str(request.config.rootdir)
+    return str(request.config.rootdir)  # type: ignore
 
 
 @pytest.fixture(scope="session")  # pragma: no cover
@@ -69,6 +77,37 @@ class DiffInfo(NamedTuple):
 DiffInfoCallableType = Callable[[ImageFileType, Optional[str]], DiffInfo]
 
 
+class DiffCompareResult:
+    value: float
+    thresh: float
+    diff_path: Optional[str]
+
+    CompareType = Union[int, float, "DiffCompareResult"]
+
+    def __init__(self, value: float, thresh: float, diff_path: Optional[str] = None):
+        self.value = value
+        self.thresh = thresh
+        self.diff_path = diff_path
+
+    def __repr__(self) -> str:
+        return f"diff: {round(self.value, 4)}; thresh: {self.thresh}; diff path: {self.diff_path}"
+
+    def __bool__(self) -> bool:
+        return self.value <= self.thresh
+
+    def __gt__(self, other: CompareType) -> bool:
+        return self.value > other
+
+    def __ge__(self, other: CompareType) -> bool:
+        return self.value >= other
+
+    def __lt__(self, other: CompareType) -> bool:
+        return self.value < other
+
+    def __le__(self, other: CompareType) -> bool:
+        return self.value <= other
+
+
 @pytest.fixture(scope="function")
 def _image_diff_info(
     request: FixtureRequest, image_diff_reference_dir: str, image_diff_dir: str
@@ -108,6 +147,7 @@ def image_regression(
     request: FixtureRequest,
     _image_diff_info: DiffInfoCallableType,
     image_diff_threshold: float,
+    image_diff_throw_exception: bool,
 ) -> Generator[ImageRegressionCallableType, None, None]:
     """
     Check regression image.
@@ -122,35 +162,43 @@ def _factory(
         image: ImageFileType,
         threshold: Optional[float] = None,
         suffix: Optional[str] = None,
-    ) -> bool:
+        throw_exception: Optional[bool] = None,
+    ) -> "DiffCompareResult":
         if threshold is None:
             threshold = image_diff_threshold
+        if throw_exception is None:
+            throw_exception = image_diff_throw_exception
         diff_info = _image_diff_info(image, suffix)
         reference_name = diff_info.reference_name
         diff_name = diff_info.diff_name
         image_name = diff_info.image_name
         if not os.path.exists(reference_name):
             ensure_dirs(reference_name)
             image_save(image, reference_name)
-            return True
+            return DiffCompareResult(0, threshold)
 
         def _cleanup() -> None:
             if request.node.rep_setup.passed and request.node.rep_call.failed:
                 # Do not cleanup regression artifacts
                 return
             for f in [image_name, diff_name]:
                 if os.path.exists(f):
-                    os.unlink(f)
+                    try:
+                        os.unlink(f)
+                    except PermissionError:
+                        # Opps, have windows issue
+                        pass
 
         request.addfinalizer(_cleanup)
 
         ensure_dirs(diff_name)
         ensure_dirs(image_name)
         image_save(image, image_name)
         diff_ratio = _diff(reference_name, image_name, diff_name)
-        assert diff_ratio <= threshold, "Image not equals!"  # noqa
-
-        return True
+        cond = DiffCompareResult(diff_ratio, threshold, diff_name)
+        if throw_exception:
+            assert cond, "Image not equals!"  # noqa
+        return cond
 
     yield _factory
 
@@ -160,6 +208,7 @@ def image_diff(
     request: FixtureRequest,
     _image_diff_info: DiffInfoCallableType,
     image_diff_threshold: float,
+    image_diff_throw_exception: bool,
 ) -> Generator[ImageDiffCallableType, None, None]:
     """
     Compare two image
@@ -176,9 +225,12 @@ def _factory(
         image_2: ImageFileType,
         threshold: Optional[float] = None,
         suffix: Optional[str] = None,
-    ) -> bool:
+        throw_exception: Optional[bool] = None,
+    ) -> DiffCompareResult:
         if threshold is None:
             threshold = image_diff_threshold
+        if throw_exception is None:
+            throw_exception = image_diff_throw_exception
 
         _info = _image_diff_info(image, suffix)
         diff_path = cast(str, _info.diff_name)
@@ -201,7 +253,9 @@ def _cleanup() -> None:
             image_save(image, path=image_temp_file)
             image_save(image_2, path=image_2_temp_file)
             diff_ratio = _diff(image_temp_file, image_2_temp_file, diff_path)
-        assert diff_ratio <= threshold, "Image not equals! See %s" % diff_path  # noqa
-        return True
+        cond = DiffCompareResult(diff_ratio, threshold, diff_path)
+        if throw_exception:
+            assert cond, "Image not equals! See %s" % diff_path  # noqa
+        return cond
 
     yield _factory

pytest_image_diff/splinter.py

@@ -1,11 +1,14 @@
 import os
-from typing import Optional, Generator
+from typing import Optional, Generator, TYPE_CHECKING
 
 import pytest
 from typing_extensions import Protocol
 
 from .helpers import temp_file
 
+if TYPE_CHECKING:
+    from .plugin import DiffCompareResult
+
 try:
     # Check pytest-splinter
     from pytest_splinter.plugin import Browser
@@ -15,23 +18,24 @@
 if Browser:
     from ._types import ImageRegressionCallableType
 
+    __all__ = ["screenshot_regression"]
+
     class ScreenshotRegressionCallableType(Protocol):
         def __call__(
             self,
             browser: Optional[Browser] = None,
             threshold: Optional[float] = None,
             suffix: Optional[str] = None,
             xpath: Optional[str] = "",
-        ) -> bool:
+        ) -> "DiffCompareResult":
             pass
 
-    __all__ = ["screenshot_regression"]
-
     @pytest.fixture(scope="function")
     def screenshot_regression(
         browser: Browser,
         image_regression: ImageRegressionCallableType,
         image_diff_threshold: float,
+        image_diff_throw_exception: bool,
     ) -> Generator[ScreenshotRegressionCallableType, None, None]:
         """
         Check regression browser screenshot
@@ -49,13 +53,17 @@ def _factory(
             threshold: Optional[float] = None,
             suffix: Optional[str] = "",
             xpath: Optional[str] = "",
-        ) -> bool:
+            throw_exception: Optional[bool] = None,
+        ) -> "DiffCompareResult":
             if browser is None:
                 browser = default_browser
 
             if threshold is None:
                 threshold = image_diff_threshold
 
+            if throw_exception is None:
+                throw_exception = image_diff_throw_exception
+
             with temp_file(suffix=".png") as temp_image_path:
                 screenshot_path = os.fspath(temp_image_path)
 
@@ -75,7 +83,9 @@ def _factory(
                 else:
                     browser.driver.save_screenshot(screenshot_path)
 
-                result = image_regression(screenshot_path, threshold, suffix)
+                result = image_regression(
+                    screenshot_path, threshold, suffix, throw_exception=throw_exception
+                )
                 return result
 
         yield _factory

requirements-dev.txt

@@ -5,8 +5,8 @@ tox
 twine
 
 # Tests
-pytest>=6,<7
-pytest-cov==2.12.1
+pytest
+pytest-cov
 
 # Docs