diff --git a/.ruff.toml b/.ruff.toml index 30b1bc59a..2b7671d8a 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -35,3 +35,11 @@ ignore = [ 'D104', 'D106', ] +"betty/test_utils/coverage_fixtures/*" = [ + 'D100', + 'D101', + 'D102', + 'D103', + 'D104', + 'D106', +] diff --git a/betty/importlib.py b/betty/importlib.py index 3ca478f90..aff45e89f 100644 --- a/betty/importlib.py +++ b/betty/importlib.py @@ -12,8 +12,11 @@ def import_any(fully_qualified_type_name: str) -> Any: Import any symbol in a module by its fully qualified type name. """ try: - module_name, attrs = fully_qualified_type_name.rsplit(":", 1) - module = import_module(module_name) - return reduce(getattr, attrs.split("."), module) - except (AttributeError, ImportError, ValueError): - raise ImportError(f'Cannot import "{fully_qualified_type_name}".') from None + if ":" in fully_qualified_type_name: + module_name, attrs = fully_qualified_type_name.rsplit(":", 1) + module = import_module(module_name) + return reduce(getattr, attrs.split("."), module) + else: + return import_module(fully_qualified_type_name) + except (AttributeError, ImportError, ValueError) as error: + raise ImportError(f'Cannot import "{fully_qualified_type_name}".') from error diff --git a/betty/test_utils/coverage.py b/betty/test_utils/coverage.py new file mode 100644 index 000000000..f3dc71a19 --- /dev/null +++ b/betty/test_utils/coverage.py @@ -0,0 +1,316 @@ +""" +Utilities for asserting test coverage. +""" + +from __future__ import annotations + +import pkgutil +from abc import abstractmethod, ABC +from configparser import ConfigParser +from contextlib import suppress +from enum import Enum +from functools import lru_cache +from pathlib import Path +from typing import TypeVar, Generic, Union, TYPE_CHECKING, Iterator, TypeAlias, override + + +from betty.fs import ROOT_DIRECTORY_PATH +from betty.importlib import import_any +from betty.string import snake_case_to_upper_camel_case + +if TYPE_CHECKING: + from collections.abc import ( + Sequence, + ) + from typing import Any + + +Errors: TypeAlias = Iterator[tuple[Path, str]] + + +class MissingReason(Enum): + """ + Reasons why test coverage is missing. + """ + + ABSTRACT = "This testable is abstract" + COVERAGERC = "This testable is excluded by .coveragerc" + INTERNAL = "This testable is internal to Betty itself" + PRIVATE = "This testable is private" + SHOULD_BE_COVERED = "This testable should be covered by a test but isn't yet" + + +@lru_cache +def get_coveragerc_ignore_modules() -> Sequence[Path]: + """ + Get modules that are ignored by .coveragerc. + """ + coveragerc = ConfigParser() + coveragerc.read(ROOT_DIRECTORY_PATH / ".coveragerc") + omit = coveragerc.get("run", "omit").strip().split("\n") + modules = [] + for omit_pattern in omit: + for module_path in Path().glob(omit_pattern): + if module_path.suffix != ".py": + continue + if not module_path.is_file(): + continue + modules.append(module_path.resolve()) + return modules + + +def _name_to_path(fully_qualified_type_name: str) -> Path: + module_name = ( + fully_qualified_type_name.rsplit(":", 1)[0] + if ":" in fully_qualified_type_name + else fully_qualified_type_name + ) + return Path(import_any(module_name).__file__) + + +_ParentT = TypeVar("_ParentT", bound="_Testable[Any] | None") +_ChildT = TypeVar("_ChildT", bound="_Testable[Any]") + + +class _Testable(ABC, Generic[_ParentT]): + _parent: _ParentT + + def __init__(self, name: str, *, missing: MissingReason | None = None): + self._name = name + self._missing = self.auto_ignore + if missing: + assert not self.missing, f"{self} is already ignored ({self.missing.value})" + self._missing = missing + + @property + def missing(self) -> MissingReason | None: + return self._missing + + @property + def parent(self) -> _ParentT: + return self._parent + + @parent.setter + def parent(self, parent: _ParentT) -> None: + self._parent = parent + + @property + def auto_ignore(self) -> MissingReason | None: + if self.testable_name.startswith("_"): + return MissingReason.PRIVATE + return None + + @property + def testable_name(self) -> str: + return self._name + + @property + def testable_file_path(self) -> Path: + return _name_to_path(self.testable_name) + + @property + def testable_exists(self) -> bool: + return self._exists(self.testable_name) + + @property + @abstractmethod + def test_name(self) -> str: + pass + + @property + @abstractmethod + def test_file_path(self) -> Path: + pass + + def test_exists(self) -> bool: + return self._exists(self.test_name) + + def _exists(self, name: str) -> bool: + try: + import_any(name) + return True + except ImportError: + return False + + def validate(self) -> Errors: + if not self.testable_exists: + yield self.testable_file_path, f"{self.testable_name} does not exist" + return + + if self.missing and self.test_exists(): + yield ( + self.testable_file_path, + f"{self.testable_name} was marked lacking a test, but {self.test_name} unexpectedly exists", + ) + if not self.missing and not self.test_exists(): + yield ( + self.testable_file_path, + f"{self.testable_name} unexpectedly lacks a matching test {self.test_name}", + ) + + +class _HasChildren(_Testable[_ParentT], Generic[_ParentT, _ChildT]): + def __init__( + self, + name: str, + *, + missing: MissingReason | None = None, + children: set[_ChildT] | None = None, + ): + super().__init__(name, missing=missing) + self._children = children or set() + self._auto_children() + for child in self.children: + child.parent = self + + def _auto_children(self) -> None: + pass + + @property + def children(self) -> set[_ChildT]: + return self._children + + @override + def validate(self) -> Errors: + yield from super().validate() + for child in self.children: + yield from child.validate() + + +class Module(_Testable[_ParentT], Generic[_ParentT]): + pass + + +# @todo Riiiight, and this cannot extend Module directly, because parents and children are different... +# @todo +# @todo +class RootModule(Module[None]): + pass + + + +class ChildModule(_HasChildren[Module, Union[Module, "Class", "Function"]]): + """ + A testable module. + """ + + @override + def _auto_children(self) -> None: + # @todo Also add ignores from get_coveragerc_ignore_modules() + # @todo, No, do that in the Betty-specific concrete test! + if self.testable_file_path.name == "__init__.py": + child_testable_names = { + child.testable_name + for child in self.children + if isinstance(child, Module) + } + for module_info in pkgutil.iter_modules( + [str(self.testable_file_path.parent)] + ): + module_testable_name = f"{self.testable_name}.{module_info.name}" + if ( + module_testable_name not in child_testable_names + and _name_to_path(module_testable_name) + not in get_coveragerc_ignore_modules() + ): + self._children = { + *self._children, + Module(module_testable_name), + } + + @property + def testable_module_name(self) -> str: + """ + The testable's module name. + """ + return self.testable_name.split(".")[-1] + + @override + @property + def test_name(self) -> str: + # @todo Require a parent!!! + # @todo + # @todo + # @todo We want something like a root module so we can stop hardcoding "betty.tests." + # @todo + # @todo + if self.testable_file_path.name == "__init__.py": + return f"betty.tests.{self.testable_name[6:]}.test___init__" + else: + return f"betty.tests.{self.testable_name[6:-len(self.testable_module_name)]}test_{self.testable_module_name}" + + @override + @property + def test_file_path(self) -> Path: + raise NotImplementedError + + + +class InternalModule(Module): + """ + A module that is internal and does not need test coverage. + """ + + def __init__(self, name: str): + super().__init__(name, missing=MissingReason.INTERNAL) + + @override + def _auto_children(self) -> None: + return None + + +class Function(_Testable[Module]): + """ + A testable module function. + """ + + @override + @property + def test_name(self) -> str: + _, testable_function_name = self.testable_name.split(":") + test_module_name = f"betty.tests.{self.testable_name[6:]}" + test_class_name = ( + f"Test{snake_case_to_upper_camel_case(testable_function_name)}" + ) + return f"{test_module_name}:{test_class_name}" + + +class Method(_Testable["Class"]): + """ + A testable method. + """ + + @override + @property + def auto_ignore(self) -> MissingReason | None: + missing = super().auto_ignore + if missing is not None: + return missing + with suppress(ImportError): + if getattr(import_any(self.testable_name), "__isabstractmethod__", False): + return MissingReason.ABSTRACT + return None + + @override + @property + def test_name(self) -> str: + _, testable_attrs = self.testable_name.split(":") + test_module_name = f"betty.tests.{self.testable_name[6:]}" + testable_class_name, testable_method_name = testable_attrs.split(".") + test_class_name = f"Test{testable_class_name}" + test_method_name = f"test_{testable_method_name}" + return f"{test_module_name}:{test_class_name}.{test_method_name}" + + +class Class(_HasChildren[Module, Method]): + """ + A testable class. + """ + + @override + @property + def test_name(self) -> str: + testable_module_name, testable_class_name = self.testable_name.split(":") + test_module_name = f"betty.tests.{testable_module_name[6:]}" + test_class_name = f"Test{testable_class_name}" + return f"{test_module_name}:{test_class_name}" diff --git a/betty/test_utils/coverage_fixtures/__init__.py b/betty/test_utils/coverage_fixtures/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/betty/test_utils/coverage_fixtures/_module_private/__init__.py b/betty/test_utils/coverage_fixtures/_module_private/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/betty/test_utils/coverage_fixtures/module_class_function_with_test.py b/betty/test_utils/coverage_fixtures/module_class_function_with_test.py new file mode 100644 index 000000000..5beabb50f --- /dev/null +++ b/betty/test_utils/coverage_fixtures/module_class_function_with_test.py @@ -0,0 +1,3 @@ +class Src: + def src(self) -> None: + pass # pragma: no cover diff --git a/betty/test_utils/coverage_fixtures/module_class_function_without_test.py b/betty/test_utils/coverage_fixtures/module_class_function_without_test.py new file mode 100644 index 000000000..5beabb50f --- /dev/null +++ b/betty/test_utils/coverage_fixtures/module_class_function_without_test.py @@ -0,0 +1,3 @@ +class Src: + def src(self) -> None: + pass # pragma: no cover diff --git a/betty/test_utils/coverage_fixtures/module_class_with_test.py b/betty/test_utils/coverage_fixtures/module_class_with_test.py new file mode 100644 index 000000000..ae71df69d --- /dev/null +++ b/betty/test_utils/coverage_fixtures/module_class_with_test.py @@ -0,0 +1,2 @@ +class Src: + pass # pragma: no cover diff --git a/betty/test_utils/coverage_fixtures/module_class_without_test.py b/betty/test_utils/coverage_fixtures/module_class_without_test.py new file mode 100644 index 000000000..ae71df69d --- /dev/null +++ b/betty/test_utils/coverage_fixtures/module_class_without_test.py @@ -0,0 +1,2 @@ +class Src: + pass # pragma: no cover diff --git a/betty/test_utils/coverage_fixtures/module_function_with_test.py b/betty/test_utils/coverage_fixtures/module_function_with_test.py new file mode 100644 index 000000000..aa15a8a21 --- /dev/null +++ b/betty/test_utils/coverage_fixtures/module_function_with_test.py @@ -0,0 +1,2 @@ +def src() -> None: + pass # pragma: no cover diff --git a/betty/test_utils/coverage_fixtures/module_function_without_test.py b/betty/test_utils/coverage_fixtures/module_function_without_test.py new file mode 100644 index 000000000..aa15a8a21 --- /dev/null +++ b/betty/test_utils/coverage_fixtures/module_function_without_test.py @@ -0,0 +1,2 @@ +def src() -> None: + pass # pragma: no cover diff --git a/betty/test_utils/coverage_fixtures/module_with_docstring_only/__init__.py b/betty/test_utils/coverage_fixtures/module_with_docstring_only/__init__.py new file mode 100644 index 000000000..62a93223d --- /dev/null +++ b/betty/test_utils/coverage_fixtures/module_with_docstring_only/__init__.py @@ -0,0 +1 @@ +"""Provide a source module with a docstring only.""" diff --git a/betty/test_utils/coverage_fixtures/module_with_test/__init__.py b/betty/test_utils/coverage_fixtures/module_with_test/__init__.py new file mode 100644 index 000000000..321971ebd --- /dev/null +++ b/betty/test_utils/coverage_fixtures/module_with_test/__init__.py @@ -0,0 +1 @@ +_this_is_not_an_empty_file = True diff --git a/betty/test_utils/coverage_fixtures/module_without_test/__init__.py b/betty/test_utils/coverage_fixtures/module_without_test/__init__.py new file mode 100644 index 000000000..321971ebd --- /dev/null +++ b/betty/test_utils/coverage_fixtures/module_without_test/__init__.py @@ -0,0 +1 @@ +_this_is_not_an_empty_file = True diff --git a/betty/tests/test_coverage.py b/betty/tests/test_coverage.py new file mode 100644 index 000000000..72f47716b --- /dev/null +++ b/betty/tests/test_coverage.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from collections import defaultdict + +from betty.fs import ROOT_DIRECTORY_PATH +from betty.test_utils.coverage import ( + Module, + MissingReason, + InternalModule, + Function, + Class, +) + + +# This baseline MUST NOT be extended. It SHOULD decrease in size as more coverage is added to Betty over time. +_BASELINE = Module( + "betty", + missing=MissingReason.SHOULD_BE_COVERED, + children={ + Module( + "betty.about", + missing=MissingReason.SHOULD_BE_COVERED, + children={ + Function( + "betty.about:is_development", + missing=MissingReason.SHOULD_BE_COVERED, + ), + Function( + "betty.about:is_stable", missing=MissingReason.SHOULD_BE_COVERED + ), + Function("betty.about:report", missing=MissingReason.SHOULD_BE_COVERED), + }, + ), + Module( + "betty.assets", + missing=MissingReason.SHOULD_BE_COVERED, + children={ + Class( + "betty.assets:AssetRepository", + missing=MissingReason.SHOULD_BE_COVERED, + ), + }, + ), + InternalModule("betty.test_utils"), + InternalModule("betty.tests"), + }, +) + + +class TestCoverage: + async def test(self) -> None: + errors = defaultdict(list) + for error_file_path, error_message in _BASELINE.validate(): + errors[error_file_path].append(error_message) + if len(errors): + message = "Missing test coverage:" + total_error_count = 0 + for file_path in sorted(errors.keys()): + file_error_count = len(errors[file_path]) + total_error_count += file_error_count + if not file_error_count: + continue + message += f"\n{file_path.relative_to(ROOT_DIRECTORY_PATH)}: {file_error_count} error(s)" + for error in errors[file_path]: + message += f"\n - {error}" + message += f"\nTOTAL: {total_error_count} error(s)" + + raise AssertionError(message) diff --git a/betty/tests/test_utils/coverage_fixtures/__init__.py b/betty/tests/test_utils/coverage_fixtures/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/betty/tests/test_utils/coverage_fixtures/_module_private.py b/betty/tests/test_utils/coverage_fixtures/_module_private.py new file mode 100644 index 000000000..cc6b0a3c7 --- /dev/null +++ b/betty/tests/test_utils/coverage_fixtures/_module_private.py @@ -0,0 +1 @@ +"""Provide a private source module.""" diff --git a/betty/tests/test_utils/coverage_fixtures/module_class_function_with_test.py b/betty/tests/test_utils/coverage_fixtures/module_class_function_with_test.py new file mode 100644 index 000000000..c7d0e915b --- /dev/null +++ b/betty/tests/test_utils/coverage_fixtures/module_class_function_with_test.py @@ -0,0 +1,13 @@ +"""Provide fixtures for a source method with a matching test method.""" + + +class Src: + """Provide a fixture source class.""" + + def src(self) -> None: + pass # pragma: no cover + + +class TestSrc: + def test_src(self) -> None: + pass # pragma: no cover diff --git a/betty/tests/test_utils/coverage_fixtures/module_class_function_without_test.py b/betty/tests/test_utils/coverage_fixtures/module_class_function_without_test.py new file mode 100644 index 000000000..78bf1052d --- /dev/null +++ b/betty/tests/test_utils/coverage_fixtures/module_class_function_without_test.py @@ -0,0 +1,12 @@ +"""Provide fixtures for a source method without a matching test method.""" + + +class Src: + """Provide a fixture source method.""" + + def src(self) -> None: + pass # pragma: no cover + + +class TestSrc: + pass # pragma: no cover diff --git a/betty/tests/test_utils/coverage_fixtures/module_class_with_test.py b/betty/tests/test_utils/coverage_fixtures/module_class_with_test.py new file mode 100644 index 000000000..6c5c2fd94 --- /dev/null +++ b/betty/tests/test_utils/coverage_fixtures/module_class_with_test.py @@ -0,0 +1,11 @@ +"""Provide fixtures for a source class with a matching test class.""" + + +class Src: + """Provide a fixture source class.""" + + pass # pragma: no cover + + +class TestSrc: + pass # pragma: no cover diff --git a/betty/tests/test_utils/coverage_fixtures/module_class_without_test.py b/betty/tests/test_utils/coverage_fixtures/module_class_without_test.py new file mode 100644 index 000000000..7ebf213a8 --- /dev/null +++ b/betty/tests/test_utils/coverage_fixtures/module_class_without_test.py @@ -0,0 +1,7 @@ +"""Provide fixtures for a source class without a matching test class.""" + + +class Src: + """Provide a fixture source class.""" + + pass # pragma: no cover diff --git a/betty/tests/test_utils/coverage_fixtures/module_function_with_test.py b/betty/tests/test_utils/coverage_fixtures/module_function_with_test.py new file mode 100644 index 000000000..d0768b42b --- /dev/null +++ b/betty/tests/test_utils/coverage_fixtures/module_function_with_test.py @@ -0,0 +1,10 @@ +"""Provide fixtures for a source function with matching test class.""" + + +def src() -> None: + """Provide a fixture source function.""" + pass # pragma: no cover + + +class TestSrc: + pass # pragma: no cover diff --git a/betty/tests/test_utils/coverage_fixtures/module_function_without_test.py b/betty/tests/test_utils/coverage_fixtures/module_function_without_test.py new file mode 100644 index 000000000..0614e169d --- /dev/null +++ b/betty/tests/test_utils/coverage_fixtures/module_function_without_test.py @@ -0,0 +1,6 @@ +"""Provide fixtures for a source function without a matching test class.""" + + +def src() -> None: + """Provide a fixture source function.""" + pass # pragma: no cover diff --git a/betty/tests/test_utils/coverage_fixtures/module_with_docstring_only/__init__.py b/betty/tests/test_utils/coverage_fixtures/module_with_docstring_only/__init__.py new file mode 100644 index 000000000..62a93223d --- /dev/null +++ b/betty/tests/test_utils/coverage_fixtures/module_with_docstring_only/__init__.py @@ -0,0 +1 @@ +"""Provide a source module with a docstring only.""" diff --git a/betty/tests/test_utils/coverage_fixtures/module_with_test/__init__.py b/betty/tests/test_utils/coverage_fixtures/module_with_test/__init__.py new file mode 100644 index 000000000..2daa70cc6 --- /dev/null +++ b/betty/tests/test_utils/coverage_fixtures/module_with_test/__init__.py @@ -0,0 +1,3 @@ +"""Provide a source module with a matching test module.""" + +_this_is_not_an_empty_file = True diff --git a/betty/tests/test_utils/coverage_fixtures/module_with_test/test.py b/betty/tests/test_utils/coverage_fixtures/module_with_test/test.py new file mode 100644 index 000000000..4cc80b804 --- /dev/null +++ b/betty/tests/test_utils/coverage_fixtures/module_with_test/test.py @@ -0,0 +1 @@ +"""Provide a test module for this fixture's source module.""" diff --git a/betty/tests/test_utils/coverage_fixtures/module_without_test/__init__.py b/betty/tests/test_utils/coverage_fixtures/module_without_test/__init__.py new file mode 100644 index 000000000..411c254d8 --- /dev/null +++ b/betty/tests/test_utils/coverage_fixtures/module_without_test/__init__.py @@ -0,0 +1,3 @@ +"""Provide a source module without a matching test module.""" + +_this_is_not_an_empty_file = True diff --git a/betty/tests/test_utils/test_coverage.py b/betty/tests/test_utils/test_coverage.py new file mode 100644 index 000000000..240732ea9 --- /dev/null +++ b/betty/tests/test_utils/test_coverage.py @@ -0,0 +1,23 @@ +import pytest + +from betty.test_utils.coverage import Module, MissingReason + + +class TestModule: + @pytest.mark.parametrize( + ("errors_expected", "sut"), + [ + (False, Module("betty.tests.test_utils.coverage_fixtures._module_private")), + # (False, Module("betty.tests.test_utils.coverage_fixtures.module_with_test")), + # ( + # False, + # Module( + # "betty.tests.test_utils.coverage_fixtures.module_without_test", + # missing=MissingReason.SHOULD_BE_COVERED, + # ), + # ), + # (True, Module("betty.tests.test_utils.coverage_fixtures.module_without_test")), + ], + ) + async def test(self, errors_expected: bool, sut: Module) -> None: + assert (len(list(sut.validate())) > 0) is errors_expected