`` element with the ``blockquote`` class."""
-
- def __call__(self, element, converter):
- blockquote = converter.copy_element(element, tag="div", add_classes="blockquote", copy_attrs=False)
- element.addnext(blockquote)
- element.getparent().remove(element)
- return blockquote
-
-
-# TODO abt: merge MakeCard and ConvertCard into one operation class
-class BS3to4MakeCard(ElementOperation):
- """
- Pre-processe a BS3 panel, thumbnail, or well element to be converted to a BS4 card.
-
- Card components conversion is then handled by the ``ConvertCard`` operation class.
- """
-
- def __call__(self, element, converter):
- card = converter.element_factory("
")
- card_body = converter.copy_element(
- element, tag="div", add_classes="card-body", remove_classes=ALL, copy_attrs=False
- )
- card.append(card_body)
- element.addnext(card)
- element.getparent().remove(element)
- return card
-
-
-# TODO abt: refactor code
-class BS3to4ConvertCard(ElementOperation):
- """Fully convert a BS3 panel, thumbnail, or well element and their contents to a BS4 card."""
-
- POST_CONVERSIONS = {
- "title": ["card-title"],
- "description": ["card-description"],
- "category": ["card-category"],
- "panel-danger": ["card", "bg-danger", "text-white"],
- "panel-warning": ["card", "bg-warning"],
- "panel-info": ["card", "bg-info", "text-white"],
- "panel-success": ["card", "bg-success", "text-white"],
- "panel-primary": ["card", "bg-primary", "text-white"],
- "panel-footer": ["card-footer"],
- "panel-body": ["card-body"],
- "panel-title": ["card-title"],
- "panel-heading": ["card-header"],
- "panel-default": [],
- "panel": ["card"],
- }
-
- def _convert_child(self, child, old_card, new_card, converter):
- old_card_classes = get_classes(old_card)
-
- classes = get_classes(child)
-
- if "header" in classes or ("image" in classes and len(child)):
- add_classes = "card-header"
- remove_classes = ["header", "image"]
- elif "content" in classes:
- add_classes = "card-img-overlay" if "card-background" in old_card_classes else "card-body"
- remove_classes = "content"
- elif {"card-footer", "footer", "text-center"} & set(classes):
- add_classes = "card-footer"
- remove_classes = "footer"
- else:
- new_card.append(child)
- return
-
- new_child = converter.copy_element(
- child, "div", add_classes=add_classes, remove_classes=remove_classes, copy_attrs=True
- )
-
- if "image" in classes:
- [img_el] = new_child.xpath("./img")[:1] or [None]
- if img_el is not None and "src" in img_el:
- new_child.attrib["style"] = (
- f'background-image: url("http://github.com/%7Bimg_el.attrib[%22src%22]%7D"); '
- "background-position: center center; "
- "background-size: cover;"
- )
- new_child.remove(img_el)
-
- new_card.append(new_child)
-
- if "content" in classes: # TODO abt: consider skipping for .card-background
- [footer] = new_child.xpath(converter.adapt_xpath("./*[hasclass('footer')]"))[:1] or [None]
- if footer is not None:
- self._convert_child(footer, old_card, new_card, converter)
- new_child.remove(footer)
-
- def _postprocess(self, new_card, converter):
- for old_class, new_classes in self.POST_CONVERSIONS.items():
- for element in new_card.xpath(converter.adapt_xpath(f"(.|.//*)[hasclass('{old_class}')]")):
- edit_element_classes(element, add=new_classes, remove=old_class)
-
- def __call__(self, element, converter):
- classes = get_classes(element)
- new_card = converter.copy_element(element, tag="div", copy_attrs=True, copy_contents=False)
- wrapper = new_card
- if "card-horizontal" in classes:
- wrapper = etree.SubElement(new_card, "div", {"class": "row"})
-
- for child in element:
- self._convert_child(child, element, wrapper, converter)
-
- self._postprocess(new_card, converter)
- element.addnext(new_card)
- element.getparent().remove(element)
- return new_card
-
-
-class BS4to5ConvertCloseButton(ElementOperation):
- """
- Convert BS4 ``button.close`` elements to BS5 ``button.btn-close``.
-
- Also fixes the ``data-dismiss`` attribute to ``data-bs-dismiss``, and removes any inner contents.
- """
-
- def __call__(self, element, converter):
- new_btn = converter.copy_element(element, remove_classes="close", add_classes="btn-close", copy_contents=False)
-
- if "data-dismiss" in element.attrib:
- new_btn.attrib["data-bs-dismiss"] = element.attrib["data-dismiss"]
- del new_btn.attrib["data-dismiss"]
-
- element.addnext(new_btn)
- element.getparent().remove(element)
-
- return new_btn
-
-
-class BS4to5ConvertCardDeck(ElementOperation):
- """Convert BS4 ``.card-deck`` elements to grid components (``.row``, ``.col``, etc.)."""
-
- def __call__(self, element, converter):
- cards = element.xpath(converter.adapt_xpath("./*[hasclass('card')]"))
-
- cols_class = f"row-cols-{len(cards)}" if len(cards) in range(1, 7) else "row-cols-auto"
- edit_element_classes(element, add=["row", cols_class], remove="card-deck", is_qweb=converter.is_qweb)
-
- for card in cards:
- new_col = converter.build_element("div", classes=["col"])
- card.addprevious(new_col)
- new_col.append(card)
-
- return element
-
-
-class BS4to5ConvertFormInline(ElementOperation):
- """Convert BS4 ``.form-inline`` elements to grid components (``.row``, ``.col``, etc.)."""
-
- def __call__(self, element, converter):
- edit_element_classes(element, add="row row-cols-lg-auto", remove="form-inline", is_qweb=converter.is_qweb)
-
- children_selector = converter.adapt_xpath(
- CSS(".form-control,.form-group,.form-check,.input-group,.custom-select,button", prefix="./")
- )
- indexed_children = sorted([(element.index(c), c) for c in element.xpath(children_selector)], key=lambda x: x[0])
-
- nest_groups = []
- last_idx = -1
- for idx, child in indexed_children:
- nest_start, nest_end = idx, idx
- labels = [label for label in child.xpath("preceding-sibling::label") if element.index(label) > last_idx]
- labels = [
- label
- for label in labels
- if "for" in label.attrib and child.xpath(f"descendant-or-self::*[@id='{label.attrib['for']}']")
- ] or labels[-1:]
- if labels:
- first_label = labels[0]
- assert last_idx < element.index(first_label) < idx, "label must be between last group and current"
- nest_start = element.index(first_label)
-
- assert nest_start <= nest_end, f"expected start {nest_start} to be <= end {nest_end}"
- nest_groups.append(element[nest_start : nest_end + 1])
- last_idx = nest_end
-
- for els in nest_groups:
- wrapper = converter.build_element("div", classes=["col-12"])
- els[0].addprevious(wrapper)
- for el in els:
- wrapper.append(el)
- assert el not in element, f"expected {el!r} to be removed from {element!r}"
-
- return element
-
-
-class BootstrapConverter:
- """
- Class for converting XML or HTML Bootstrap code across versions.
-
- :param etree.ElementTree tree: the parsed XML or HTML tree to convert.
- :param bool is_html: whether the tree is an HTML document.
- """
-
- MIN_VERSION = "3.0"
- """Minimum supported Bootstrap version."""
-
- # Conversions definitions by destination version.
- # It's a dictionary of version strings to a list of (xpath, operations_list) tuples.
- # For operations that implement the `xpath()` method, the `op()` class method can be used
- # to directly define the operation and return the tuple with the corresponding XPath expression
- # and the operation list.
- # The `convert()` method will then process the conversions list in order, and for each tuple
- # match the elements in the tree using the XPath expression and apply the operations list to them.
- CONVERSIONS = {
- "4.0": [
- # inputs
- (CSS(".form-group .control-label"), [ReplaceClasses("control-label", "form-control-label")]),
- (CSS(".form-group .text-help"), [ReplaceClasses("text-help", "form-control-feedback")]),
- (CSS(".control-group .help-block"), [ReplaceClasses("help-block", "form-text")]),
- ReplaceClasses.op("form-group-sm", "form-control-sm"),
- ReplaceClasses.op("form-group-lg", "form-control-lg"),
- (CSS(".form-control .input-sm"), [ReplaceClasses("input-sm", "form-control-sm")]),
- (CSS(".form-control .input-lg"), [ReplaceClasses("input-lg", "form-control-lg")]),
- # hide
- ReplaceClasses.op("hidden-xs", "d-none"),
- ReplaceClasses.op("hidden-sm", "d-sm-none"),
- ReplaceClasses.op("hidden-md", "d-md-none"),
- ReplaceClasses.op("hidden-lg", "d-lg-none"),
- ReplaceClasses.op("visible-xs", "d-block d-sm-none"),
- ReplaceClasses.op("visible-sm", "d-block d-md-none"),
- ReplaceClasses.op("visible-md", "d-block d-lg-none"),
- ReplaceClasses.op("visible-lg", "d-block d-xl-none"),
- # image
- ReplaceClasses.op("img-rounded", "rounded"),
- ReplaceClasses.op("img-circle", "rounded-circle"),
- ReplaceClasses.op("img-responsive", ("d-block", "img-fluid")),
- # buttons
- ReplaceClasses.op("btn-default", "btn-secondary"),
- ReplaceClasses.op("btn-xs", "btn-sm"),
- (CSS(".btn-group.btn-group-xs"), [ReplaceClasses("btn-group-xs", "btn-group-sm")]),
- (CSS(".dropdown .divider"), [ReplaceClasses("divider", "dropdown-divider")]),
- ReplaceClasses.op("badge", "badge badge-pill"),
- ReplaceClasses.op("label", "badge"),
- RegexReplaceClass.op(rf"{BS}label-(default|primary|success|info|warning|danger){BE}", r"badge-\1"),
- (CSS(".breadcrumb > li"), [ReplaceClasses("breadcrumb", "breadcrumb-item")]),
- # li
- (CSS(".list-inline > li"), [AddClasses("list-inline-item")]),
- # pagination
- (CSS(".pagination > li"), [AddClasses("page-item")]),
- (CSS(".pagination > li > a"), [AddClasses("page-link")]),
- # carousel
- (CSS(".carousel .carousel-inner > .item"), [ReplaceClasses("item", "carousel-item")]),
- # pull
- ReplaceClasses.op("pull-right", "float-right"),
- ReplaceClasses.op("pull-left", "float-left"),
- ReplaceClasses.op("center-block", "mx-auto"),
- # well
- (CSS(".well"), [BS3to4MakeCard()]),
- (CSS(".thumbnail"), [BS3to4MakeCard()]),
- # blockquote
- (CSS("blockquote"), [BS3to4ConvertBlockquote()]),
- (CSS(".blockquote.blockquote-reverse"), [ReplaceClasses("blockquote-reverse", "text-right")]),
- # dropdown
- (CSS(".dropdown-menu > li > a"), [AddClasses("dropdown-item")]),
- (CSS(".dropdown-menu > li"), [PullUp()]),
- # in
- ReplaceClasses.op("in", "show"),
- # table
- (CSS("tr.active, td.active"), [ReplaceClasses("active", "table-active")]),
- (CSS("tr.success, td.success"), [ReplaceClasses("success", "table-success")]),
- (CSS("tr.info, td.info"), [ReplaceClasses("info", "table-info")]),
- (CSS("tr.warning, td.warning"), [ReplaceClasses("warning", "table-warning")]),
- (CSS("tr.danger, td.danger"), [ReplaceClasses("danger", "table-danger")]),
- (CSS("table.table-condesed"), [ReplaceClasses("table-condesed", "table-sm")]),
- # navbar
- (CSS(".nav.navbar > li > a"), [AddClasses("nav-link")]),
- (CSS(".nav.navbar > li"), [AddClasses("nav-intem")]),
- ReplaceClasses.op("navbar-btn", "nav-item"),
- (CSS(".navbar-nav"), [ReplaceClasses("navbar-right nav", "ml-auto")]),
- ReplaceClasses.op("navbar-toggler-right", "ml-auto"),
- (CSS(".navbar-nav > li > a"), [AddClasses("nav-link")]),
- (CSS(".navbar-nav > li"), [AddClasses("nav-item")]),
- (CSS(".navbar-nav > a"), [AddClasses("navbar-brand")]),
- ReplaceClasses.op("navbar-fixed-top", "fixed-top"),
- ReplaceClasses.op("navbar-toggle", "navbar-toggler"),
- ReplaceClasses.op("nav-stacked", "flex-column"),
- (CSS("nav.navbar"), [AddClasses("navbar-expand-lg")]),
- (CSS("button.navbar-toggle"), [ReplaceClasses("navbar-toggle", "navbar-expand-md")]),
- # card
- (CSS(".panel"), [BS3to4ConvertCard()]),
- (CSS(".card"), [BS3to4ConvertCard()]),
- # grid
- RegexReplaceClass.op(rf"{BS}col((?:-\w{{2}})?)-offset-(\d{{1,2}}){BE}", r"offset\1-\2"),
- ],
- "5.0": [
- # links
- RegexReplaceClass.op(rf"{BS}text-(?!o-)", "link-", xpath=CSS("a")),
- (CSS(".nav-item.active > .nav-link"), [AddClasses("active")]),
- (CSS(".nav-link.active") + CSS(".nav-item.active", prefix="/parent::"), [RemoveClasses("active")]),
- # badges
- ReplaceClasses.op("badge-pill", "rounded-pill"),
- RegexReplaceClass.op(rf"{BS}badge-", r"text-bg-"),
- # buttons
- ("//*[hasclass('btn-block')]/parent::div", [AddClasses("d-grid gap-2")]),
- ("//*[hasclass('btn-block')]/parent::p[count(./*)=1]", [AddClasses("d-grid gap-2")]),
- RemoveClasses.op("btn-block"),
- (CSS("button.close"), [BS4to5ConvertCloseButton()]),
- # card
- # TODO abt: .card-columns (unused in odoo)
- (CSS(".card-deck"), [BS4to5ConvertCardDeck()]),
- # jumbotron
- ReplaceClasses.op("jumbotron", "container-fluid py-5"),
- # new data-bs- attributes
- RenameAttribute.op("data-display", "data-bs-display", "[not(@data-snippet='s_countdown')]"),
- *[
- RenameAttribute.op(f"data-{attr}", f"data-bs-{attr}")
- for attr in [
- "animation",
- "attributes",
- "autohide",
- "backdrop",
- "body",
- "container",
- "content",
- "delay",
- "dismiss",
- "focus",
- "interval",
- "margin-right",
- "no-jquery",
- "offset",
- "original-title",
- "padding-right",
- "parent",
- "placement",
- "ride",
- "sanitize",
- "show",
- "slide",
- "slide-to",
- "spy",
- "target",
- "toggle",
- "touch",
- "trigger",
- "whatever",
- ]
- ],
- # popover
- (CSS(".popover .arrow"), [ReplaceClasses("arrow", "popover-arrow")]),
- # form
- ReplaceClasses.op("form-row", "row"),
- ("//*[hasclass('form-group')]/parent::form", [AddClasses("row")]),
- ReplaceClasses.op("form-group", "col-12 py-2"),
- (CSS(".form-inline"), [BS4to5ConvertFormInline()]),
- ReplaceClasses.op("custom-checkbox", "form-check"),
- RegexReplaceClass.op(rf"{BS}custom-control-(input|label){BE}", r"form-check-\1"),
- RegexReplaceClass.op(rf"{BS}custom-control-(input|label){BE}", r"form-check-\1"),
- RegexReplaceClass.op(rf"{BS}custom-(check|select|range){BE}", r"form-\1"),
- (CSS(".custom-switch"), [ReplaceClasses("custom-switch", "form-check form-switch")]),
- ReplaceClasses.op("custom-radio", "form-check"),
- RemoveClasses.op("custom-control"),
- (CSS(".custom-file"), [PullUp()]),
- RegexReplaceClass.op(rf"{BS}custom-file-", r"form-file-"),
- RegexReplaceClass.op(rf"{BS}form-file(?:-input)?{BE}", r"form-control"),
- (CSS("label.form-file-label"), [RemoveElement()]),
- (regex_xpath(rf"{BS}input-group-(prepend|append){BE}", "class"), [PullUp()]),
- ("//label[not(hasclass('form-check-label'))]", [AddClasses("form-label")]),
- ReplaceClasses.op("form-control-file", "form-control"),
- ReplaceClasses.op("form-control-range", "form-range"),
- # TODO abt: .form-text no longer sets display, add some class?
- # table
- RegexReplaceClass.op(rf"{BS}thead-(light|dark){BE}", r"table-\1"),
- # grid
- RegexReplaceClass.op(rf"{BS}col-((?:\w{{2}}-)?)offset-(\d{{1,2}}){BE}", r"offset-\1\2"), # from BS4
- # gutters
- ReplaceClasses.op("no-gutters", "g-0"),
- # logical properties
- RegexReplaceClass.op(rf"{BS}left-((?:\w{{2,3}}-)?[0-9]+|auto){BE}", r"start-\1"),
- RegexReplaceClass.op(rf"{BS}right-((?:\w{{2,3}}-)?[0-9]+|auto){BE}", r"end-\1"),
- RegexReplaceClass.op(rf"{BS}((?:float|border|rounded|text)(?:-\w+)?)-left{BE}", r"\1-start"),
- RegexReplaceClass.op(rf"{BS}((?:float|border|rounded|text)(?:-\w+)?)-right{BE}", r"\1-end"),
- RegexReplaceClass.op(rf"{BS}rounded-sm(-(?:start|end|top|bottom))?", r"rounded\1-1"),
- RegexReplaceClass.op(rf"{BS}rounded-lg(-(?:start|end|top|bottom))?", r"rounded\1-3"),
- RegexReplaceClass.op(rf"{BS}([mp])l-((?:\w{{2,3}}-)?(?:[0-9]+|auto)){BE}", r"\1s-\2"),
- RegexReplaceClass.op(rf"{BS}([mp])r-((?:\w{{2,3}}-)?(?:[0-9]+|auto)){BE}", r"\1e-\2"),
- ReplaceClasses.op("dropdown-menu-left", "dropdown-menu-start"),
- ReplaceClasses.op("dropdown-menu-right", "dropdown-menu-end"),
- ReplaceClasses.op("dropleft", "dropstart"),
- ReplaceClasses.op("dropright", "dropend"),
- # tooltips
- (
- "//*[hasclass('tooltip') or @role='tooltip']//*[hasclass('arrow')]",
- [ReplaceClasses("arrow", "tooltip-arrow")],
- ),
- # utilities
- ReplaceClasses.op("text-monospace", "font-monospace"),
- RegexReplaceClass.op(rf"{BS}font-weight-", r"fw-"),
- RegexReplaceClass.op(rf"{BS}font-style-", r"fst-"),
- ReplaceClasses.op("font-italic", "fst-italic"),
- # helpers
- RegexReplaceClass.op(rf"{BS}embed-responsive-(\d+)by(\d+)", r"ratio-\1x\2"),
- RegexReplaceClass.op(rf"{BS}ratio-(\d+)by(\d+)", r"ratio-\1x\2"),
- RegexReplaceClass.op(rf"{BS}embed-responsive(?!-)", r"ratio"),
- RegexReplaceClass.op(rf"{BS}sr-only(-focusable)?", r"visually-hidden\1"),
- # media
- ReplaceClasses.op("media-body", "flex-grow-1"),
- ReplaceClasses.op("media", "d-flex"),
- ],
- }
-
- def __init__(self, tree, is_html=False, is_qweb=False):
- self.tree = tree
- self.is_html = is_html
- self.is_qweb = is_qweb
-
- @classmethod
- def _get_sorted_conversions(cls):
- """Return the conversions dict sorted by version, from oldest to newest."""
- return sorted(cls.CONVERSIONS.items(), key=lambda kv: Version(kv[0]))
-
- @classmethod
- @lru_cache(maxsize=8)
- def get_conversions(cls, src_ver, dst_ver, is_qweb=False):
- """
- Return the list of conversions to convert Bootstrap from ``src_ver`` to ``dst_ver``, with compiled XPaths.
-
- :param str src_ver: the source Bootstrap version.
- :param str dst_ver: the destination Bootstrap version.
- :param bool is_qweb: whether to adapt conversions for QWeb (to support ``t-att(f)-`` conversions).
- :rtype: list[(etree.XPath, list[ElementOperation])]
- """
- if Version(dst_ver) < Version(src_ver):
- raise NotImplementedError("Downgrading Bootstrap versions is not supported.")
- if Version(src_ver) < Version(cls.MIN_VERSION):
- raise NotImplementedError(f"Conversion from Bootstrap version {src_ver} is not supported")
- result = []
- for version, conversions in BootstrapConverter._get_sorted_conversions():
- if Version(src_ver) < Version(version) <= Version(dst_ver):
- result.extend(conversions)
- if not result:
- if Version(src_ver) == Version(dst_ver):
- _logger.info("Source and destination versions are the same, no conversion needed.")
- else:
- raise NotImplementedError(f"Conversion from {src_ver} to {dst_ver} is not supported")
- if is_qweb:
- result = [(adapt_xpath_for_qweb(xpath), conversions) for xpath, conversions in result]
- return [(etree.XPath(xpath), conversions) for xpath, conversions in result]
-
- def convert(self, src_version, dst_version):
- """
- Convert the loaded document inplace from the source version to the destination, returning the converted document and the number of conversion operations applied.
-
- :param str src_version: the source Bootstrap version.
- :param str dst_version: the destination Bootstrap version.
- :rtype: etree.ElementTree, int
- """
- conversions = self.get_conversions(src_version, dst_version, is_qweb=self.is_qweb)
- applied_operations_count = 0
- for xpath, operations in conversions:
- for element in xpath(self.tree):
- for operation in operations:
- if element is None: # previous operations that returned None (i.e. deleted element)
- raise ValueError("Matched xml element is not available anymore! Check operations.")
- element = operation(element, self) # noqa: PLW2901
- applied_operations_count += 1
- return self.tree, applied_operations_count
-
- @classmethod
- def convert_arch(cls, arch, src_version, dst_version, is_html=False, **converter_kwargs):
- """
- Class method for converting a string of XML or HTML code.
-
- :param str arch: the XML or HTML code to convert.
- :param str src_version: the source Bootstrap version.
- :param str dst_version: the destination Bootstrap version.
- :param bool is_html: whether the arch is an HTML document.
- :param dict converter_kwargs: additional keyword arguments to pass to the converter.
- :return: the converted XML or HTML code.
- :rtype: str
- """
- stripped_arch = arch.strip()
- doc_header_match = re.search(r"^<\?xml .+\?>\s*", stripped_arch)
- doc_header = doc_header_match.group(0) if doc_header_match else ""
- stripped_arch = stripped_arch[doc_header_match.end() :] if doc_header_match else stripped_arch
-
- tree = etree.fromstring(f"
{stripped_arch}", parser=html_utf8_parser if is_html else None)
-
- tree, ops_count = cls(tree, is_html, **converter_kwargs).convert(src_version, dst_version)
- if not ops_count:
- return arch
-
- wrap_node = tree.xpath("//wrap")[0]
- return doc_header + "\n".join(
- etree.tostring(child, encoding="unicode", with_tail=True, method="html" if is_html else None)
- for child in wrap_node
- )
-
- @classmethod
- def convert_file(cls, path, src_version, dst_version, is_html=None, **converter_kwargs):
- """
- Class method for converting an XML or HTML file inplace.
-
- :param str path: the path to the XML or HTML file to convert.
- :param str src_version: the source Bootstrap version.
- :param str dst_version: the destination Bootstrap version.
- :param bool is_html: whether the file is an HTML document.
- If not set, will be detected from the file extension.
- :param dict converter_kwargs: additional keyword arguments to pass to the converter.
- :rtype: None
- """
- if is_html is None:
- is_html = os.path.splitext(path)[1].startswith("htm")
- tree = etree.parse(path, parser=html_utf8_parser if is_html else None)
-
- tree, ops_count = cls(tree, is_html, **converter_kwargs).convert(src_version, dst_version)
- if not ops_count:
- logging.info("No conversion operations applied, skipping file %s", path)
- return
-
- tree.write(path, encoding="utf-8", method="html" if is_html else None, xml_declaration=not is_html)
-
- def element_factory(self, *args, **kwargs):
- """
- Create new elements using the correct document type.
-
- Basically a wrapper for either etree.XML or etree.HTML depending on the type of document loaded.
-
- :param args: positional arguments to pass to the etree.XML or etree.HTML function.
- :param kwargs: keyword arguments to pass to the etree.XML or etree.HTML function.
- :return: the created element.
- """
- return etree.HTML(*args, **kwargs) if self.is_html else etree.XML(*args, **kwargs)
-
- def build_element(self, tag, classes=None, contents=None, **attributes):
- """
- Create a new element with the given tag, classes, contents and attributes.
-
- Like :meth:`~.element_factory`, can be used by operations to create elements abstracting away the document type.
-
- :param str tag: the tag of the element to create.
- :param typing.Iterable[str] | None classes: the classes to set on the new element.
- :param str | None contents: the contents of the new element (i.e. inner text/HTML/XML).
- :param dict[str, str] attributes: attributes to set on the new element, provided as keyword arguments.
- :return: the created element.
- :rtype: etree.ElementBase
- """
- element = self.element_factory(f"<{tag}>{contents or ''}")
- for name, value in attributes.items():
- element.attrib[name] = value
- if classes:
- set_classes(element, classes)
- return element
-
- def copy_element(
- self,
- element,
- tag=None,
- add_classes=None,
- remove_classes=None,
- copy_attrs=True,
- copy_contents=True,
- **attributes,
- ):
- """
- Create a copy of an element, optionally changing the tag, classes, contents and attributes.
-
- Like :meth:`~.element_factory`, can be used by operations to copy elements abstracting away the document type.
-
- :param etree.ElementBase element: the element to copy.
- :param str | None tag: if specified, overrides the tag of the new element.
- :param str | typing.Iterable[str] | None add_classes: if specified, adds the given class(es) to the new element.
- :param str | typing.Iterable[str] | ALL | None remove_classes: if specified, removes the given class(es)
- from the new element. The `ALL` sentinel value can be specified to remove all classes.
- :param bool copy_attrs: if True, copies the attributes of the source element to the new one. Defaults to True.
- :param bool copy_contents: if True, copies the contents of the source element to the new one. Defaults to True.
- :param dict[str, str] attributes: attributes to set on the new element, provided as keyword arguments.
- Will be str merged with the attributes of the source element, overriding the latter.
- :return: the new copied element.
- :rtype: etree.ElementBase
- """
- tag = tag or element.tag
- contents = innerxml(element, is_html=self.is_html) if copy_contents else None
- if copy_attrs:
- attributes = {**element.attrib, **attributes}
- new_element = self.build_element(tag, contents=contents, **attributes)
- edit_element_classes(new_element, add_classes, remove_classes, is_qweb=self.is_qweb)
- return new_element
-
- def adapt_xpath(self, xpath):
- """Adapts an xpath to match qweb ``t-att(f)-*`` attributes, if ``is_qweb`` is True."""
- return adapt_xpath_for_qweb(xpath) if self.is_qweb else xpath
-
-
-def convert_tree(tree, src_version, dst_version, **converter_kwargs):
- """
- Convert an already parsed lxml tree from Bootstrap v3 to v4 inplace.
-
- :param etree.ElementTree tree: the lxml tree to convert.
- :param str src_version: the version of Bootstrap the document is currently using.
- :param str dst_version: the version of Bootstrap to convert the document to.
- :param dict converter_kwargs: additional keyword arguments to initialize :class:`~.BootstrapConverter`.
- :return: the converted lxml tree.
- :rtype: etree.ElementTree
- """
- tree, _ops_count = BootstrapConverter(tree, **converter_kwargs).convert(src_version, dst_version)
- return tree
-
-
-convert_arch = BootstrapConverter.convert_arch
-convert_file = BootstrapConverter.convert_file
-
-
-class BootstrapHTMLConverter:
- def __init__(self, src, dst):
- self.src = src
- self.dst = dst
-
- def __call__(self, content):
- if not content:
- return False, content
- converted_content = convert_arch(content, self.src, self.dst, is_html=True, is_qweb=True)
- return content != converted_content, converted_content
diff --git a/src/util/data.py b/src/util/data.py
deleted file mode 100644
index 86ba1af9d..000000000
--- a/src/util/data.py
+++ /dev/null
@@ -1,154 +0,0 @@
-# -*- coding: utf-8 -*-
-import logging
-
-from .helpers import table_of_model
-from .pg import SQLStr, format_query, table_exists
-from .records import ref, remove_records, replace_record_references_batch
-
-_logger = logging.getLogger(__name__.rpartition(".")[0])
-
-# python3 shims
-try:
- basestring # noqa: B018
-except NameError:
- basestring = str
-
-
-def uniq_tags(cr, model, uniq_column="name", order="id"):
- """
- Deduplicate "tag" models entries.
-
- In standard, should only be referenced as many2many
- But with a customization, could be referenced as many2one
-
- By using `uniq_column=lower(name)` and `order=name`
- you can prioritize tags in CamelCase/UPPERCASE.
- """
- table = table_of_model(cr, model)
-
- query = format_query(
- cr,
- """
- SELECT unnest((array_agg(id ORDER BY {order}))[2:]),
- (array_agg(id ORDER BY {order}))[1]
- FROM {table}
- GROUP BY {uniq_column}
- HAVING count(id) > 1
- """,
- table=table,
- order=SQLStr(order),
- uniq_column=SQLStr(uniq_column),
- )
- cr.execute(query)
- if not cr.rowcount:
- return
- mapping = dict(cr.fetchall())
- replace_record_references_batch(cr, mapping, model)
- remove_records(cr, model, mapping.keys())
-
-
-def split_group(cr, from_groups, to_group):
- """Users have all `from_groups` will be added into `to_group`."""
-
- def check_group(g):
- if isinstance(g, basestring):
- gid = ref(cr, g)
- if not gid:
- _logger.warning("split_group(): Unknown group: %r", g)
- return gid
- return g
-
- if not isinstance(from_groups, (list, tuple, set)):
- from_groups = [from_groups]
-
- from_groups = [g for g in map(check_group, from_groups) if g]
- if not from_groups:
- return
-
- if isinstance(to_group, basestring):
- to_group = ref(cr, to_group)
-
- assert to_group
-
- cr.execute(
- """
- INSERT INTO res_groups_users_rel(uid, gid)
- SELECT uid, %s
- FROM res_groups_users_rel
- GROUP BY uid
- HAVING array_agg(gid) @> %s
- EXCEPT
- SELECT uid, gid
- FROM res_groups_users_rel
- WHERE gid = %s
- """,
- [to_group, from_groups, to_group],
- )
-
-
-def drop_workflow(cr, osv):
- if not table_exists(cr, "wkf"):
- # workflows have been removed in 10.saas~14
- # noop if there is no workflow tables anymore...
- return
-
- cr.execute(
- """
- -- we want to first drop the foreign keys on the workitems because
- -- it slows down the process a lot
- ALTER TABLE wkf_triggers DROP CONSTRAINT wkf_triggers_workitem_id_fkey;
- ALTER TABLE wkf_workitem DROP CONSTRAINT wkf_workitem_act_id_fkey;
- ALTER TABLE wkf_workitem DROP CONSTRAINT wkf_workitem_inst_id_fkey;
- ALTER TABLE wkf_triggers DROP CONSTRAINT wkf_triggers_instance_id_fkey;
-
- -- if this workflow is used as a subflow, complete workitem running this subflow
- UPDATE wkf_workitem wi
- SET state = 'complete'
- FROM wkf_instance i JOIN wkf w ON (w.id = i.wkf_id)
- WHERE wi.subflow_id = i.id
- AND w.osv = %(osv)s
- AND wi.state = 'running'
- ;
-
- -- delete the workflow and dependencies
- WITH deleted_wkf AS (
- DELETE FROM wkf WHERE osv = %(osv)s RETURNING id
- ),
- deleted_wkf_instance AS (
- DELETE FROM wkf_instance i
- USING deleted_wkf w
- WHERE i.wkf_id = w.id
- RETURNING i.id
- ),
- _delete_triggers AS (
- DELETE FROM wkf_triggers t
- USING deleted_wkf_instance i
- WHERE t.instance_id = i.id
- ),
- deleted_wkf_activity AS (
- DELETE FROM wkf_activity a
- USING deleted_wkf w
- WHERE a.wkf_id = w.id
- RETURNING a.id
- )
- DELETE FROM wkf_workitem wi
- USING deleted_wkf_instance i
- WHERE wi.inst_id = i.id
- ;
-
- -- recreate constraints
- ALTER TABLE wkf_triggers ADD CONSTRAINT wkf_triggers_workitem_id_fkey
- FOREIGN KEY (workitem_id) REFERENCES wkf_workitem(id)
- ON DELETE CASCADE;
- ALTER TABLE wkf_workitem ADD CONSTRAINT wkf_workitem_act_id_fkey
- FOREIGN key (act_id) REFERENCES wkf_activity(id)
- ON DELETE CASCADE;
- ALTER TABLE wkf_workitem ADD CONSTRAINT wkf_workitem_inst_id_fkey
- FOREIGN KEY (inst_id) REFERENCES wkf_instance(id)
- ON DELETE CASCADE;
- ALTER TABLE wkf_triggers ADD CONSTRAINT wkf_triggers_instance_id_fkey
- FOREIGN KEY (instance_id) REFERENCES wkf_instance(id)
- ON DELETE CASCADE;
- """,
- {"osv": osv},
- )
diff --git a/src/util/domains.py b/src/util/domains.py
deleted file mode 100644
index b4efeb75b..000000000
--- a/src/util/domains.py
+++ /dev/null
@@ -1,571 +0,0 @@
-# -*- coding: utf-8 -*-
-import ast
-import collections
-import functools
-import logging
-import re
-import sys
-import warnings
-
-import lxml
-
-try:
- from contextlib import suppress
-except ImportError:
- # python2 code, use the openerp vendor
- try:
- from openerp.tools.misc import ignore as suppress
- except ImportError:
- # this is to allow v7.0 DBs to import this module without errors
- # note: some functions on this module will fail (like adapt_domains)
- suppress = None
-
-try:
- from html import unescape
-except ImportError:
- # should not be needed in python2
- unescape = lambda x: x
-
-try:
- from odoo.tools import exception_to_unicode
-except ImportError:
- from openerp.tools import exception_to_unicode
-
-from .const import NEARLYWARN
-from .helpers import _dashboard_actions, _validate_model, resolve_model_fields_path
-from .inherit import for_each_inherit
-from .misc import SelfPrintEvalContext, ast_unparse, literal_replace, safe_eval, version_gte
-from .pg import column_exists, get_value_or_en_translation, table_exists
-from .records import edit_view
-
-# python3 shims
-try:
- basestring # noqa: B018
-except NameError:
- basestring = unicode = str
-
-# import from domains/expression
-try:
- if not version_gte("saas~18.2"):
- raise ImportError("Use osv.expression to migrate") # noqa: TRY301
- import odoo.orm.domains as _dom
-
- FALSE_LEAF = _dom._FALSE_LEAF
- TRUE_LEAF = _dom._TRUE_LEAF
- NOT_OPERATOR = _dom.DomainNot.OPERATOR
- AND_OPERATOR = _dom.DomainAnd.OPERATOR
- OR_OPERATOR = _dom.DomainOr.OPERATOR
- DOMAIN_OPERATORS = {NOT_OPERATOR, AND_OPERATOR, OR_OPERATOR}
-
- # normalization functions based on odoo.orm.domains
-
- def normalize_domain(domain):
- """Return a normalized version of the domain.
-
- The normalized version have all implicit '&' operators explicit.
- One property of normalized domain expressions is that they
- can be easily combined together as if they were single domain components.
- """
- return list(_dom.Domain(domain))
-
- def normalize_leaf(leaf):
- if not is_leaf(leaf):
- raise TypeError("Leaf must be a tuple or list of 3 values")
- domain = _dom.Domain(*leaf)
- assert isinstance(domain, _dom.DomainCondition)
- return next(iter(domain))
-
- def is_leaf(leaf):
- return (
- isinstance(leaf, (tuple, list))
- and len(leaf) == 3
- and leaf[1] in _dom.CONDITION_OPERATORS
- and (isinstance(leaf[0], str) or leaf in (TRUE_LEAF, FALSE_LEAF))
- )
-except ImportError:
- try:
- from odoo.osv import expression
- except ImportError:
- from openerp.osv import expression
- FALSE_LEAF = expression.FALSE_LEAF
- TRUE_LEAF = expression.TRUE_LEAF
- NOT_OPERATOR = expression.NOT_OPERATOR
- AND_OPERATOR = expression.AND_OPERATOR
- OR_OPERATOR = expression.OR_OPERATOR
- DOMAIN_OPERATORS = expression.DOMAIN_OPERATORS
- warnings.filterwarnings("ignore", message=r"Since 19\.0, use Domain\(\) object", category=DeprecationWarning)
- normalize_domain = expression.normalize_domain
- normalize_leaf = expression.normalize_leaf
- is_leaf = expression.is_leaf
- del expression # unset so it's not used directly
-
-
-AST_CONSTANT_TYPES = (
- (ast.Str,)
- if sys.version_info < (3, 0)
- else (ast.Constant, ast.Str)
- if sys.version_info < (3, 12)
- else (ast.Constant,)
-)
-
-_logger = logging.getLogger(__name__)
-DomainField = collections.namedtuple("DomainField", "table domain_column model_select")
-"""
-Domain field
-
-:meta private: exclude from online docs
-"""
-
-
-class _Skip(Exception):
- pass
-
-
-def _get_domain_fields(cr):
- # haaa, if only we had a `fields.Domain`, we would just have to get all the domains from `ir_model_fields`
- # Meanwile, we have to enumerate them explicitly
- # false friends: the `domain` fields on `website` and `amazon.marketplace` are actually domain names.
- # NOTE: domains on transient models have been ignored
- result = []
- if column_exists(cr, "mail_mass_mailing", "mailing_model_id"):
- # >= saas~18
- result = [
- DomainField(
- "mail_mass_mailing", "mailing_domain", "(SELECT model FROM ir_model m WHERE m.id = t.mailing_model_id)"
- )
- ]
- elif column_exists(cr, "mail_mass_mailing", "mailing_model"):
- # >= saas~4
- result = [DomainField("mail_mass_mailing", "mailing_domain", "mailing_model")]
- else:
- mail_template = "mail_template" if table_exists(cr, "mail_template") else "email_template"
- result = [
- DomainField(
- "mail_mass_mailing",
- "mailing_domain",
- "(SELECT model FROM {} m WHERE m.id = t.template_id)".format(mail_template),
- )
- ]
-
- if column_exists(cr, "base_automation", "action_server_id"):
- result += [
- DomainField(
- "base_automation",
- "filter_domain",
- "(SELECT model_name FROM ir_act_server WHERE id = t.action_server_id)",
- ),
- DomainField(
- "base_automation",
- "filter_pre_domain",
- "(SELECT model_name FROM ir_act_server WHERE id = t.action_server_id)",
- ),
- ]
- else:
- result = result + [
- DomainField("base_automation", "filter_domain", "(SELECT model FROM ir_model m WHERE m.id = t.model_id)"),
- DomainField(
- "base_automation", "filter_pre_domain", "(SELECT model FROM ir_model m WHERE m.id = t.model_id)"
- ),
- ]
-
- documents_domains_target = "'documents.document'" if table_exists(cr, "documents_document") else "'ir.attachment'"
-
- result = result + [
- DomainField("ir_model_fields", "domain", "model"),
- DomainField("ir_act_window", "domain", "res_model"),
- DomainField("ir_act_report_xml", "domain", "(SELECT model FROM ir_model m WHERE m.id = t.binding_model_id)"),
- DomainField("ir_filters", "domain", "model_id"), # model_id is a varchar
- DomainField("ir_embedded_actions", "domain", "parent_res_model"),
- DomainField("ir_rule", "domain_force", "(SELECT model FROM ir_model m WHERE m.id = t.model_id)"),
- DomainField("document_directory", "domain", "(SELECT model FROM ir_model m WHERE m.id = t.ressource_type_id)"),
- DomainField(
- "mailing_mailing", "mailing_domain", "(SELECT model FROM ir_model m WHERE m.id = t.mailing_model_id)"
- ),
- DomainField("base_action_rule", "filter_domain", "(SELECT model FROM ir_model m WHERE m.id = t.model_id)"),
- DomainField("base_action_rule", "filter_pre_domain", "(SELECT model FROM ir_model m WHERE m.id = t.model_id)"),
- DomainField("gamification_goal_definition", "domain", "(SELECT model FROM ir_model m WHERE m.id = t.model_id)"),
- DomainField("marketing_campaign", "domain", "model_name"),
- DomainField(
- "marketing_activity", "domain", "(SELECT model_name FROM marketing_campaign WHERE id = t.campaign_id)"
- ),
- DomainField(
- "marketing_activity",
- "activity_domain",
- "(SELECT model_name FROM marketing_campaign WHERE id = t.campaign_id)",
- ),
- DomainField("data_merge_model", "domain", "res_model_name"),
- # static target model
- DomainField("account_financial_html_report_line", "domain", "'account.move.line'"),
- DomainField("account_financial_html_report_line", "control_domain", "'account.move.line'"),
- DomainField("gamification_challenge", "user_domain", "'res.users'"),
- DomainField("pos_cache", "product_domain", "'product.product'"),
- DomainField("sale_coupon_rule", "rule_partners_domain", "'res.partner'"),
- DomainField("sale_coupon_rule", "rule_products_domain", "'product.product'"),
- DomainField("coupon_rule", "rule_partners_domain", "'res.partner'"),
- DomainField("coupon_rule", "rule_products_domain", "'product.product'"),
- DomainField("sale_subscription_template", "good_health_domain", "'sale.subscription'"),
- DomainField("sale_subscription_template", "bad_health_domain", "'sale.subscription'"),
- DomainField("website_crm_score", "domain", "'crm.lead'"),
- DomainField("team_user", "team_user_domain", "'crm.lead'"),
- DomainField("crm_team_member", "assignment_domain", "'crm.lead'"),
- DomainField("crm_team", "score_team_domain", "'crm.lead'"),
- DomainField("crm_team", "assignment_domain", "'crm.lead'"),
- DomainField("social_post", "visitor_domain", "'website.visitor'"),
- DomainField("social_post_template", "visitor_domain", "'website.visitor'"),
- DomainField("documents_share", "domain", documents_domains_target),
- DomainField("documents_workflow_rule", "domain", documents_domains_target),
- DomainField("loyalty_rule", "rule_domain", "'product.product'"),
- DomainField("loyalty_rule", "product_domain", "'product.product'"),
- DomainField("loyalty_reward", "discount_product_domain", "'product.product'"),
- ]
-
- for df in result:
- if column_exists(cr, df.table, df.domain_column):
- yield df
-
-
-def _model_of_path(cr, model, path):
- if not path:
- return model
- path = tuple(path)
- resolved_parts = resolve_model_fields_path(cr, model, path)
- if len(resolved_parts) == len(path):
- return resolved_parts[-1].relation_model
- return None
-
-
-def _valid_path_to(cr, path, from_, to):
- model = _model_of_path(cr, from_, path)
- return model is not None and model == to
-
-
-def _replace_path(cr, old, new, src_model, dst_model, path_str):
- """
- Replace `old` by `new` in the fields path `path_str` assuming the path starts from `src_model`.
-
- The replace only takes place if `old` points at `dst_model`.
- """
- dot_old = old.split(".")
- dot_new = new.split(".")
- path = path_str.split(".")
- for idx in range(len(path) - len(dot_old), -1, -1):
- r = slice(idx, idx + len(dot_old))
- if path[r] == dot_old and _valid_path_to(cr, path[:idx], src_model, dst_model):
- path[r] = dot_new
- return ".".join(path)
-
-
-def _adapt_one_domain(cr, target_model, old, new, model, domain, adapter=None, force_adapt=False):
- if not isinstance(domain, basestring):
- # defer to the old adapter that replaces invalid terms
- return _adapt_one_domain_old(cr, target_model, old, new, model, domain, adapter, force_adapt)
-
- def domain_check(ast_node):
- if all(
- (
- isinstance(term, (ast.Tuple, ast.List))
- and len(term.elts) == 3
- and isinstance(term.elts[1], AST_CONSTANT_TYPES)
- )
- or isinstance(term, AST_CONSTANT_TYPES)
- for term in ast_node.elts
- ) and any(isinstance(term, (ast.Tuple, ast.List)) and len(term.elts) == 3 for term in ast_node.elts):
- # it looks like a domain, let's adapt it
- internal_domain = _adapt_one_domain_old(
- cr,
- target_model,
- old,
- new,
- model,
- ast_unparse(ast_node),
- adapter=adapter,
- force_adapt=force_adapt,
- )
- return ast.parse(unicode(internal_domain), mode="eval").body if internal_domain else ast_node
- return ast_node
-
- new_domain = literal_replace(domain, {ast.List([literal_replace.WILDCARD], None): domain_check})
- if new_domain != domain:
- return new_domain
- return None
-
-
-def _adapt_one_domain_old(cr, target_model, old, new, model, domain, adapter=None, force_adapt=False):
- if not adapter:
- adapter = lambda leaf, _, __: [leaf]
-
- # pre-check domain
- if isinstance(domain, basestring):
- try:
- replaced_domain, ctx = SelfPrintEvalContext.preprocess(domain)
- eval_dom = normalize_domain(safe_eval(replaced_domain, ctx))
- except Exception as e:
- oops = exception_to_unicode(e)
- _logger.log(NEARLYWARN, "Cannot evaluate %r domain: %r: %s", model, domain, oops)
- return None
- else:
- try:
- eval_dom = normalize_domain(domain)
- except Exception as e:
- oops = exception_to_unicode(e)
- _logger.log(NEARLYWARN, "Invalid %r domain: %r: %s", model, domain, oops)
- return None
-
- dot_old = old.split(".")
- clean_path = functools.partial(_replace_path, cr, old, new, model, target_model)
-
- def clean_term(term):
- if isinstance(term, basestring) or not isinstance(term[0], basestring):
- return term
- left, op, right = term
- left = clean_path(left)
- if op in ("any", "not any"):
- new_right = _adapt_one_domain(
- cr,
- target_model,
- old,
- new,
- model=_model_of_path(cr, model, left.split(".")),
- domain=right,
- adapter=adapter,
- force_adapt=force_adapt,
- )
- if new_right is not None:
- right = new_right
- return (left, op, right)
-
- final_dom = []
- changed = False
- op_arity = {NOT_OPERATOR: 1, AND_OPERATOR: 2, OR_OPERATOR: 2}
- op_stack = [] # (operator, number of terms missing)
- for element in eval_dom:
- while op_stack and op_stack[-1][1] == 0:
- op_stack.pop() # found all terms current operator was expecting, pop it
- op_stack[-1][1] -= 1 # previous operator now got one more term
-
- if isinstance(element, basestring):
- if element not in op_arity:
- _logger.log(NEARLYWARN, "Invalid domain on %r: %s", model, domain)
- return None
- op_stack.append([element, op_arity[element]])
- final_dom.append(element)
- continue
-
- if not is_leaf(element):
- _logger.log(NEARLYWARN, "Invalid domain on %r: %s", model, domain)
- return None
-
- if op_stack:
- op_stack[-1][1] -= 1 # previous operator got a term
-
- if tuple(element) in [TRUE_LEAF, FALSE_LEAF]:
- final_dom.append(element)
- continue
-
- is_or = False
- neg = False
- for op, _ in reversed(op_stack):
- if op != NOT_OPERATOR:
- is_or = op == OR_OPERATOR
- break
- neg = not neg
-
- leaf = normalize_leaf(element)
- path = leaf[0].split(".")
- # force_adapt=True -> always adapt if found anywhere on left path
- # otherwise adapt only when {old} field is the last parts of left path
- search_limit = len(path) - len(dot_old)
- search_range = range(search_limit + 1) if force_adapt else [search_limit]
- if any(
- path[i : i + len(dot_old)] == dot_old and _valid_path_to(cr, path[:i], model, target_model)
- for i in search_range
- ):
- dom = [clean_term(term) for term in adapter(leaf, is_or, neg)]
- else:
- dom = [clean_term(leaf)]
-
- if dom != [leaf]:
- changed = True
-
- final_dom.extend(dom)
-
- if not changed:
- return None
-
- _logger.debug("%s: %r -> %r", model, domain, final_dom)
- return final_dom
-
-
-if ast_unparse is None:
- _adapt_one_domain = _adapt_one_domain_old
-
-
-def adapt_domains(cr, model, old, new, adapter=None, skip_inherit=(), force_adapt=False):
- """
- Replace `old` by `new` in domains using `model` and inheriting models.
-
- `adapter` is a callback function to adapt leaves. Adapter functions must take three
- arguments and return a `domain
`_ that substitutes the original
- leaf. The arguments are:
-
- - `leaf`: a domain leaf which is a `tuple` of the form `(left, op, right)`
- - `in_or`: a boolean, when `True` it means the leaf is part of an OR (`"|"`) domain,
- otherwise it is part of an AND (`"&"`) domain
- - `negated`: a boolean, when `True` it means that the leaf is negated (`"!"`)
-
- .. example::
- .. code-block:: python
-
- def adapter(leaf, in_or, negated):
- left, op, right = leaf
- ok, ko = (1, 2) if not negated else (2, 1)
- if op == "="
- return [(left, "=", ok)]
- elif op == "!=":
- return [(left, "=", ko)]
- return [leaf]
-
- `adapter` is called only on leaves that use the `old` field of `model` as **last**
- part of the `left` part of leaves, unless `force_adapt` is `True`. In the latter case
- the adapter is called if the field appears anywhere in the path, useful for relational
- fields only.
-
- The domains returned by an adapter do not need to have the `old` field replaced by
- `new` in the `left` part of the input leaf. The replace will be done anyway to the
- whole domain returned by the adapter. The usual purpose of the `adapter` is to modify
- the operator and the `right` part of the input leaf. When `adapter` is not set only
- the replacement takes place.
-
- .. example::
- When replacing `"field1"` by `"field2"`, the following happens:
-
- - `("foo.bar.baz.field1", "=", 1)` gets adapted *only* if the record pointed to by
- `foo.bar.baz` is of the requested `model`.
- - `("foo.field1.baz", "=", 1)` is *not* adapted *even* if `foo` points to `model`,
- unless `force_adapt` is `True`, because `field1` is not the last part of `left`
- in this leaf.
-
- .. note::
- This function will replace domains in all *standard* domain fields. Including
- filters, dashboards, and standard fields known to represent a domain.
-
- :param str model: name of the model for which to adapt the domains
- :param str old: name of the field to be adapted
- :param str new: name of the field that should replace `old`
- :param function adapter: adapter for leaves
- :param list(str) skip_inherit: list of inheriting model names to don't adapt (skip)
- :param bool force_adapt: when `True`, run the `adapter` on all leaves having `new` in
- `left` part of the leaf (path), useful when deleting a field
- (in which case `new` is ignored).
- """
- _validate_model(model)
- target_model = model
-
- match_old = r"\y{}\y".format(re.escape(old))
- for df in _get_domain_fields(cr):
- cr.execute(
- """
- SELECT id, {df.model_select}, {df.domain_column}
- FROM {df.table} t
- WHERE {df.domain_column} ~ %s
- """.format(df=df),
- [match_old],
- )
- for id_, model, domain in cr.fetchall(): # noqa: PLR1704
- new_domain = _adapt_one_domain(
- cr, target_model, old, new, model, domain, adapter=adapter, force_adapt=force_adapt
- )
- if new_domain:
- cr.execute(
- "UPDATE {df.table} SET {df.domain_column} = %s WHERE id = %s".format(df=df),
- [unicode(new_domain), id_],
- )
-
- # adapt search views
- arch_db = (
- get_value_or_en_translation(cr, "ir_ui_view", "arch_db")
- if column_exists(cr, "ir_ui_view", "arch_db")
- else "arch"
- )
- active_col = "active" if column_exists(cr, "ir_ui_view", "active") else "true"
- cr.execute("SELECT id, model, {} FROM ir_ui_view WHERE {} ~ %s".format(active_col, arch_db), [match_old])
- for view_id, view_model, view_active in cr.fetchall():
- # Note: active=None is important to not reactivate views!
- try:
- with suppress(_Skip), edit_view(cr, view_id=view_id, active=None) as view:
- modified = False
- for node in view.xpath(
- "//filter[contains(@domain, '{0}')]|//field[contains(@filter_domain, '{0}')]".format(old)
- ):
- attr = "domain" if "domain" in node.attrib else "filter_domain"
- domain = _adapt_one_domain(
- cr, target_model, old, new, view_model, node.get(attr), adapter=adapter, force_adapt=force_adapt
- )
- if domain:
- node.set(attr, unicode(domain))
- modified = True
-
- for node in view.xpath("//field[contains(@domain, '{0}')]".format(old)):
- # as can happen in sub-views, we should determine the actual model the field belongs to
- path = list(reversed([p.get("name") for p in node.iterancestors("field")])) + [node.get("name")]
- field_model = _model_of_path(cr, view_model, path)
- if not field_model:
- continue
-
- domain = _adapt_one_domain(
- cr,
- target_model,
- old,
- new,
- field_model,
- node.get("domain"),
- adapter=adapter,
- force_adapt=force_adapt,
- )
- if domain:
- node.set("domain", unicode(domain))
- modified = True
-
- if not modified:
- raise _Skip
- except lxml.etree.XMLSyntaxError as e:
- if e.msg.startswith("Opening and ending tag mismatch") or not view_active:
- # this view is already wrong, we don't change it
- _logger.warning(
- "Skipping domain adaptation for %sinvalid view (id=%s):\n%s",
- "" if view_active else "inactive, ",
- view_id,
- e.msg,
- )
- continue
- _logger.error("Cannot adapt domain of invalid view (id=%s)", view_id) # noqa: TRY400
- raise
-
- # adapt domain in dashboards.
- # NOTE: does not filter on model at dashboard selection for handle dotted domains
- for _, act in _dashboard_actions(cr, match_old):
- if act.get("domain"):
- try:
- act_id = int(act.get("name", "FAIL"))
- except ValueError:
- continue
-
- cr.execute("SELECT res_model FROM ir_act_window WHERE id = %s", [act_id])
- if not cr.rowcount:
- continue
- [act_model] = cr.fetchone()
-
- domain = act.get("domain")
- if any(entity in domain for entity in ("", "&", "<", ">")):
- # There is a bug Odoo 16.0 that double escape the domains in dashboard...
- # See https://github.com/odoo/odoo/pull/119518
- domain = unescape(domain)
- domain = _adapt_one_domain(
- cr, target_model, old, new, act_model, domain, adapter=adapter, force_adapt=force_adapt
- )
- if domain:
- act.set("domain", unicode(domain))
-
- # down on inherits
- for inh in for_each_inherit(cr, target_model, skip_inherit):
- adapt_domains(cr, inh.model, old, new, adapter, skip_inherit=skip_inherit, force_adapt=force_adapt)
diff --git a/src/util/exceptions.py b/src/util/exceptions.py
deleted file mode 100644
index 834cad67f..000000000
--- a/src/util/exceptions.py
+++ /dev/null
@@ -1,21 +0,0 @@
-# -*- coding: utf-8 -*-
-
-
-class UpgradeError(Exception):
- pass
-
-
-class SleepyDeveloperError(ValueError):
- pass
-
-
-class UnknownModuleError(AssertionError):
- pass
-
-
-class UpgradeWarning(Warning):
- pass
-
-
-# Compat
-MigrationError = UpgradeError
diff --git a/src/util/fields.py b/src/util/fields.py
deleted file mode 100644
index ee66fda1f..000000000
--- a/src/util/fields.py
+++ /dev/null
@@ -1,1788 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-Utility functions for modifying model fields.
-
-Field operations are best done in `pre-` script of the involved modules. In some cases a
-preliminary operation could be done in pre and finished in post. A common example is to
-remove a field in pre- keeping its column, later used in post when the column is finally
-dropped.
-"""
-
-import base64
-import logging
-import re
-import warnings
-from ast import literal_eval
-
-import psycopg2
-from psycopg2 import sql
-from psycopg2.extras import Json
-
-try:
- from odoo import modules
-except ImportError:
- from openerp import modules
-
-try:
- from odoo import release
-except ImportError:
- from openerp import release
-
-from .domains import FALSE_LEAF, TRUE_LEAF
-
-try:
- from odoo.tools.sql import make_index_name
-except ImportError:
-
- def make_index_name(table_name, column_name):
- return "%s_%s_index" % (table_name, column_name)
-
-
-try:
- from odoo.tools import pickle
-except ImportError:
- import pickle
-
-
-from . import json
-from .const import ENVIRON
-from .domains import _adapt_one_domain, _replace_path, _valid_path_to, adapt_domains
-from .exceptions import SleepyDeveloperError, UpgradeError
-from .helpers import _dashboard_actions, _validate_model, resolve_model_fields_path, table_of_model
-from .inherit import for_each_inherit
-from .misc import AUTO, log_progress, safe_eval, version_gte
-from .orm import env, invalidate
-from .pg import (
- PGRegexp,
- SQLStr,
- alter_column_type,
- column_exists,
- column_type,
- create_m2m,
- explode_execute,
- explode_query_range,
- format_query,
- get_columns,
- get_value_or_en_translation,
- parallel_execute,
- pg_replace,
- pg_text2html,
- remove_column,
- table_exists,
- target_of,
-)
-from .records import _remove_import_export_paths
-from .report import add_to_migration_reports, get_anchor_link_to_record
-
-# python3 shims
-try:
- basestring # noqa: B018
-except NameError:
- basestring = unicode = str
-
-
-_logger = logging.getLogger(__name__)
-IMD_FIELD_PATTERN = "field_%s__%s" if version_gte("saas~11.2") else "field_%s_%s"
-
-_CONTEXT_KEYS_TO_CLEAN = (
- "group_by",
- "pivot_measures",
- "pivot_column_groupby",
- "pivot_row_groupby",
- "graph_groupbys",
- "orderedBy",
-)
-
-
-def ensure_m2o_func_field_data(cr, src_table, column, dst_table):
- """
- Fix broken m2o relations.
-
- If any `column` not present in `dst_table`, remove column from `src_table` in
- order to force recomputation of the function field
-
- WARN: only call this method on m2o function/related fields!!
-
- :meta private: exclude from online docs
- """
- if not column_exists(cr, src_table, column):
- return
- cr.execute(
- """
- SELECT count(1)
- FROM "{src_table}"
- WHERE "{column}" NOT IN (SELECT id FROM "{dst_table}")
- """.format(src_table=src_table, column=column, dst_table=dst_table)
- )
- if cr.fetchone()[0]:
- remove_column(cr, src_table, column, cascade=True)
-
-
-def _remove_field_from_filters(cr, model, field):
- cr.execute(
- "SELECT id, name, context FROM ir_filters WHERE model_id = %s AND context ~ %s",
- [model, r"\y{}\y".format(field)],
- )
- for id_, name, context_s in cr.fetchall():
- context = safe_eval(context_s or "{}")
- changed = _remove_field_from_context(context, field)
- cr.execute("UPDATE ir_filters SET context = %s WHERE id = %s", [unicode(context), id_])
- if changed:
- add_to_migration_reports(("ir.filters", id_, name), "Filters/Dashboards")
-
- if column_exists(cr, "ir_filters", "sort"):
- cr.execute(
- """
- WITH to_update AS (
- SELECT f.id,
- COALESCE(ARRAY_TO_JSON(ARRAY_AGG(s.sort_item ORDER BY rn) filter (WHERE s.sort_item not in %s)), '[]') AS sort
- FROM ir_filters f
- JOIN LATERAL JSONB_ARRAY_ELEMENTS_TEXT(f.sort::jsonb)
- WITH ORDINALITY AS s(sort_item, rn)
- ON true
- WHERE f.model_id = %s
- AND f.sort ~ %s
- GROUP BY id
- )
- UPDATE ir_filters f
- SET sort = t.sort
- FROM to_update t
- WHERE f.id = t.id
- """,
- [(field, field + " desc"), model, r"\y{}\y".format(field)],
- )
-
-
-def _remove_field_from_context(context, fieldname):
- def filter_value(key, value):
- if key == "orderedBy" and isinstance(value, dict):
- res = {k: (filter_value(None, v) if k == "name" else v) for k, v in value.items()}
- # return if name didn't match fieldname
- return res if "name" not in res or res["name"] is not None else None
- if not isinstance(value, basestring):
- # if not a string, ignore it
- return value
- if value.split(":")[0] != fieldname:
- # only return if not matching fieldname
- return value
- return None # value filtered out
-
- if not isinstance(context, dict):
- return False
-
- changed = False
- for key in _CONTEXT_KEYS_TO_CLEAN:
- if context.get(key):
- context_part = [filter_value(key, e) for e in context[key]]
- changed |= context_part != context[key]
- context[key] = [e for e in context_part if e is not None]
-
- for vt in ["pivot", "graph", "cohort"]:
- key = "{}_measure".format(vt)
- if key in context:
- new_value = filter_value(key, context[key])
- changed |= context[key] != new_value
- context[key] = new_value if new_value is not None else "id"
-
- if vt in context:
- changed |= _remove_field_from_context(context[vt], fieldname)
-
- return changed
-
-
-def remove_field(cr, model, fieldname, cascade=False, drop_column=True, skip_inherit=(), keep_as_attachments=False):
- """
- Remove a field and its references from the database.
-
- This function also removes the field from inheriting models, unless exceptions are
- specified in `skip_inherit`. When the field is stored we can choose to not drop the
- column.
-
- :param str model: model name of the field to remove
- :param str fieldname: name of the field to remove
- :param bool cascade: whether the field column(s) are removed in `CASCADE` mode
- :param bool drop_column: whether the field's column is dropped
- :param list(str) or str skip_inherit: list of inheriting models to skip the removal
- of the field, use `"*"` to skip all
- :param bool keep_as_attachments: for binary fields, whether the data should be kept
- as attachments
- """
- _validate_model(model)
-
- ENVIRON["__renamed_fields"][model][fieldname] = None
-
- # clean dashboard's contexts
- for id_, action in _dashboard_actions(cr, r"\y{}\y".format(fieldname), model):
- context = safe_eval(action.get("context", "{}"))
- changed = _remove_field_from_context(context, fieldname)
- action.set("context", unicode(context))
- if changed:
- add_to_migration_reports(
- ("ir.ui.view.custom", id_, action.get("string", "ir.ui.view.custom")), "Filters/Dashboards"
- )
-
- _remove_field_from_filters(cr, model, fieldname)
-
- _remove_import_export_paths(cr, model, fieldname)
-
- def adapter(leaf, is_or, negated):
- # replace by TRUE_LEAF, unless negated or in a OR operation but not negated
- if is_or ^ negated:
- return [FALSE_LEAF]
- return [TRUE_LEAF]
-
- related = None
- if column_exists(cr, "ir_model_fields", "related"):
- cr.execute("SELECT related FROM ir_model_fields WHERE model=%s AND name=%s", [model, fieldname])
- if cr.rowcount:
- related = cr.fetchone()[0]
-
- if related:
- update_field_usage(cr, model, fieldname, related, skip_inherit=skip_inherit)
- else:
- # clean domains
- adapt_domains(cr, model, fieldname, "ignored", adapter=adapter, skip_inherit=skip_inherit, force_adapt=True)
-
- if table_exists(cr, "ir_server_object_lines"):
- cr.execute(
- """
- DELETE FROM ir_server_object_lines
- WHERE col1 IN (SELECT id
- FROM ir_model_fields
- WHERE model = %s
- AND name = %s)
- """,
- [model, fieldname],
- )
-
- # update tracking values
- if column_exists(cr, "mail_tracking_value", "field_info"):
- cr.execute(
- """
- SELECT id, field_description, name, ttype
- FROM ir_model_fields
- WHERE model=%s
- AND name=%s
- """,
- (model, fieldname),
- )
- if cr.rowcount:
- (field_id, field_desc_w_translation, name, ttype) = cr.fetchone()
- field_desc = field_desc_w_translation.get("en_US", next(iter(field_desc_w_translation.values())))
- fields_info = {
- "desc": field_desc,
- "name": name,
- "type": ttype,
- }
- cr.execute(
- """
- UPDATE mail_tracking_value
- SET field_info = %s
- WHERE field_id = %s
- """,
- (psycopg2.extras.Json(fields_info), field_id),
- )
-
- # remove this field from dependencies of other fields
- if column_exists(cr, "ir_model_fields", "depends"):
- cr.execute(
- "SELECT id,model,depends,COALESCE(compute,''),name FROM ir_model_fields WHERE state='manual' AND depends ~ %s",
- [r"\m{}\M".format(fieldname)],
- )
- for id, from_model, deps, compute_code, dep_field_name in cr.fetchall():
- parts = []
- for part in deps.split(","):
- path = part.strip().split(".")
- if not any(
- path[i] == fieldname and _valid_path_to(cr, path[:i], from_model, model) for i in range(len(path))
- ):
- parts.append(part)
- elif fieldname in compute_code:
- _logger.warning(
- "Field %s.%s depends on removed field %s.%s and its compute code references it, "
- "this may lead to errors.",
- from_model,
- dep_field_name,
- model,
- fieldname,
- )
-
- if len(parts) != len(deps.split(",")):
- cr.execute("UPDATE ir_model_fields SET depends=%s WHERE id=%s", [", ".join(parts) or None, id])
-
- # drop m2m table if needed
- if drop_column and column_exists(cr, "ir_model_fields", "relation_table"): # appears in version 9.0
- # verify that there aren't any other m2m pointing to the relation table
- cr.execute(
- """
- SELECT relation_table
- FROM ir_model_fields
- WHERE relation_table = (
- SELECT relation_table
- FROM ir_model_fields
- WHERE model = %s
- AND name = %s
- AND ttype = 'many2many'
- )
- AND ttype = 'many2many'
- GROUP BY relation_table
- HAVING count(*) = 1
- """,
- [model, fieldname],
- )
- for (m2m_rel,) in cr.fetchall():
- cr.execute('DROP TABLE IF EXISTS "{}" CASCADE'.format(m2m_rel))
- cr.execute(
- "DELETE FROM ir_model_relation r USING ir_model m WHERE m.id = r.model AND r.name = %s", [m2m_rel]
- )
- ENVIRON.setdefault("_gone_m2m", {})[m2m_rel] = "%s:%s" % (model, fieldname)
-
- # cleanup translations
- if table_exists(cr, "ir_translation"):
- cr.execute(
- """
- DELETE FROM ir_translation
- WHERE name=%s
- AND type in ('field', 'help', 'model', 'model_terms', 'selection') -- ignore wizard_* translations
- """,
- ["%s,%s" % (model, fieldname)],
- )
-
- # remove default values set for aliases
- if column_exists(cr, "mail_alias", "alias_defaults"):
- cr.execute(
- """
- SELECT a.id, a.alias_defaults
- FROM mail_alias a
- JOIN ir_model m ON m.id = a.alias_model_id
- WHERE m.model = %s
- AND a.alias_defaults ~ %s
- """,
- [model, r"\y%s\y" % (fieldname,)],
- )
- for alias_id, defaults_s in cr.fetchall():
- try:
- defaults = dict(literal_eval(defaults_s))
- except Exception:
- continue
- defaults.pop(fieldname, None)
- cr.execute("UPDATE mail_alias SET alias_defaults = %s WHERE id = %s", [repr(defaults), alias_id])
-
- table = table_of_model(cr, model)
- # NOTE table_exists is needed to avoid altering views
- stored = table_exists(cr, table) and column_exists(cr, table, fieldname)
-
- if keep_as_attachments:
- if stored and column_type(cr, table, fieldname) == "bytea":
- _extract_data_as_attachment(cr, model, fieldname, set_res_field=False)
- if column_exists(cr, "ir_attachment", "res_field"):
- query = cr.mogrify(
- "UPDATE ir_attachment SET res_field = NULL WHERE res_model = %s AND res_field = %s", [model, fieldname]
- ).decode()
- explode_execute(cr, query, table="ir_attachment")
-
- elif column_exists(cr, "ir_attachment", "res_field"):
- parallel_execute(
- cr,
- explode_query_range(
- cr,
- cr.mogrify(
- "DELETE FROM ir_attachment WHERE res_model = %s AND res_field = %s", [model, fieldname]
- ).decode(),
- table="ir_attachment",
- ),
- )
-
- table = table_of_model(cr, model)
- if drop_column and stored:
- remove_column(cr, table, fieldname, cascade=cascade)
-
- # relation_field_id is a FK with ON DELETE CASCADE
- if column_exists(cr, "ir_model_fields", "relation_field_id"):
- cr.execute(
- """
- SELECT d.model,
- d.name
- FROM ir_model_fields d
- JOIN ir_model_fields f
- ON d.relation_field_id = f.id
- WHERE f.model = %s
- AND f.name = %s
- AND d.ttype = 'one2many'
- AND d.state = 'manual'
- """,
- [model, fieldname],
- )
- for rel_model, rel_field in cr.fetchall():
- _logger.info("Cascade removing one2many field %s.%s", rel_model, rel_field)
- remove_field(cr, rel_model, rel_field)
-
- # remove the ir.model.fields entry (and its xmlid)
- cr.execute(
- """
- WITH del AS (
- DELETE FROM ir_model_fields WHERE model=%s AND name=%s RETURNING id
- )
- DELETE FROM ir_model_data
- USING del
- WHERE model = 'ir.model.fields'
- AND res_id = del.id
- """,
- [model, fieldname],
- )
-
- # remove field on inherits
- for inh in for_each_inherit(cr, model, skip_inherit):
- remove_field(
- cr,
- inh.model,
- fieldname,
- cascade=cascade,
- drop_column=drop_column,
- skip_inherit=skip_inherit,
- keep_as_attachments=keep_as_attachments,
- )
-
-
-def make_field_non_stored(cr, model, field, selectable=False):
- """
- Convert field to non-stored.
-
- :param str model: model name of the field to convert
- :param str fieldname: name of the field to convert
- :param bool selectable: whether the field is still selectable, if True custom `ir.filters` are not updated
- """
- _validate_model(model)
-
- remove_column(cr, table_of_model(cr, model), field)
- if not selectable:
- _remove_field_from_filters(cr, model, field)
-
-
-def remove_field_metadata(cr, model, fieldname, skip_inherit=()):
- """
- Remove metadata of a field.
-
- Due to a bug of the ORM [1], mixins doesn't create/register xmlids for fields created in children models
- Thus, when a field is no more defined in a child model, their xmlids should be removed explicitly to
- avoid the fields to be considered as missing and being removed at the end of the upgrade.
-
- [1] https://github.com/odoo/odoo/issues/49354
-
- :meta private: exclude from online docs
- """
- _validate_model(model)
-
- cr.execute(
- """
- DELETE FROM ir_model_data
- WHERE model = 'ir.model.fields'
- AND res_id IN (SELECT id FROM ir_model_fields WHERE model=%s AND name=%s)
- """,
- [model, fieldname],
- )
- for inh in for_each_inherit(cr, model, skip_inherit):
- remove_field_metadata(cr, inh.model, fieldname, skip_inherit=skip_inherit)
-
-
-def move_field_to_module(cr, model, fieldname, old_module, new_module, skip_inherit=()):
- """
- Move a field from one module to another.
-
- This functions updates all references to a specific field, switching from the source
- module to the destination module. It avoid data removal after the registry is fully
- loaded. The field in inheriting models are also moved unless skipped.
-
- :param str model: name of the owner model of the field to move
- :param str fieldname: name of the field to move
- :param str old_module: source module name from which the field is moved
- :param str new_module: target module name into which the field is moved
- :param list(str) or str skip_inherit: list of inheriting models for which the field is
- not to be moved, use `"*"` to skip all
- """
- _validate_model(model)
- name = IMD_FIELD_PATTERN % (model.replace(".", "_"), fieldname)
- columns = get_columns(cr, "ir_model_data", ignore=["id", "module"])
- query = format_query(
- cr,
- """
- INSERT INTO ir_model_data({0}, module)
- SELECT {0}, %s
- FROM ir_model_data
- WHERE model = 'ir.model.fields'
- AND name = %s
- AND module = %s
- ON CONFLICT DO NOTHING
- """,
- columns,
- )
- cr.execute(query, [new_module, name, old_module])
- cr.execute(
- "DELETE FROM ir_model_data WHERE model = 'ir.model.fields' AND name = %s AND module = %s",
- [name, old_module],
- )
- # move field on inherits
- for inh in for_each_inherit(cr, model, skip_inherit):
- move_field_to_module(cr, inh.model, fieldname, old_module, new_module, skip_inherit=skip_inherit)
-
-
-def rename_field(cr, model, old, new, update_references=True, domain_adapter=None, skip_inherit=()):
- """
- Rename a field and its references from `old` to `new` on the given `model`.
-
- The field is updated in all inheriting models, except for models specified in
- `skip_inherit`.
-
- This functions also updates references, indirect or direct ones, including filters,
- server actions, related fields, emails, dashboards, domains, and many more. See
- :func:`~odoo.upgrade.util.fields.update_field_usage`
-
- For the update of domains a special adapter function can be used. The default adapter
- just replaces `old` by `new` in each domain leaf. Refer to
- :func:`~odoo.upgrade.util.domains.adapt_domains` for information about domain adapters.
-
- :param str model: model name of the field to rename
- :param str old: current name of the field to rename
- :param str new: new name of the field to rename
- :param bool update_references: whether to update all references
- :param function domain_adapter: adapter to use for domains, see
- :func:`~odoo.upgrade.util.domains.adapt_domains`
- :param list(str) or str skip_inherit: models to skip when renaming the field in
- inheriting models, use `"*"` to skip all
- """
- _validate_model(model)
-
- rf = ENVIRON["__renamed_fields"][model]
- rf[new] = rf.pop(old, old)
-
- if update_references:
- # skip all inherit, they will be handled by the recursive call
- update_field_usage(cr, model, old, new, domain_adapter=domain_adapter, skip_inherit="*")
-
- # search for an existing custom field.
- cr.execute("SELECT 1 FROM ir_model_fields WHERE model = %s AND name IN %s", [model, (old, new)])
- if cr.rowcount == 2:
- # If a field with the same name already exists for this model (e.g. due to a custom module),
- # rename it to avoid clashing and warn the customer
- custom_name = new + "_custom"
- rename_field(cr, model, new, custom_name, update_references, domain_adapter=None, skip_inherit=skip_inherit)
- msg = (
- "The field %r from the model %r is now a standard Odoo field, but it already existed in the database "
- "(coming from a non-standard module) and thus has been renamed to %r" % (new, model, custom_name)
- )
- add_to_migration_reports(msg, "Non-standard fields")
-
- cr.execute("UPDATE ir_model_fields SET name=%s WHERE model=%s AND name=%s RETURNING id", (new, model, old))
- [fid] = cr.fetchone() or [None]
-
- if fid:
- # recreate the xmlids
- name = IMD_FIELD_PATTERN % (model.replace(".", "_"), new)
-
- cr.execute(
- """
- DELETE FROM ir_model_data
- WHERE model = 'ir.model.fields'
- AND res_id = %s
- AND module != '__export__'
- RETURNING module
- """,
- [fid],
- )
- if cr.rowcount:
- modules = {mod for (mod,) in cr.fetchall()}
-
- # match noupdate value with implementation which changed with commit
- # https://github.com/odoo/odoo/commit/a99e04dfeeef11a781b35574dd625e4007ab5654
- noupdate = False if version_gte("saas~11.5") else None
-
- cr.execute(
- """
- INSERT INTO ir_model_data(module, name, model, res_id, noupdate)
- SELECT unnest(%s), %s, 'ir.model.fields', %s, %s
- ON CONFLICT DO NOTHING
- RETURNING module
- """,
- [list(modules), name, fid, noupdate],
- )
- conflicted = modules - {mod for (mod,) in cr.fetchall()}
- if conflicted:
- # Duplicated key. May happen due to some_model.sub_id and some_model_sub.id
- # both leading to some_model_sub_id xmlid before saas~11.2 where the pattern
- # was fixed to be some_module__sub_id and some_model_sub__id respectively.
- on_conflict_name = "{}_{}".format(name, fid)
- cr.execute(
- """
- INSERT INTO ir_model_data(module, name, model, res_id, noupdate)
- SELECT unnest(%s), %s, 'ir.model.fields', %s, %s
- """,
- [list(conflicted), on_conflict_name, fid, noupdate],
- )
-
- if table_exists(cr, "ir_property"):
- cr.execute("UPDATE ir_property SET name=%s WHERE fields_id=%s", [new, fid])
-
- # Updates custom field relation name to match the renamed standard field during upgrade.
- cr.execute(
- """
- UPDATE ir_model_fields
- SET relation_field = %s
- WHERE relation = %s
- AND relation_field = %s
- AND state = 'manual'
- """,
- [new, model, old],
- )
-
- if table_exists(cr, "ir_translation"):
- cr.execute(
- """
- UPDATE ir_translation
- SET name=%s
- WHERE name=%s
- AND type in ('field', 'help', 'model', 'model_terms', 'selection') -- ignore wizard_* translations
- """,
- ["%s,%s" % (model, new), "%s,%s" % (model, old)],
- )
-
- if column_exists(cr, "ir_attachment", "res_field"):
- cr.execute(
- """
- UPDATE ir_attachment
- SET res_field = %s
- WHERE res_model = %s
- AND res_field = %s
- """,
- [new, model, old],
- )
-
- if table_exists(cr, "ir_values"):
- cr.execute(
- """
- UPDATE ir_values
- SET name = %s
- WHERE model = %s
- AND name = %s
- AND key = 'default'
- """,
- [new, model, old],
- )
-
- if column_type(cr, "mail_tracking_value", "field") == "varchar":
- # From saas~13.1, column `field` is a m2o to the `ir.model.fields`
- cr.execute(
- """
- UPDATE mail_tracking_value v
- SET field = %s
- FROM mail_message m
- WHERE v.mail_message_id = m.id
- AND m.model = %s
- AND v.field = %s
- """,
- [new, model, old],
- )
-
- table = table_of_model(cr, model)
- # NOTE table_exists is needed to avoid altering views
- if table_exists(cr, table) and column_exists(cr, table, old):
- cr.execute('ALTER TABLE "{0}" RENAME COLUMN "{1}" TO "{2}"'.format(table, old, new))
- # Rename corresponding index
- new_index_name = make_index_name(table, new)
- old_index_name = make_index_name(table, old)
- cr.execute('ALTER INDEX IF EXISTS "{0}" RENAME TO "{1}"'.format(old_index_name, new_index_name))
-
- # rename field on inherits
- for inh in for_each_inherit(cr, model, skip_inherit):
- rename_field(cr, inh.model, old, new, update_references=update_references, skip_inherit=skip_inherit)
-
-
-def invert_boolean_field(cr, model, old, new, skip_inherit=()):
- """Rename a boolean field and invert its value."""
- _validate_model(model)
- table = table_of_model(cr, model)
- if column_type(cr, table, old) != "bool":
- raise ValueError("Field {!r} of model {!r} is not a stored boolean field".format(old, model))
-
- def _adapter(leaf, _or, _neg):
- return ["!", leaf]
-
- if old == new:
- # same name but inverted value, just adapt domains
- adapt_domains(cr, model, old, new, adapter=_adapter, skip_inherit=skip_inherit, force_adapt=True)
- else:
- rename_field(cr, model, old, new, update_references=True, domain_adapter=_adapter, skip_inherit=skip_inherit)
-
- query = format_query(cr, "UPDATE {t} SET {c} = {c} IS NOT TRUE", t=table, c=new)
- explode_execute(cr, query, table=table)
-
-
-def convert_field_to_html(cr, model, field, skip_inherit=()):
- _validate_model(model)
- table = table_of_model(cr, model)
- if column_exists(cr, table, field): # only exists for list-based inherits (not for dict-based)
- jsonb_column = column_type(cr, table, field) == "jsonb"
- if jsonb_column:
- query = """
- WITH html_values AS (
- SELECT id, jsonb_object_agg(t.key, {2}) AS value
- FROM "{0}", jsonb_each_text("{1}") AS t
- WHERE {{parallel_filter}}
- GROUP BY id
- )
- UPDATE "{0}"
- SET "{1}" = h.value
- FROM html_values AS h
- WHERE "{0}".id = h.id
- """.format(table, field, pg_text2html("t.value"))
- else:
- query = 'UPDATE "{0}" SET "{1}" = {2} WHERE "{1}" IS NOT NULL'.format(table, field, pg_text2html(field))
- parallel_execute(cr, explode_query_range(cr, query, table=table))
-
- # Update translations
- if table_exists(cr, "ir_translation"):
- wrap = None if version_gte("saas~11.5") else "p"
- ttype = "model_terms" if version_gte("saas~11.5") else "model"
- cr.execute(
- r"""
- UPDATE ir_translation
- SET src = {}, value = {}, type = %s
- WHERE type = 'model'
- AND name = %s
- """.format(pg_text2html("src", wrap=wrap), pg_text2html("value", wrap=wrap)),
- [ttype, "%s,%s" % (model, field)],
- )
- cr.execute(
- """
- UPDATE ir_model_fields AS imf
- SET ttype = 'html'
- FROM ir_model_fields AS f
- WHERE imf.related_field_id = f.id
- AND f.name = %s
- AND f.model = %s
- AND imf.state = 'manual'
- """,
- [field, model],
- )
-
- # Update ir.default
- if table_exists(cr, "ir_default"):
- json_value_html = pg_text2html("json_value::jsonb->>0")
- query = """
- UPDATE ir_default AS d
- SET json_value = to_json({})::text
- FROM ir_model_fields AS imf
- WHERE imf.name = %s
- AND imf.model = %s
- AND imf.id = d.field_id
- """
- cr.execute(format_query(cr, query, sql.SQL(json_value_html)), [field, model])
- for inh in for_each_inherit(cr, model, skip_inherit):
- convert_field_to_html(cr, inh.model, field, skip_inherit=skip_inherit)
-
-
-def convert_m2o_field_to_m2m(cr, model, field, new_name=None, m2m_table=None, col1=None, col2=None):
- """
- Convert a Many2one field to a Many2many.
-
- It creates the relation table and fill it.
-
- :param str model: model name of the field to convert
- :param str field: current name of the field to convert
- :param str new_name: new name of the field to convert.
- If not provide, it's `field` with a trailing "s".
- :param str m2m_table: name of the relation table. Automatically generated if not provided.
- :param str col1: name of the column referencing `model`.
- :param str col2: name of the column referencing the target of `field`.
- """
- _validate_model(model)
- if new_name is None:
- if not field.endswith("_id"):
- raise ValueError("Please specify the new name explicitly")
- new_name = field + "s"
-
- table1 = table_of_model(cr, model)
- table2, _, _ = target_of(cr, table1, field)
-
- if col1 is None:
- col1 = "{}_id".format(table1)
- if col2 is None:
- col2 = "{}_id".format(table2)
-
- m2m_table = create_m2m(cr, m2m_table or AUTO, table1, table2, col1, col2)
-
- dedup = SQLStr("ON CONFLICT DO NOTHING")
- if cr._cnx.server_version < 90500:
- # Approximate version for older PG versions
- dedup = format_query(
- cr,
- """
- EXCEPT
- SELECT {col1}, {col2}
- FROM {m2m}
- """,
- m2m=m2m_table,
- col1=col1,
- col2=col2,
- )
-
- fill_query = format_query(
- cr,
- """
- INSERT INTO {m2m}({col1}, {col2})
- SELECT id, {field}
- FROM {table}
- WHERE {field} IS NOT NULL
- {dedup}
- """,
- m2m=m2m_table,
- col1=col1,
- col2=col2,
- field=field,
- table=table1,
- dedup=dedup,
- )
- cr.execute(fill_query)
- remove_column(cr, table1, field)
- rename_field(cr, model, field, new_name)
-
-
-# Because we can.
-m2o2m2m = convert_m2o_field_to_m2m
-
-
-def _convert_field_to_company_dependent(
- cr, model, field, type, target_model=None, default_value=None, default_value_ref=None, company_field="company_id"
-):
- _validate_model(model)
- if target_model:
- _validate_model(target_model)
-
- type2field = {"char", "float", "boolean", "integer", "text", "many2one", "date", "datetime", "selection", "html"}
- assert type in type2field
-
- table = table_of_model(cr, model)
-
- # update "ir_model_fields"."company_dependent" as an identifier for indirect reference
- cr.execute(
- """
- UPDATE ir_model_fields
- SET company_dependent = TRUE
- WHERE model = %s
- AND name = %s
- RETURNING id
- """,
- (model, field),
- )
- if not cr.rowcount:
- # no ir_model_fields, no column
- remove_column(cr, table, field, cascade=True)
- return
- [field_id] = cr.fetchone()
-
- if default_value is None: # noqa: SIM108
- where_condition = "{0} IS NOT NULL"
- else:
- where_condition = cr.mogrify("{0} != %s", [default_value]).decode()
-
- if company_field:
- where_condition += format_query(cr, " AND {} IS NOT NULL", company_field)
- using = format_query(cr, "jsonb_build_object({}, {{0}})", company_field)
- else:
- using = format_query(
- cr,
- "(SELECT jsonb_object_agg(c.id::text, {}.{{0}}) FROM res_company c)",
- table,
- )
- alter_column_type(cr, table, field, "jsonb", using=using, where=where_condition)
-
- # delete all old default
- cr.execute("DELETE FROM ir_default WHERE field_id = %s", [field_id])
-
- # add fallback
- if default_value:
- cr.execute(
- "INSERT INTO ir_default(field_id, json_value) VALUES (%s, %s) RETURNING id",
- (field_id, json.dumps(default_value)),
- )
- [default_id] = cr.fetchone()
- if default_value_ref:
- module, _, xid = default_value_ref.partition(".")
- cr.execute(
- """
- INSERT INTO ir_model_data(module, name, model, res_id, noupdate)
- VALUES (%s, %s, 'ir.default', %s, True)
- """,
- [module, xid, default_id],
- )
-
-
-def _convert_field_to_property(
- cr, model, field, type, target_model=None, default_value=None, default_value_ref=None, company_field="company_id"
-):
- """
- Convert a field to a property field.
-
- Notes
- -----
- `target_model` is only use when `type` is "many2one".
- The `company_field` can be an sql expression.
- You may use `t` to refer the model's table.
-
- :meta private: exclude from online docs
-
- """
- _validate_model(model)
- if target_model:
- _validate_model(target_model)
- type2field = {
- "char": "value_text",
- "float": "value_float",
- "boolean": "value_integer",
- "integer": "value_integer",
- "text": "value_text",
- "binary": "value_binary",
- "many2one": "value_reference",
- "date": "value_datetime",
- "datetime": "value_datetime",
- "selection": "value_text",
- }
-
- assert type in type2field
- value_field = type2field[type]
-
- table = table_of_model(cr, model)
-
- cr.execute("SELECT id FROM ir_model_fields WHERE model=%s AND name=%s", (model, field))
- if not cr.rowcount:
- # no ir_model_fields, no ir_property
- remove_column(cr, table, field, cascade=True)
- return
- [fields_id] = cr.fetchone()
-
- if default_value is None:
- where_clause = "{field} IS NOT NULL".format(field=field)
- else:
- where_clause = "{field} != %(default_value)s".format(field=field)
-
- if type == "boolean":
- value_select = "%s::integer" % field
- elif type != "many2one":
- value_select = field
- else:
- # for m2o, the store value is a reference field, so in format `model,id`
- value_select = "CONCAT('{target_model},', {field})".format(**locals())
-
- if is_field_anonymized(cr, model, field):
- # if field is anonymized, we need to create a property for each record
- where_clause = "true"
- # and we need to unanonymize its values
- ano_default_value = cr.mogrify("%s", [default_value])
- if type != "many2one": # noqa: SIM108
- ano_value_select = "%(value)s"
- else:
- ano_value_select = "CONCAT('{0},', %(value)s)".format(target_model)
-
- register_unanonymization_query(
- cr,
- model,
- field,
- """
- UPDATE ir_property
- SET {value_field} = CASE WHEN %(value)s IS NULL THEN {ano_default_value}
- ELSE {ano_value_select} END
- WHERE res_id = CONCAT('{model},', %(id)s)
- AND name='{field}'
- AND type='{type}'
- AND fields_id={fields_id}
- """.format(**locals()),
- )
-
- cr.execute(
- """
- WITH cte AS (
- SELECT CONCAT('{model},', id) as res_id, {value_select} as value,
- ({company_field})::integer as company
- FROM {table} t
- WHERE {where_clause}
- )
- INSERT INTO ir_property(name, type, fields_id, company_id, res_id, {value_field})
- SELECT %(field)s, %(type)s, %(fields_id)s, cte.company, cte.res_id, cte.value
- FROM cte
- WHERE NOT EXISTS(SELECT 1
- FROM ir_property
- WHERE fields_id=%(fields_id)s
- AND company_id IS NOT DISTINCT FROM cte.company
- AND res_id=cte.res_id)
- """.format(**locals()),
- locals(),
- )
- # default property
- if default_value:
- cr.execute(
- """
- INSERT INTO ir_property(name, type, fields_id, {value_field})
- VALUES (%s, %s, %s, %s)
- RETURNING id
- """.format(value_field=value_field),
- (field, type, fields_id, default_value),
- )
- [prop_id] = cr.fetchone()
- if default_value_ref:
- module, _, xid = default_value_ref.partition(".")
- cr.execute(
- """
- INSERT INTO ir_model_data(module, name, model, res_id, noupdate)
- VALUES (%s, %s, %s, %s, %s)
- """,
- [module, xid, "ir.property", prop_id, True],
- )
-
- remove_column(cr, table, field, cascade=True)
-
-
-# alias with a name related to the new API to declare property fields (company_dependent=True attribute)
-if version_gte("saas~17.5"):
- make_field_company_dependent = _convert_field_to_company_dependent
-else:
- make_field_company_dependent = _convert_field_to_property
-
-# retro-compatibility
-convert_field_to_property = make_field_company_dependent
-
-
-def convert_binary_field_to_attachment(cr, model, field, encoded=True, name_field=None):
- _extract_data_as_attachment(cr, model, field, encoded, name_field)
- # free PG space
- table = table_of_model(cr, model)
- remove_column(cr, table, field)
-
-
-def _extract_data_as_attachment(cr, model, field, encoded=True, name_field=None, set_res_field=True):
- _validate_model(model)
- table = table_of_model(cr, model)
- if not column_exists(cr, table, field):
- return
- name_query = cr.mogrify(
- format_query(cr, "COALESCE({}, CONCAT(%s, '(', id, ').', %s))", name_field if name_field else SQLStr("NULL")),
- [model.title().replace(".", ""), field],
- ).decode()
-
- if not column_exists(cr, "ir_attachment", "res_field"):
- res_field_query = SQLStr("")
- elif set_res_field:
- res_field_query = SQLStr(cr.mogrify(", res_field = %s", [field]).decode())
- else:
- res_field_query = SQLStr(", res_field = NULL")
-
- update_query = format_query(
- cr,
- """
- UPDATE ir_attachment
- SET res_model = %s,
- res_id = %s
- {}
- WHERE id = %s
- """,
- res_field_query,
- )
-
- cr.execute(format_query(cr, "SELECT count(*) FROM {} WHERE {} IS NOT NULL", table, field))
- [count] = cr.fetchone()
- if count == 0:
- return
-
- A = env(cr)["ir.attachment"]
- iter_cur = cr._cnx.cursor("fetch_binary")
- iter_cur.itersize = 1
- iter_cur.execute(
- format_query(
- cr,
- "SELECT id, {field}, {name_query} FROM {table} WHERE {field} IS NOT NULL",
- field=field,
- name_query=sql.SQL(name_query),
- table=table,
- )
- )
- logger = _logger.getChild("convert_binary_field_to_attachment")
- for rid, data, name in log_progress(iter_cur, logger=logger, qualifier="rows", size=count):
- # we can't save create the attachment with res_model & res_id as it will fail computing
- # `res_name` field for non-loaded models. Store it naked and change it via SQL after.
- data = bytes(data) # noqa: PLW2901
- if re.match(b"^\\d+ (bytes|[KMG]b)$", data, re.I):
- # badly saved data, no need to create an attachment.
- continue
- if not encoded:
- data = base64.b64encode(data) # noqa: PLW2901
- att = A.create({"name": name, "datas": data, "type": "binary"})
-
- cr.execute(update_query, [model, rid, att.id])
- invalidate(att)
-
- iter_cur.close()
-
-
-if version_gte("16.0"):
-
- def convert_field_to_translatable(cr, model, field):
- table = table_of_model(cr, model)
- ctype = column_type(cr, table, field)
- if not ctype or ctype == "jsonb":
- return
- alter_column_type(cr, table, field, "jsonb", "jsonb_build_object('en_US', {0})")
-
- def convert_field_to_untranslatable(cr, model, field, type="varchar"):
- assert type in ("varchar", "text")
- table = table_of_model(cr, model)
- ctype = column_type(cr, table, field)
- if not ctype or ctype != "jsonb":
- return
- alter_column_type(cr, table, field, type, "{0}->>'en_US'")
-
-else:
- # Older versions, these functions are no-op
- def convert_field_to_translatable(cr, model, field):
- pass
-
- def convert_field_to_untranslatable(cr, model, field, type="varchar"):
- pass
-
-
-def change_field_selection_values(cr, model, field, mapping, skip_inherit=()):
- """
- Replace references of values of a selection field.
-
- This function replaces all references to selection values according to a mapping.
- Domains are also updated.
-
- :param str model: model name of the selection field to update
- :param str field: name of the selection field to update
- :param dict mapping: selection values to update, key values are replaced by
- their corresponding values in the mapping
- :param list(str) or str skip_inherit: list of inheriting models to skip in the update
- of the selection values, use `"*"` to skip all
- """
- _validate_model(model)
- if not mapping:
- return
- table = table_of_model(cr, model)
-
- if column_exists(cr, table, field):
- ctype = column_type(cr, table, field)
- if ctype == "varchar":
- query = "UPDATE {table} t SET {column} = %(json)s::jsonb->>t.{column} WHERE t.{column} = ANY(%(keys)s)"
- elif ctype == "jsonb":
- # company dependent selection field
- query = """
- WITH upd AS (
- SELECT t.id,
- jsonb_object_agg(v.key, COALESCE(%(json)s::jsonb->>v.value, v.value)) AS value
- FROM {table} t
- JOIN LATERAL jsonb_each_text(t.{column}) v
- ON true
- WHERE jsonb_path_query_array(t.{column}, '$.*') ?| %(keys)s
- AND {{parallel_filter}}
- GROUP BY t.id
- )
- UPDATE {table} t
- SET {column} = upd.value
- FROM upd
- WHERE upd.id = t.id
- """
- else:
- raise UpgradeError("unsupported column type for selection field: {}".format(ctype))
-
- data = {
- "keys": list(mapping),
- "json": json.dumps(mapping),
- }
- queries = [
- cr.mogrify(q, data).decode()
- for q in explode_query_range(cr, format_query(cr, query, table=table, column=field), table=table, alias="t")
- ]
- parallel_execute(cr, queries)
-
- elif table_exists(cr, "ir_property"):
- cr.execute("SELECT id FROM ir_model_fields WHERE model=%s AND name=%s", [model, field])
- if cr.rowcount:
- fields_id = cr.fetchone()[0]
- query = """
- UPDATE ir_property
- SET value_text = %(json)s::jsonb->>value_text
- WHERE value_text IN %(keys)s
- AND fields_id = %(fields_id)s
- """
- data = {
- "keys": tuple(mapping),
- "json": json.dumps(mapping),
- "fields_id": fields_id,
- }
- queries = [cr.mogrify(q, data).decode() for q in explode_query_range(cr, query, table="ir_property")]
- parallel_execute(cr, queries)
-
- if table_exists(cr, "ir_model_fields_selection"):
- cr.execute(
- """
- DELETE FROM ir_model_fields_selection s
- USING ir_model_fields f
- WHERE f.id = s.field_id
- AND f.model = %s
- AND f.name = %s
- AND s.value = ANY(%s)
- """,
- [model, field, [k for k in mapping if k not in mapping.values()]],
- )
-
- if table_exists(cr, "ir_default"):
- query = """
- UPDATE ir_default d
- SET json_value = (%(json)s::jsonb->>d.json_value)
- FROM ir_model_fields f
- WHERE d.field_id = f.id
- AND f.model = %(model)s
- AND f.name = %(name)s
- AND d.json_value IN %(keys)s
- """
- dumped_map = {json.dumps(k): json.dumps(v) for k, v in mapping.items()}
- else:
- query = """
- UPDATE ir_values
- SET value = %(json)s::jsonb->>value
- WHERE model = %(model)s
- AND name = %(name)s
- AND key = 'default'
- AND value IN %(keys)s
- """
- dumped_map = {pickle.dumps(k): pickle.dumps(v) for k, v in mapping.items()}
-
- data = {
- "keys": tuple(dumped_map),
- "json": json.dumps(dumped_map),
- "model": model,
- "name": field,
- }
- cr.execute(query, data)
-
- def adapter(leaf, _or, _neg):
- left, op, right = leaf
- if isinstance(right, (tuple, list)): # noqa: SIM108
- right = [mapping.get(r, r) for r in right]
- else:
- right = mapping.get(right, right)
- return [(left, op, right)]
-
- # skip all inherit, they will be handled by the recursive call
- adapt_domains(cr, model, field, field, adapter=adapter, skip_inherit="*")
-
- # rename field on inherits
- for inh in for_each_inherit(cr, model, skip_inherit):
- change_field_selection_values(cr, inh.model, field, mapping=mapping, skip_inherit=skip_inherit)
-
-
-def is_field_anonymized(cr, model, field):
- from .modules import module_installed # noqa: PLC0415
-
- _validate_model(model)
- if not module_installed(cr, "anonymization"):
- return False
- cr.execute(
- """
- SELECT id
- FROM ir_model_fields_anonymization
- WHERE model_name = %s
- AND field_name = %s
- AND state = 'anonymized'
- """,
- [model, field],
- )
- return bool(cr.rowcount)
-
-
-def register_unanonymization_query(cr, model, field, query, query_type="sql", sequence=10):
- _validate_model(model)
- cr.execute(
- """
- INSERT INTO ir_model_fields_anonymization_migration_fix(
- target_version, sequence, query_type, model_name, field_name, query
- ) VALUES (%s, %s, %s, %s, %s, %s)
- """,
- [release.major_version, sequence, query_type, model, field, query],
- )
-
-
-def update_field_usage(cr, model, old, new, domain_adapter=None, skip_inherit=()):
- """
- Replace all references to the field `old` by `new` in different places.
-
- Search in:
- - ir_filters
- - ir_exports_line
- - ir_act_server
- - mail_alias
- - ir_ui_view_custom (dashboard)
- - domains (using `domain_adapter`)
- - related fields
-
- This function can be used to replace the usage of a field by another. Domains are
- updated using the `domain_adapter`. By default the domain adapter just replaces `old`
- by `new` in domain leaves. See :func:`~odoo.upgrade.util.domains.adapt_domains` for
- more information about domain adapters.
-
- :param str model: model name of the field
- :param str old: source name of the field to replace
- :param str new: target name of the field to set
- :param function domain_adapter: adapter to use for domains, see
- :func:`~odoo.upgrade.util.domains.adapt_domains`
- :param list(str) or str skip_inherit: models to skip when renaming the field in
- inheriting models, use `"*"` to skip all
- """
- return _update_field_usage_multi(cr, [model], old, new, domain_adapter=domain_adapter, skip_inherit=skip_inherit)
-
-
-def update_field_references(cr, old, new, only_models=None, domain_adapter=None, skip_inherit=()):
- warnings.warn(
- "The `update_field_references` function is deprecated. Use `update_field_usage` instead.",
- DeprecationWarning,
- stacklevel=1,
- )
- models = "*" if only_models is None else only_models
- return _update_field_usage_multi(cr, models, old, new, domain_adapter=domain_adapter, skip_inherit=skip_inherit)
-
-
-def _update_impex_renamed_fields_paths(cr, old_field_name, new_field_name, only_models):
- export_q = cr.mogrify(
- """
- SELECT el.id,
- e.resource,
- STRING_TO_ARRAY(el.name, '/')
- FROM ir_exports_line el
- JOIN ir_exports e
- ON el.export_id = e.id
- WHERE el.name ~ %s
- """,
- [r"\y{}\y".format(old_field_name)],
- ).decode()
- impex_data = [(export_q, "ir_exports_line", "name")]
- if table_exists(cr, "base_import_mapping"):
- import_q = cr.mogrify(
- """
- SELECT id,
- res_model,
- STRING_TO_ARRAY(field_name, '/')
- FROM base_import_mapping
- WHERE field_name ~ %s
- """,
- [r"\y{}\y".format(old_field_name)],
- ).decode()
- impex_data.append((import_q, "base_import_mapping", "field_name"))
-
- for query, table, column in impex_data:
- cr.execute(query)
- if not cr.rowcount:
- continue
- fixed_paths = {}
- for record_id, related_model, path in cr.fetchall():
- has_id = path[-1] == ".id"
- if has_id:
- path = path[:-1] # noqa: PLW2901
-
- new_path = [
- new_field_name
- if field.field_name == old_field_name and field.field_model in only_models
- else field.field_name
- for field in resolve_model_fields_path(cr, related_model, path)
- ]
- if len(new_path) == len(path) and new_path != path:
- if has_id:
- new_path.append(".id")
- fixed_paths[record_id] = "/".join(new_path)
- if fixed_paths:
- cr.execute(
- format_query(cr, "UPDATE {} SET {} = (%s::jsonb)->>(id::text) WHERE id IN %s", table, column),
- [Json(fixed_paths), tuple(fixed_paths)],
- )
-
-
-def _update_field_usage_multi(cr, models, old, new, domain_adapter=None, skip_inherit=()):
- assert models
- only_models = None if models == "*" else tuple(models)
-
- if only_models:
- for model in only_models:
- _validate_model(model)
-
- p = {
- "old": PGRegexp(r"\y{}\y".format(re.escape(old))),
- "old_pattern": PGRegexp(r"""[.'"]{0}\y""".format(re.escape(old))),
- "new": new,
- "def_old": PGRegexp(r"\ydefault_{}\y".format(re.escape(old))),
- "def_new": "default_{}".format(new),
- "models": tuple(only_models) if only_models else (),
- }
-
- # ir.action.server
- if column_exists(cr, "ir_act_server", "update_path") and only_models:
- cr.execute(
- """
- SELECT a.id,
- a.update_path,
- m.model
- FROM ir_act_server a
- JOIN ir_model m
- ON a.model_id = m.id
- WHERE a.state = 'object_write'
- AND a.update_path ~ %(old)s
- """,
- p,
- )
- to_update = {}
- for act_id, update_path, src_model in cr.fetchall():
- for dst_model in only_models:
- new_path = _replace_path(cr, old, new, src_model, dst_model, update_path)
- if new_path != update_path:
- to_update[act_id] = new_path
- if to_update:
- cr.execute(
- "UPDATE ir_act_server SET update_path = %s::jsonb->>id::text WHERE id IN %s",
- [Json(to_update), tuple(to_update)],
- )
-
- # Modifying server action is dangerous.
- # The search pattern can be anywhere in the code, leading to invalid codes.
- # Moreover, limiting to some models will ignore some SA that should be modified.
- # Just search for the potential SA that need update.
- col_prefix = ""
- if not column_exists(cr, "ir_act_server", "condition"):
- col_prefix = "--" # sql comment the line
- standard_modules = set(modules.get_modules()) | {"__upgrade__"}
- q = """
- SELECT a.id, a.{name}
- FROM ir_act_server a
- LEFT JOIN ir_model_data d
- ON d.res_id = a.id
- AND d.model = 'ir.actions.server'
- WHERE a.state = 'code'
- AND (a.code ~ %(old_pattern)s
- {col_prefix} OR a.condition ~ %(old)s
- )
- AND ( d.module IS NULL -- custom action
- OR d.module NOT IN %(standard_modules)s
- OR (d.module IN %(standard_modules)s AND d.noupdate)
- )
- """.format(col_prefix=col_prefix, name=get_value_or_en_translation(cr, "ir_act_server", "name"))
- cr.execute(q, {"old_pattern": p["old_pattern"], "old": p["old"], "standard_modules": tuple(standard_modules)})
- li = ""
- if cr.rowcount:
- li = "".join(
- "{}".format(get_anchor_link_to_record("ir.actions.server", aid, aname))
- for aid, aname in cr.fetchall()
- )
-
- if column_exists(cr, "ir_model_fields", "compute"):
- # Modifying compute methods is similarly as dangerous.
- # Only report potential compute methods that may need an update.
- q = """
- SELECT f.id, f.name
- FROM ir_model_fields f
- LEFT JOIN ir_model_data d
- ON d.res_id = f.id
- AND d.model = 'ir.model.fields'
- WHERE f.compute ~ %(old_pattern)s
- AND ( d.module IS NULL -- custom action
- OR d.module NOT IN %(standard_modules)s
- OR (d.module IN %(standard_modules)s AND d.noupdate)
- )
- """
- cr.execute(q, {"old_pattern": p["old_pattern"], "standard_modules": tuple(standard_modules)})
- if cr.rowcount:
- li += "".join(
- "{}".format(get_anchor_link_to_record("ir.model.fields", fid, fname))
- for fid, fname in cr.fetchall()
- )
- if li:
- model_text = "All models"
- if only_models:
- model_text = "Models " + ", ".join("{}".format(m) for m in only_models)
- add_to_migration_reports(
- """
-
-
- {model_text}: the field {old} has been renamed to {new}. The following server actions and compute methods of other fields may need an update:
- If a server action or a field is a standard one and you haven't made any modifications, you may ignore them.
-
-
-
- """.format(**locals()),
- category="Fields renamed",
- format="html",
- )
-
- # if we stay on the same model. (no usage of dotted-path) (only works for domains and related)
- if "." not in old and "." not in new:
- # ir.filters
- col_prefix = ""
- if not column_exists(cr, "ir_filters", "sort"):
- col_prefix = "--" # sql comment the line
-
- q = format_query(
- cr,
- """
- UPDATE ir_filters
- SET {col_prefix} sort = {sort_repl},
- context = {context_repl}
- WHERE {cond}
- AND (
- context ~ %(old)s
- OR context ~ %(def_old)s
- {col_prefix} OR sort ~ %(old)s
- )
- """,
- col_prefix=SQLStr(col_prefix),
- sort_repl=pg_replace("sort", [(p["old"], p["new"])]),
- context_repl=pg_replace("context", [(p["old"], p["new"]), (p["def_old"], p["def_new"])]),
- cond=SQLStr("model_id IN %(models)s") if only_models else SQLStr("true"),
- )
- cr.execute(q, p)
-
- # ir.exports.line, base_import.mapping # noqa
- if only_models:
- _update_impex_renamed_fields_paths(cr, old, new, only_models)
-
- # mail.alias
- if column_exists(cr, "mail_alias", "alias_defaults"):
- q = """
- UPDATE mail_alias a
- SET alias_defaults = regexp_replace(a.alias_defaults, %(old)s, %(new)s, 'g')
- """
- if only_models:
- q += """
- FROM ir_model m
- WHERE m.id = a.alias_model_id
- AND m.model IN %(models)s
- AND
- """
- else:
- q += "WHERE "
- q += "a.alias_defaults ~ %(old)s"
- cr.execute(q, p)
-
- # ir.ui.view.custom
- # adapt the context. The domain will be done by `adapt_domain`
- def_old = "default_{}".format(old)
- def_new = "default_{}".format(new)
- match = "{0[old]}|{0[def_old]}".format(p)
-
- def adapt_value(key, value):
- if key == "orderedBy" and isinstance(value, dict):
- # only adapt the "name" key
- return {k: (adapt_value(None, v) if k == "name" else v) for k, v in value.items()}
-
- if not isinstance(value, basestring):
- # ignore if not a string
- return value
-
- parts = value.split(":", 1)
- if parts[0] != old:
- # if not match old, leave it
- return value
- # change to new, and return it
- parts[0] = new
- return ":".join(parts)
-
- def adapt_dict(d):
- # adapt (in place) dictionary values
- if not isinstance(d, dict):
- return
-
- for key in _CONTEXT_KEYS_TO_CLEAN:
- if d.get(key):
- d[key] = [adapt_value(key, e) for e in d[key]]
-
- for vt in ["pivot", "graph", "cohort"]:
- key = "{}_measure".format(vt)
- if key in d:
- d[key] = adapt_value(key, d[key])
-
- if vt in d:
- adapt_dict(d[vt])
-
- for _, act in _dashboard_actions(cr, match, *only_models or ()):
- context = safe_eval(act.get("context", "{}"))
- adapt_dict(context)
-
- if def_old in context:
- context[def_new] = context.pop(def_old)
- act.set("context", unicode(context))
-
- # domains, related and inhited models
- if only_models:
- for model in only_models:
- # skip all inherit, they will be handled by the recursive call
- adapt_domains(cr, model, old, new, adapter=domain_adapter, skip_inherit="*", force_adapt=True)
- adapt_related(cr, model, old, new, skip_inherit="*")
- adapt_depends(cr, model, old, new, skip_inherit="*")
-
- inherited_models = tuple(
- inh.model for model in only_models for inh in for_each_inherit(cr, model, skip_inherit)
- )
- if inherited_models:
- _update_field_usage_multi(
- cr, inherited_models, old, new, domain_adapter=domain_adapter, skip_inherit=skip_inherit
- )
-
-
-def adapt_depends(cr, model, old, new, skip_inherit=()):
- # adapt depends for custom compute fields only. Standard fields will be updated by the ORM.
- _validate_model(model)
-
- if not column_exists(cr, "ir_model_fields", "depends"):
- # this field only appears in 9.0
- return
-
- target_model = model
-
- match_old = r"\y{}\y".format(re.escape(old))
- cr.execute(
- """
- SELECT id, model, depends
- FROM ir_model_fields
- WHERE state = 'manual'
- AND depends ~ %s
- """,
- [match_old],
- )
- for id_, field_model, depends in cr.fetchall():
- temp_depends = depends.split(",")
- for i in range(len(temp_depends)):
- domain = _adapt_one_domain(
- cr, target_model, old, new, field_model, [(temp_depends[i], "=", "depends")], force_adapt=True
- )
- if domain:
- temp_depends[i] = domain[0][0]
- new_depends = ",".join(temp_depends)
- if new_depends != depends:
- cr.execute("UPDATE ir_model_fields SET depends = %s WHERE id = %s", [new_depends, id_])
-
- # down on inherits
- for inh in for_each_inherit(cr, target_model, skip_inherit):
- adapt_depends(cr, inh.model, old, new, skip_inherit=skip_inherit)
-
-
-def adapt_related(cr, model, old, new, skip_inherit=()):
- _validate_model(model)
-
- if not column_exists(cr, "ir_model_fields", "related"):
- # this field only appears in 9.0
- return
-
- target_model = model
-
- match_old = r"\y{}\y".format(re.escape(old))
- cr.execute(
- """
- SELECT id, model, related
- FROM ir_model_fields
- WHERE related ~ %s
- """,
- [match_old],
- )
- for id_, field_model, related in cr.fetchall():
- domain = _adapt_one_domain(
- cr, target_model, old, new, field_model, [(related, "=", "related")], force_adapt=True
- )
- if domain:
- cr.execute("UPDATE ir_model_fields SET related = %s WHERE id = %s", [domain[0][0], id_])
-
- # TODO adapt paths in email templates?
-
- # down on inherits
- for inh in for_each_inherit(cr, target_model, skip_inherit):
- adapt_related(cr, inh.model, old, new, skip_inherit=skip_inherit)
-
-
-def update_server_actions_fields(cr, src_model, dst_model=None, fields_mapping=None):
- """
- Update server action fields.
-
- When some fields of `src_model` have ben copied to `dst_model` and/or have
- been copied to fields with another name, some references have to be moved.
-
- For server actions, this function starts by updating `ir_server_object_lines`
- to refer to new fields. If no `fields_mapping` is provided, all references to
- fields that exist in both models (source and destination) are moved. If no `dst_model`
- is given, the `src_model` is used as `dst_model`.
- Then, if `dst_model` is set, `ir_act_server` referred by modified `ir_server_object_lines`
- are also updated. A chatter message informs the customer about this modification.
-
- :meta private: exclude from online docs
- """
- if dst_model is None and fields_mapping is None:
- raise SleepyDeveloperError(
- "at least dst_model or fields_mapping must be given to the move_field_references function."
- )
-
- _dst_model = dst_model if dst_model is not None else src_model
-
- # update ir_server_object_lines to point to new fields
- if fields_mapping is None:
- cr.execute(
- """
- WITH field_ids AS (
- SELECT mf1.id as old_field_id, mf2.id as new_field_id
- FROM ir_model_fields mf1
- JOIN ir_model_fields mf2 ON mf2.name = mf1.name
- WHERE mf1.model = %s
- AND mf2.model = %s
- )
- UPDATE ir_server_object_lines
- SET col1 = f.new_field_id
- FROM field_ids f
- WHERE col1 = f.old_field_id
- RETURNING server_id
- """,
- [src_model, _dst_model],
- )
- else:
- psycopg2.extras.execute_values(
- cr._obj,
- """
- WITH field_ids AS (
- SELECT mf1.id as old_field_id, mf2.id as new_field_id
- FROM (VALUES %s) AS mapping(src_model, dst_model, old_name, new_name)
- JOIN ir_model_fields mf1 ON mf1.name = mapping.old_name AND mf1.model = mapping.src_model
- JOIN ir_model_fields mf2 ON mf2.name = mapping.new_name AND mf2.model = mapping.dst_model
- )
- UPDATE ir_server_object_lines
- SET col1 = f.new_field_id
- FROM field_ids f
- WHERE col1 = f.old_field_id
- RETURNING server_id
- """,
- [(src_model, _dst_model, fm[0], fm[1]) for fm in fields_mapping],
- )
-
- # update ir_act_server records to point to the right model if set
- if dst_model is not None and src_model != dst_model and cr.rowcount > 0:
- action_ids = tuple({row[0] for row in cr.fetchall()}) # uniquify ids
-
- cr.execute(
- """
- UPDATE ir_act_server
- SET model_name = %s, model_id = ir_model.id
- FROM ir_model
- WHERE ir_model.model = %s
- AND ir_act_server.id IN %s
- RETURNING ir_act_server.name
- """,
- [dst_model, dst_model, action_ids],
- )
-
- action_names = [row[0] for row in cr.fetchall()]
-
- # inform the customer through the chatter about this modification
- msg = (
- "The following server actions have been updated due to moving "
- "fields from '%(src_model)s' to '%(dst_model)s' model and need "
- "a checking from your side: %(actions)s"
- ) % {"src_model": src_model, "dst_model": dst_model, "actions": ", ".join(action_names)}
-
- add_to_migration_reports(message=msg, category="Server Actions")
diff --git a/src/util/helpers.py b/src/util/helpers.py
deleted file mode 100644
index a34c6b9c3..000000000
--- a/src/util/helpers.py
+++ /dev/null
@@ -1,321 +0,0 @@
-# -*- coding: utf-8 -*-
-import logging
-import os
-import sys
-from collections import namedtuple
-
-import lxml
-
-from .exceptions import SleepyDeveloperError
-from .misc import splitlines, version_between, version_gte
-
-_logger = logging.getLogger(__name__.rpartition(".")[0])
-
-_VALID_MODELS = frozenset(
- {
- "_unknown",
- "website_pricelist",
- "ir_actions_account_report_download",
- # see test_testing_utilities/models.py
- "o2m_readonly_subfield_parent",
- "o2m_readonly_subfield_child",
- "o2m_changes_parent",
- "o2m_changes_children",
- }
- | ({"l10n_pl_tax_office"} if version_between("16.0", "17.0") else set())
- | {m.strip() for m in os.getenv("UPG_VALID_MODELS", "").split(";")} - {""}
-)
-
-# python3 shims
-try:
- unicode # noqa: B018
-except NameError:
- unicode = str
-
-
-def table_of_model(cr, model):
- exceptions = dict(
- line.split()
- for line in splitlines(
- """
- ir.actions.actions ir_actions
- ir.actions.act_url ir_act_url
- ir.actions.act_window ir_act_window
- ir.actions.act_window_close ir_actions
- ir.actions.act_window.view ir_act_window_view
- ir.actions.client ir_act_client
- ir.actions.report.xml ir_act_report_xml
- ir.actions.report ir_act_report_xml
- ir.actions.server ir_act_server
- ir.actions.wizard ir_act_wizard
-
- stock.picking.in stock_picking
- stock.picking.out stock_picking
-
- workflow wkf
- workflow.activity wkf_activity
- workflow.instance wkf_instance
- workflow.transition wkf_transition
- workflow.triggers wkf_triggers
- workflow.workitem wkf_workitem
-
- # mass_mailing
- mail.mass_mailing.list_contact_rel mail_mass_mailing_contact_list_rel
- mailing.contact.subscription mailing_contact_list_rel
-
- # `mail.notification` was a "normal" model in versions <9.0
- # and a named m2m in >=saas~13
- # and renamed as a "normal" model table in >= saas~14.3
- {gte_saas13_lte_saas14_3} mail.notification mail_message_res_partner_needaction_rel
-
- project.task.stage.personal project_task_user_rel
- """.format(gte_saas13_lte_saas14_3="" if version_gte("9.saas~13") and not version_gte("saas~14.3") else "#")
- )
- )
- return exceptions.get(model, model.replace(".", "_"))
-
-
-def model_of_table(cr, table):
- exceptions = dict(
- line.split()
- for line in splitlines(
- """
- # can also be act_window_close, but there are chances it wont be useful for anyone...
- ir_actions ir.actions.actions
- ir_act_url ir.actions.act_url
- ir_act_window ir.actions.act_window
- ir_act_window_view ir.actions.act_window.view
- ir_act_client ir.actions.client
- ir_act_report_xml {action_report_model}
- ir_act_server ir.actions.server
- ir_act_wizard ir.actions.wizard
-
- wkf workflow
- wkf_activity workflow.activity
- wkf_instance workflow.instance
- wkf_transition workflow.transition
- wkf_triggers workflow.triggers
- wkf_workitem workflow.workitem
-
- _unknown _unknown
-
- mail_mass_mailing_contact_list_rel mail.mass_mailing.list_contact_rel
- mailing_contact_list_rel mailing.contact.subscription
- # Not a real model until saas~13
- {gte_saas13_lte_saas14_3} mail_message_res_partner_needaction_rel mail.notification
-
- project_task_user_rel project.task.stage.personal
- """.format(
- action_report_model="ir.actions.report" if version_gte("10.saas~17") else "ir.actions.report.xml",
- gte_saas13_lte_saas14_3="" if version_gte("9.saas~13") and not version_gte("saas~14.3") else "#",
- )
- )
- )
- try:
- return exceptions[table]
- except KeyError:
- cr.execute(
- """
- SELECT model
- FROM ir_model
- WHERE replace(model, '.', '_') = %s
- """,
- [table],
- )
- candidates = [m for (m,) in cr.fetchall()]
- if candidates:
- if len(candidates) > 1:
- _logger.critical("cannot determine model of table %r. Multiple candidates: %r", table, candidates)
- return candidates[0]
-
- fallback = table.replace("_", ".")
- _logger.critical(
- "cannot determine model of table %r. No candidates found in the `ir_model` table. Fallback to %r",
- table,
- fallback,
- )
- return fallback
-
-
-def _validate_model(model):
- if "_" in model and "." not in model and not model.startswith("x_") and model not in _VALID_MODELS:
- raise SleepyDeveloperError("`{}` seems to be a table name instead of model name".format(model))
- return model
-
-
-def _validate_table(table):
- if "." in table:
- raise SleepyDeveloperError("`{}` seems to be a model name instead of table name".format(table))
- return table
-
-
-def _ir_values_value(cr, prefix=None):
- # returns the casting from bytea to text needed in saas~17 for column `value` of `ir_values`
- # returns tuple(column_read, cast_write)
- cache = getattr(_ir_values_value, "cache", None)
-
- if cache is None:
- from .pg import column_type # noqa: PLC0415
-
- if column_type(cr, "ir_values", "value") == "bytea":
- cr.execute("SELECT character_set_name FROM information_schema.character_sets")
- (charset,) = cr.fetchone()
- column_read = "convert_from(%%svalue, '%s')" % charset
- cast_write = "convert_to(%%s, '%s')" % charset
- else:
- column_read = "%svalue"
- cast_write = "%s"
- _ir_values_value.cache = (column_read, cast_write)
- else:
- column_read, cast_write = cache
-
- prefix = prefix + "." if prefix else ""
- return column_read % prefix, cast_write
-
-
-def _dashboard_actions(cr, arch_match, *models):
- """Yield (dashboard_id, action) of dashboards that match `arch_match` and apply on `models` (if specified)."""
- q = """
- SELECT id, arch
- FROM ir_ui_view_custom
- WHERE arch ~ %s
- """
- cr.execute(q, [arch_match])
- for dash_id, arch in cr.fetchall():
- try:
- if isinstance(arch, unicode):
- arch = arch.encode("utf-8") # noqa: PLW2901
- dash = lxml.etree.fromstring(arch)
- except lxml.etree.XMLSyntaxError:
- _logger.exception("Cannot parse dashboard %s", dash_id)
- continue
- for act in dash.xpath("//action"):
- if models:
- try:
- act_id = int(act.get("name", "FAIL"))
- except ValueError:
- continue
-
- cr.execute("SELECT res_model FROM ir_act_window WHERE id = %s", [act_id])
- [act_model] = cr.fetchone() or [None]
- if act_model not in models:
- continue
- yield dash_id, act
-
- cr.execute(
- "UPDATE ir_ui_view_custom SET arch = %s WHERE id = %s",
- [lxml.etree.tostring(dash, encoding="unicode"), dash_id],
- )
-
-
-def _get_theme_models():
- return {
- "theme.ir.ui.view": "ir.ui.view",
- "theme.ir.asset": "ir.asset",
- "theme.website.page": "website.page",
- "theme.website.menu": "website.menu",
- "theme.ir.attachment": "ir.attachment",
- }
-
-
-FieldsPathPart = namedtuple("FieldsPathPart", "field_model field_name relation_model")
-if sys.version_info[0] >= 3:
- FieldsPathPart.__doc__ = """
-Encapsulate information about a field within a fields path.
-
-:param str field_model: model of the field
-:param str field_name: name of the field
-:param str relation_model: target model of the field, if relational, otherwise ``None``
-"""
- for _f in FieldsPathPart._fields:
- getattr(FieldsPathPart, _f).__doc__ = None
-
-
-def resolve_model_fields_path(cr, model, path):
- """
- Resolve model fields paths.
-
- This function returns a list of :class:`~odoo.upgrade.util.helpers.FieldsPathPart`
- where each item describes a field in ``path`` (in the same order). The returned list
- could be shorter than the original ``path`` due to a missing field or model, or
- because there is a non-relational field in the path. The only non-relational field
- allowed in a fields path is the last one, in which case the returned list has the same
- length as the input ``path``.
-
- .. example::
-
- .. code-block:: python
-
- >>> util.resolve_model_fields_path(cr, "res.partner", "user_ids.partner_id.title".split("."))
- [FieldsPathPart(field_model='res.partner', field_name='user_ids', relation_model='res.users'),
- FieldsPathPart(field_model='res.users', field_name='partner_id', relation_model='res.partner'),
- FieldsPathPart(field_model='res.partner', field_name='title', relation_model='res.partner.title')]
-
- Last field is not relational:
-
- .. code-block:: python
-
- >>> resolve_model_fields_path(cr, "res.partner", "user_ids.active".split("."))
- [FieldsPathPart(field_model='res.partner', field_name='user_ids', relation_model='res.users'),
- FieldsPathPart(field_model='res.users', field_name='active', relation_model=None)]
-
- The path is wrong, it uses a non-relational field:
-
- .. code-block:: python
-
- >>> resolve_model_fields_path(cr, "res.partner", "user_ids.active.name".split("."))
- [FieldsPathPart(field_model='res.partner', field_name='user_ids', relation_model='res.users'),
- FieldsPathPart(field_model='res.users', field_name='active', relation_model=None)]
-
- The path is broken, it uses a non-existing field:
-
- .. code-block:: python
-
- >>> resolve_model_fields_path(cr, "res.partner", "user_ids.non_existing_id.active".split("."))
- [FieldsPathPart(field_model='res.partner', field_name='user_ids', relation_model='res.users')]
-
- :param str model: starting model of the fields path
- :param typing.Sequence[str] path: fields path
- :return: resolved fields path parts
- :rtype: list(:class:`~odoo.upgrade.util.helpers.FieldsPathPart`)
- """
- if not path:
- return []
- path = list(path)
- cr.execute(
- """
- WITH RECURSIVE resolved_fields_path AS (
- -- non-recursive term
- SELECT imf.model AS field_model,
- imf.name AS field_name,
- imf.relation AS relation_model,
- p.path AS path,
- 1 AS part_index
- FROM (VALUES (%(model)s, %(path)s)) p(model, path)
- JOIN ir_model_fields imf
- ON imf.model = p.model
- AND imf.name = p.path[1]
-
- UNION ALL
-
- -- recursive term
- SELECT rimf.model AS field_model,
- rimf.name AS field_name,
- rimf.relation AS relation_model,
- rfp.path AS path,
- rfp.part_index + 1 AS part_index
- FROM resolved_fields_path rfp
- JOIN ir_model_fields rimf
- ON rimf.model = rfp.relation_model
- AND rimf.name = rfp.path[rfp.part_index + 1]
- WHERE cardinality(rfp.path) > rfp.part_index
- )
- SELECT field_model,
- field_name,
- relation_model
- FROM resolved_fields_path
- ORDER BY part_index
- """,
- {"model": model, "path": list(path)},
- )
- return [FieldsPathPart(**row) for row in cr.dictfetchall()]
diff --git a/src/util/hr_payroll.py b/src/util/hr_payroll.py
deleted file mode 100644
index 57fce3aad..000000000
--- a/src/util/hr_payroll.py
+++ /dev/null
@@ -1,51 +0,0 @@
-import logging
-
-from .fields import remove_field
-from .misc import version_between
-from .records import delete_unused, ref
-
-_logger = logging.getLogger(__name__)
-
-
-def _remove_salary_rule(cr, xmlid):
- rid = ref(cr, xmlid)
- cr.execute(
- r"""
- SELECT f.name
- FROM ir_model_fields f,
- hr_salary_rule r
- JOIN hr_payroll_structure s
- ON r.struct_id = s.id
- LEFT JOIN res_country c
- ON s.country_id = c.id
- WHERE r.id = %s
- AND f.model = 'hr.payroll.report'
- AND f.name = regexp_replace(
- concat_ws(
- '_',
- 'x_l10n',
- COALESCE(lower(c.code), 'xx'),
- lower(r.code)
- ),
- '[\.\- ]',
- '_'
- )
- """,
- [rid],
- )
- for (fname,) in cr.fetchall():
- _logger.info(
- "Removing field %r from model 'hr.payroll.report' since salary rule %r is being removed",
- fname,
- xmlid,
- )
- remove_field(cr, "hr.payroll.report", fname)
- delete_unused(cr, xmlid)
-
-
-if not version_between("16.0", "saas~18.4"):
-
- def remove_salary_rule(cr, xmlid):
- delete_unused(cr, xmlid)
-else:
- remove_salary_rule = _remove_salary_rule
diff --git a/src/util/inconsistencies.py b/src/util/inconsistencies.py
deleted file mode 100644
index 2387d4bcc..000000000
--- a/src/util/inconsistencies.py
+++ /dev/null
@@ -1,473 +0,0 @@
-# -*- coding: utf-8 -*-
-import logging
-import os
-from textwrap import dedent
-
-from psycopg2.extensions import quote_ident
-from psycopg2.extras import Json
-from psycopg2.sql import SQL
-
-from .helpers import _validate_model, table_of_model
-from .misc import Sentinel, chunks, str2bool
-from .pg import format_query, get_value_or_en_translation, target_of
-from .report import add_to_migration_reports, get_anchor_link_to_record, html_escape
-
-_logger = logging.getLogger(__name__)
-
-INCLUDE_ARCHIVED_PRODUCTS = str2bool(
- os.environ.get("ODOO_MIG_DO_NOT_IGNORE_ARCHIVED_PRODUCTS_FOR_UOM_INCONSISTENCIES"),
- default=False,
-)
-FIX_PRODUCT_UOM = str2bool(os.environ.get("ODOO_MIG_FIX_ALL_UOM_INCONSISTENCIES"), default=False)
-
-FROM_ENV = Sentinel("FROM_ENV")
-
-
-def break_recursive_loops(cr, model, field, name_field="name"):
- # TODO add a variant to verify loops on m2m
- _validate_model(model)
-
- table = table_of_model(cr, model)
- trgt = target_of(cr, table, field)
- if not trgt or trgt[:2] != (table, "id"):
- raise ValueError("The column `{}` is not FK on itself".format(field))
-
- query = format_query(
- cr,
- """
- WITH RECURSIVE __loop AS (
- SELECT array[{field}] AS path,
- False AS cycle
- FROM {table}
- WHERE {field} IS NOT NULL
- GROUP BY {field}
- UNION ALL
- SELECT child.{field} || curr.path AS path,
- child.{field} = any(curr.path) AS cycle
- FROM __loop AS curr
- JOIN {table} AS child
- ON child.id = curr.path[1]
- WHERE child.{field} IS NOT NULL
- AND NOT curr.cycle
- )
- SELECT path FROM __loop WHERE cycle
- """,
- table=table,
- field=field,
- )
- cr.execute(query)
- if not cr.rowcount:
- return
-
- ids = []
- done = set()
- for (cycle,) in cr.fetchall():
- to_break = min(cycle[: cycle.index(cycle[0], 1)])
- if to_break not in done:
- ids.append(to_break)
- done.update(cycle)
-
- update_query = format_query(
- cr,
- """
- UPDATE {table}
- SET {field} = NULL
- WHERE id IN %s
- RETURNING id, {name}
- """,
- table=table,
- field=field,
- name=SQL(get_value_or_en_translation(cr, table, name_field)),
- )
- cr.execute(update_query, [tuple(ids)])
- n_updates = cr.rowcount
- _logger.warning("%s records in %s got their %r column unset to break a recursive loop.", n_updates, table, field)
- N_UPDATES_IN_REPORT = 20
- bad_data = cr.fetchmany(N_UPDATES_IN_REPORT)
-
- query = format_query(
- cr,
- """
- SELECT m.{}, f.{}
- FROM ir_model_fields f
- JOIN ir_model m
- ON m.id = f.model_id
- WHERE m.model = %s
- AND f.name = %s
- """,
- SQL(get_value_or_en_translation(cr, "ir_model", "name")),
- SQL(get_value_or_en_translation(cr, "ir_model_fields", "field_description")),
- )
- cr.execute(query, [model, field])
- model_label, field_label = cr.fetchone()
-
- add_to_migration_reports(
- """
-
-
- The following {model} were found to be recursive. Their "{field}" field has been reset.{disclaimer}
-
-
-
- """.format(
- model=html_escape(model_label),
- field=html_escape(field_label),
- disclaimer=" Find below a list of the first {} (out of {}) affected records.".format(
- N_UPDATES_IN_REPORT, n_updates
- )
- if n_updates > N_UPDATES_IN_REPORT
- else "",
- li="".join("{}".format(get_anchor_link_to_record(model, id_, name)) for id_, name in bad_data),
- ),
- format="html",
- category="Inconsistencies",
- )
-
-
-def verify_companies(
- cr, model, field_name, logger=_logger, model_company_field="company_id", comodel_company_field="company_id"
-):
- _validate_model(model)
- cr.execute(
- """
- SELECT ttype, relation, relation_table, column1, column2
- FROM ir_model_fields
- WHERE name = %s
- AND model = %s
- AND store IS TRUE
- AND ttype IN ('many2one', 'many2many')
- """,
- [field_name, model],
- )
-
- field_values = cr.dictfetchone()
-
- if not field_values:
- _logger.warning("Field %s not found on model %s.", field_name, model)
- return
-
- table = table_of_model(cr, model)
- comodel = field_values["relation"]
- cotable = table_of_model(cr, comodel)
-
- limit = 15
-
- if field_values["ttype"] == "many2one":
- query = """
- SELECT a.id, a.{model_company_field}, b.id, b.{comodel_company_field}, count(*) OVER ()
- FROM {table} a
- JOIN {cotable} b ON b.id = a.{field_name}
- WHERE a.{model_company_field} IS NOT NULL
- AND b.{comodel_company_field} IS NOT NULL
- AND a.{model_company_field} != b.{comodel_company_field}
- LIMIT {limit}
- """.format(**locals())
- else: # many2many
- m2m_relation = field_values["relation_table"]
- f1, f2 = field_values["column1"], field_values["column2"]
- query = """
- SELECT a.id, a.{model_company_field}, b.id, b.{comodel_company_field}, count(*) OVER ()
- FROM {m2m_relation} m
- JOIN {table} a ON a.id = m.{f1}
- JOIN {cotable} b ON b.id = m.{f2}
- WHERE a.{model_company_field} IS NOT NULL
- AND b.{comodel_company_field} IS NOT NULL
- AND a.{model_company_field} != b.{comodel_company_field}
- LIMIT {limit}
- """.format(**locals())
-
- cr.execute(query)
- if cr.rowcount:
- logger.warning(
- "Company field %s/%s is not consistent with %s/%s for %d records (through %s relation %s)",
- model,
- model_company_field,
- comodel,
- comodel_company_field,
- cr.rowcount,
- field_values["ttype"],
- field_name,
- )
-
- bad_rows = cr.fetchall()
- total = bad_rows[-1][-1]
- lis = "\n".join("record #%s (company=%s) -> record #%s (company=%s)" % bad[:-1] for bad in bad_rows)
-
- add_to_migration_reports(
- message="""\
-
-
- Some inconsistencies have been found on field {model}/{field_name} ({total} records affected; show top {limit})
-
-
-
- """.format(**locals()),
- category="Multi-company inconsistencies",
- format="html",
- )
-
-
-def verify_uoms(
- cr,
- model,
- uom_field="product_uom_id",
- product_field="product_id",
- include_archived_products=FROM_ENV,
- auto_fix=FROM_ENV,
- ids=None,
-):
- """
- Check if the category of uom on `model` is the same as the category of uom on `product.template`.
-
- When `ids` is not provided, every ids would be verified.
-
- Returns list of ids if inconsistencies found, else []
- """
- _validate_model(model)
- table = table_of_model(cr, model)
-
- q = lambda s: quote_ident(s, cr._cnx)
-
- if include_archived_products is FROM_ENV:
- include_archived_products = INCLUDE_ARCHIVED_PRODUCTS
-
- if auto_fix is FROM_ENV:
- auto_fix = FIX_PRODUCT_UOM
-
- query = """
- SELECT t.id line_id,
- t.{uom_column} line_uom_id,
- tu.{uom_name} line_uom_name,
- tuc.{category_name} line_uom_categ_name,
- pt.uom_id product_uom_id,
- ptu.{uom_name} product_uom_name,
- pt.id product_template_id,
- pt.{product_template_name} product_template_name,
- ptuc.{category_name} product_uom_categ_name
- FROM {table} t
- JOIN uom_uom tu ON t.{uom_column} = tu.id
- JOIN uom_category tuc ON tu.category_id = tuc.id
- JOIN product_product pp ON t.{product_column} = pp.id
- JOIN product_template pt ON pp.product_tmpl_id = pt.id
- JOIN uom_uom ptu ON pt.uom_id = ptu.id
- JOIN uom_category ptuc ON ptu.category_id = ptuc.id
- WHERE tu.category_id != ptu.category_id
- {ids}
- {active}
- """.format(
- table=q(table),
- uom_column=q(uom_field),
- product_column=q(product_field),
- uom_name=get_value_or_en_translation(cr, "uom_uom", "name"),
- category_name=get_value_or_en_translation(cr, "uom_category", "name"),
- product_template_name=get_value_or_en_translation(cr, "product_template", "name"),
- ids=" AND t.id IN %s" if ids else "",
- active=" AND pp.active" if not include_archived_products else "",
- )
-
- rows = []
- if ids is None:
- cr.execute(query)
- rows = cr.fetchall()
- elif ids:
- ids_chunks = chunks(ids, size=cr.IN_MAX, fmt=tuple)
- for chunk in ids_chunks:
- cr.execute(query, [chunk])
- rows.extend(cr.fetchall())
-
- if not rows:
- return []
-
- title = model.replace(".", " ").title()
-
- if auto_fix:
- line_new_ids = {line_id: prod_uom_id for line_id, _, _, _, prod_uom_id, _, _, _, _ in rows}
- cr.execute(
- """
- UPDATE {table} t
- SET {uom_column} = (%s::jsonb->t.id::text)::int
- WHERE t.id IN %s
- """.format(
- table=q(table),
- uom_column=q(uom_field),
- ),
- [
- Json(line_new_ids),
- tuple(line_new_ids),
- ],
- )
-
- msg = dedent(
- """
- Upon your request, we have automatically fixed the faulty UoMs by picking it from
- the Product Template and setting it on the {title}s.
-
- Please, take the time to check that the following {title}s inconsistencies have
- been updated to the right UoM:\n\n{updated_uoms}
- """
- ).format(
- title=title,
- updated_uoms="\n".join(
- " * {}(id={}): Updated UoM from `{}`(id={}, category: `{}`) to `{}`(id={}, category: `{}`) for Product Template `{}`(id={})".format(
- title,
- line_id,
- line_uom,
- line_uom_id,
- line_uom_categ,
- prod_uom,
- prod_uom_id,
- prod_uom_categ,
- prod_temp,
- prod_temp_id,
- )
- for line_id, line_uom_id, line_uom, line_uom_categ, prod_uom_id, prod_uom, prod_temp_id, prod_temp, prod_uom_categ in rows
- ),
- )
- faulty_ids = []
-
- else:
- msg = """
- There is a UoM mismatch in some {title}s. The category of the UoM defined on the
- {title} is different from that defined on the Product Template and must be the same to
- avoid errors. We allowed the upgrade to continue, but these inconsistencies may cause error
- during the upgrade or issues on the upgraded database.
-
- To avoid any issue, here are the options to consider:
-
- * fix these inconsistencies manually (below, the details of the affected records)
- * let this script automatically fix the affected records by setting the environment variable
- ODOO_MIG_FIX_ALL_UOM_INCONSISTENCIES to 1. It will take the UoM from the Product Template
- and set it on the faulty {title}s.
-
- You can also take the archived products into account for listing or fixing faulty lines by setting the
- environment variable ODOO_MIG_DO_NOT_IGNORE_ARCHIVED_PRODUCTS_FOR_UOM_INCONSISTENCIES to 1
-
- These {title}s have UoM inconsistencies:\n\n{uom_inconsistencies}
- """.format(
- title=title,
- uom_inconsistencies="\n".join(
- " * {}(id={}) has UoM `{}`(id={}, category: `{}`), Product Template `{}`(id={}) has UoM `{}`(id={}, category: `{}`)".format(
- title,
- line_id,
- line_uom,
- line_uom_id,
- line_uom_categ,
- prod_temp,
- prod_temp_id,
- prod_uom,
- prod_uom_id,
- prod_uom_categ,
- )
- for line_id, line_uom_id, line_uom, line_uom_categ, prod_uom_id, prod_uom, prod_temp_id, prod_temp, prod_uom_categ in rows
- ),
- )
- faulty_ids = [r[0] for r in rows]
-
- _logger.warning("\n%s\n", msg)
- add_to_migration_reports(category=title + " UoM Inconsistencies", message=msg, format="md")
- return faulty_ids
-
-
-def verify_products(
- cr,
- model,
- foreign_model,
- foreign_model_reference_field,
- model_product_field="product_id",
- foreign_model_product_field="product_id",
- include_archived_products=FROM_ENV,
- ids=None,
-):
- """
- Check if the product on the `foreign_model` is the same as the product on the `model`.
-
- When `ids` is not provided, every ids would be verified.
-
- The `foreign_model` should be the one that have a reference to the `model` using this
- schema:
- >>> `foreign_model`.`foreign_reference_field` = `model`.id
-
- In case where the model/foreign model own a specific product field (different than `product_id`),
- you NEED to provide it using model_product_field/foreign_model_product_field
-
- As a function example, if you want to check if the product defined on the `account_move_line`
- is the same as the product defined on the `purchase_order_line` using `purchase_line_id`
- as reference, you should call this function in this way:
- >>> verify_products(cr, "purchase.order.line", "account.move.line", "purchase_line_id", ids=ids)
-
- Returns list of ids if inconsistencies found, else [].
- """
- _validate_model(model)
- _validate_model(foreign_model)
- table = table_of_model(cr, model)
- foreign_table = table_of_model(cr, foreign_model)
-
- q = lambda s: quote_ident(s, cr._cnx)
-
- if include_archived_products is FROM_ENV:
- include_archived_products = INCLUDE_ARCHIVED_PRODUCTS
-
- query = """
- SELECT f.id,
- f.{foreign_model_product_field},
- fpt.{name},
- t.id,
- t.{model_product_field},
- tpt.{name}
- FROM {table} t
- JOIN {foreign_table} f ON f.{foreign_model_reference_field} = t.id
- JOIN product_product tpp ON t.{model_product_field} = tpp.id
- JOIN product_template tpt ON tpp.product_tmpl_id = tpt.id
- JOIN product_product fpp ON f.{foreign_model_product_field} = fpp.id
- JOIN product_template fpt ON fpp.product_tmpl_id = fpt.id
- WHERE f.{foreign_model_product_field} != t.{model_product_field}
- {ids}
- {active}
- """.format(
- name=get_value_or_en_translation(cr, "product_template", "name"),
- table=q(table),
- foreign_table=q(foreign_table),
- foreign_model_reference_field=q(foreign_model_reference_field),
- model_product_field=q(model_product_field),
- foreign_model_product_field=q(foreign_model_product_field),
- ids=" AND t.id IN %s" if ids else "",
- active=" AND tpp.active" if not include_archived_products else "",
- )
-
- rows = []
- if ids is None:
- cr.execute(query)
- rows = cr.fetchall()
- elif ids:
- ids_chunks = chunks(ids, size=cr.IN_MAX, fmt=tuple)
- for chunk in ids_chunks:
- cr.execute(query, [chunk])
- rows.extend(cr.fetchall())
-
- if not rows:
- return []
-
- title = model.replace(".", " ").title()
- foreign_title = foreign_model.replace(".", " ").title()
- msg = """
- There is a product mismatch in some {foreign_title}. The product defined on the {foreign_title}
- is different from that defined on the {title}. To allow the upgrade to continue, the product
- on the {foreign_title} and on the {title} must be the same.
- These {foreign_title} have inconsistencies:
- """.format(**locals())
- msg += "\n".join(
- " * {}(id={}) has Product `{}`(id={}), {}(id={}) has Product `{}`(id={})".format(
- foreign_title, fline_id, fline_product, fline_product_id, title, line_id, line_product, line_product_id
- )
- for fline_id, fline_product_id, fline_product, line_id, line_product_id, line_product in rows
- )
-
- add_to_migration_reports(
- category=title + " - " + foreign_title + " Products Inconsistencies",
- message=msg,
- format="md",
- )
- _logger.warning("\n%s\n", msg)
- return [r[0] for r in rows]
diff --git a/src/util/indirect_references.py b/src/util/indirect_references.py
deleted file mode 100644
index 3be63bbac..000000000
--- a/src/util/indirect_references.py
+++ /dev/null
@@ -1,173 +0,0 @@
-# -*- coding: utf-8 -*-
-import collections
-
-from .helpers import model_of_table, table_of_model
-from .pg import SQLStr, column_exists, table_exists
-
-
-class IndirectReference(
- collections.namedtuple(
- "IndirectReference", "table res_model res_id res_model_id set_unknown company_dependent_comodel"
- )
-):
- def model_filter(self, prefix="", placeholder="%s"):
- if prefix and prefix[-1] != ".":
- prefix += "."
- if self.res_model_id:
- placeholder = "(SELECT id FROM ir_model WHERE model={})".format(placeholder)
- column = self.res_model_id
- else:
- column = self.res_model
-
- if column is None:
- # `model` is not set when `company_dependent_comodel` is.
- return SQLStr("(false AND {} IS NULL)".format(placeholder))
-
- return SQLStr('{}"{}"={}'.format(prefix, column, placeholder))
-
-
-# By default, there is no `res_id`, no `res_model_id` and it is deleted when the linked model is removed
-# warning: defaults are from the last fields in the namedtuple
-IndirectReference.__new__.__defaults__ = (None, None, False, None) # https://stackoverflow.com/a/18348004
-_IR = IndirectReference
-
-INDIRECT_REFERENCES = [
- _IR("ir_attachment", "res_model", "res_id"),
- _IR("ir_cron", "model", None, set_unknown=True),
- _IR("ir_act_report_xml", "model", None, set_unknown=True),
- _IR("ir_act_window", "res_model", "res_id"),
- _IR("ir_act_window", "res_model", None),
- _IR("ir_act_window", "src_model", None),
- _IR("ir_act_server", "wkf_model_name", None),
- _IR("ir_act_server", "crud_model_name", None),
- _IR("ir_act_server", "model_name", None, "model_id", set_unknown=True),
- _IR("ir_act_client", "res_model", None, set_unknown=True),
- _IR("ir_embedded_actions", "parent_res_model", "parent_res_id"),
- _IR("ir_model", "model", None),
- _IR("ir_model_fields", "model", None),
- _IR("ir_model_fields", "relation", None), # destination of a relation field
- _IR("ir_model_data", "model", "res_id"),
- _IR("ir_filters", "model_id", None, set_unknown=True), # YUCK!, not an id
- # duplicated for versions where the `res_id` column does not exists
- _IR("ir_filters", "model_id", "embedded_parent_res_id"),
- _IR("ir_exports", "resource", None),
- _IR("ir_ui_view", "model", None, set_unknown=True),
- _IR("ir_values", "model", "res_id"),
- _IR("wkf_transition", "trigger_model", None),
- _IR("wkf_triggers", "model", None),
- _IR("ir_model_fields_anonymization", "model_name", None),
- _IR("ir_model_fields_anonymization_migration_fix", "model_name", None),
- _IR("base_import_mapping", "res_model", None),
- _IR("calendar_event", "res_model", "res_id"), # new in saas~18
- _IR("data_cleaning_model", "res_model_name", None),
- _IR("data_cleaning_record", "res_model_name", "res_id"),
- _IR("data_cleaning_rule", "res_model_name", None),
- _IR("data_merge_group", "res_model_name", None),
- _IR("data_merge_model", "res_model_name", None),
- _IR("data_merge_record", "res_model_name", "res_id"),
- _IR("documents_document", "res_model", "res_id"),
- _IR("email_template", "model", None, set_unknown=True), # stored related
- _IR("iap_extracted_words", "res_model", "res_id"),
- _IR("mail_template", "model", None, set_unknown=True), # model renamed in saas~6
- _IR("mail_activity", "res_model", "res_id", "res_model_id"),
- _IR("mail_activity_type", "res_model", None),
- _IR("mail_alias", None, "alias_force_thread_id", "alias_model_id"),
- _IR("mail_alias", None, "alias_parent_thread_id", "alias_parent_model_id"),
- _IR("mail_followers", "res_model", "res_id"),
- _IR("mail_message_subtype", "res_model", None),
- _IR("mail_message", "model", "res_id"),
- _IR("mail_compose_message", "model", "res_id"),
- _IR("mail_wizard_invite", "res_model", "res_id"),
- _IR("mail_mail_statistics", "model", "res_id"),
- _IR("mailing_trace", "model", "res_id"),
- _IR("mail_mass_mailing", "mailing_model", None, "mailing_model_id", set_unknown=True),
- _IR("mailing_mailing", None, None, "mailing_model_id", set_unknown=True),
- _IR("marketing_campaign", "model_name", None, set_unknown=True), # stored related
- _IR("marketing_participant", "model_name", "res_id", "model_id", set_unknown=True),
- _IR("payment_transaction", None, "callback_res_id", "callback_model_id"),
- _IR("project_project", "alias_model", None, set_unknown=True),
- # IR("pos_blackbox_be_log", "model_name", None), # ACTUALLY NOT. We need to keep records intact, even when renaming a model # noqa: ERA001
- _IR("quality_point", "worksheet_model_name", None),
- _IR("rating_rating", "res_model", "res_id", "res_model_id"),
- _IR("rating_rating", "parent_res_model", "parent_res_id", "parent_res_model_id"),
- _IR("snailmail_letter", "model", "res_id", set_unknown=True),
- _IR("sms_template", "model", None),
- _IR("studio_approval_rule", "model_name", None),
- _IR("spreadsheet_revision", "res_model", "res_id"),
- _IR("studio_approval_entry", "model", "res_id"),
- _IR("timer_timer", "res_model", "res_id"),
- _IR("timer_timer", "parent_res_model", "parent_res_id"),
- _IR("worksheet_template", "res_model", None),
-]
-
-
-def indirect_references(cr, bound_only=False):
- for ir in INDIRECT_REFERENCES:
- if bound_only and not ir.res_id:
- continue
- if ir.res_id and not column_exists(cr, ir.table, ir.res_id):
- continue
-
- # some `res_model/res_model_id` combination may change between
- # versions (i.e. rating_rating.res_model_id was added in saas~15).
- # we need to verify existence of columns before using them.
- if ir.res_model and not column_exists(cr, ir.table, ir.res_model):
- ir = ir._replace(res_model=None) # noqa: PLW2901
- if ir.res_model_id and not column_exists(cr, ir.table, ir.res_model_id):
- ir = ir._replace(res_model_id=None) # noqa: PLW2901
- if not ir.res_model and not ir.res_model_id:
- continue
-
- yield ir
-
- if column_exists(cr, "ir_model_fields", "company_dependent"):
- cr.execute(
- """
- SELECT model, name, relation
- FROM ir_model_fields
- WHERE company_dependent IS TRUE
- AND ttype = 'many2one'
- """,
- )
- for model_name, column_name, comodel_name in cr.fetchall():
- yield _IR(table_of_model(cr, model_name), None, column_name, company_dependent_comodel=comodel_name)
-
- # XXX Once we will get the model field of `many2one_reference` fields in the database, we should get them also
- # (and filter the one already hardcoded)
-
-
-def generate_indirect_reference_cleaning_queries(cr, ir):
- """Yield queries to clean an `IndirectReference`."""
- assert not ir.company_dependent_comodel # not supported for now
- if ir.res_model:
- query = """
- SELECT {ir.res_model}
- FROM {ir.table}
- WHERE {ir.res_model} IS NOT NULL
- GROUP BY {ir.res_model}
- """
- else:
- query = """
- SELECT m.model
- FROM {ir.table} t
- JOIN ir_model m ON m.id = t.{ir.res_model_id}
- GROUP BY m.model
- """
- cr.execute(query.format(ir=ir))
- for (model,) in cr.fetchall():
- res_table = table_of_model(cr, model)
- if table_exists(cr, res_table):
- cond = "NOT EXISTS (SELECT 1 FROM {res_table} r WHERE r.id = t.{ir.res_id})".format(**locals())
- else:
- cond = "true"
-
- model_filter = ir.model_filter()
- yield cr.mogrify(
- "DELETE FROM {ir.table} t WHERE {model_filter} AND {cond}".format(**locals()), [model]
- ).decode()
-
-
-def res_model_res_id(cr, filtered=True):
- for ir in indirect_references(cr):
- if ir.res_model:
- yield model_of_table(cr, ir.table), ir.res_model, ir.res_id
diff --git a/src/util/inherit.py b/src/util/inherit.py
deleted file mode 100644
index 5ed4570e9..000000000
--- a/src/util/inherit.py
+++ /dev/null
@@ -1,123 +0,0 @@
-# -*- coding: utf-8 -*-
-import logging
-import operator
-import os
-
-from .const import ENVIRON, NEARLYWARN
-from .misc import _cached, parse_version, version_gte
-
-_logger = logging.getLogger(__name__)
-
-
-if version_gte("saas~17.1"):
- from ._inherit import Inherit, frozendict
-
- @_cached
- def _get_inheritance_data(cr):
- base_version = _get_base_version(cr)[:2] + ("*final",)
- cr.execute(
- """
- SELECT p.model,
- array_agg(m.model ORDER BY i.id),
- array_agg(f.name ORDER BY i.id)
- FROM ir_model_inherit i
- JOIN ir_model m
- ON m.id = i.model_id
- JOIN ir_model p
- ON p.id = i.parent_id
- LEFT JOIN ir_model_fields f
- ON f.id = i.parent_field_id
- GROUP BY p.model
- """
- )
- return frozendict(
- {
- parent: [
- Inherit(model=model, born=base_version, dead=None, via=via)
- for model, via in zip(children, vias, strict=True)
- ]
- for parent, children, vias in cr.fetchall()
- }
- )
-
-else:
- from ._inherit import inheritance_data
-
- def _get_inheritance_data(cr):
- return inheritance_data
-
-
-def _get_base_version(cr):
- # base_version is normally computed in `base/0.0.0/pre-base_version.py` (and symlinks)
- # However, if theses scripts are used to upgrade custom modules afterward (like the P.S. do),
- # as the `base` module not being updated, the *base_version* MUST be set as an environment variable.
- bv = ENVIRON.get("__base_version")
- if bv:
- return bv
- # trust env variable if set
- bv = os.getenv("ODOO_BASE_VERSION")
- if bv:
- bv = ENVIRON["__base_version"] = parse_version(bv)
- else:
- cr.execute("SELECT state, latest_version FROM ir_module_module WHERE name='base'")
- state, version = cr.fetchone()
- if state != "to upgrade":
- major = ".".join(version.split(".")[:2])
- _logger.log(
- NEARLYWARN,
- "Assuming upgrading from Odoo %s. If it's not the case, specify the environment variable `ODOO_BASE_VERSION`.",
- major,
- )
- bv = ENVIRON["__base_version"] = parse_version(version)
- return bv
-
-
-def _version_comparator(cr, interval):
- if interval not in {"[]", "()", "[)", "(]"}:
- raise ValueError("Invalid interval: %r" % (interval,))
-
- op_lower = operator.le if interval[0] == "[" else operator.lt
- op_upper = operator.le if interval[1] == "]" else operator.lt
- base_version = _get_base_version(cr)
-
- return lambda inh: op_lower(inh.born, base_version) and (inh.dead is None or op_upper(base_version, inh.dead))
-
-
-def for_each_inherit(cr, model, skip=(), interval="[)"):
- if skip == "*":
- return
- cmp_ = _version_comparator(cr, interval)
- for inh in _get_inheritance_data(cr).get(model, []):
- if inh.model in skip:
- continue
- if cmp_(inh):
- yield inh
-
-
-def direct_inherit_parents(cr, model, skip=(), interval="[)"):
- """Yield the *direct* inherits parents."""
- if skip == "*":
- return
- skip = set(skip)
- cmp_ = _version_comparator(cr, interval)
- for parent, inhs in _get_inheritance_data(cr).items():
- if parent in skip:
- continue
- for inh in inhs:
- if inh.model == model and cmp_(inh):
- yield parent, inh
- skip.add(parent)
-
-
-def inherit_parents(cr, model, skip=(), interval="[)"):
- """Recursively yield all inherit parents model names."""
- if skip == "*":
- return
- skip = set(skip)
- for parent, _inh in direct_inherit_parents(cr, model, skip=skip, interval=interval):
- if parent in skip:
- continue
- yield parent
- skip.add(parent)
- for grand_parent in inherit_parents(cr, parent, skip=skip, interval=interval):
- yield grand_parent
diff --git a/src/util/jinja_to_qweb.py b/src/util/jinja_to_qweb.py
deleted file mode 100644
index 70842cb22..000000000
--- a/src/util/jinja_to_qweb.py
+++ /dev/null
@@ -1,673 +0,0 @@
-import contextlib
-import functools
-import html
-import logging
-import re
-from multiprocessing import Process, Queue
-
-import babel
-import lxml
-from dateutil import relativedelta
-from jinja2.sandbox import SandboxedEnvironment
-from markupsafe import Markup
-from werkzeug import urls
-
-from odoo import tools
-from odoo.tools import is_html_empty, mute_logger, safe_eval
-
-from .helpers import _validate_table, model_of_table
-from .orm import env as get_env
-from .pg import named_cursor, table_exists
-from .report import add_to_migration_reports, html_escape
-
-_logger = logging.getLogger(__name__)
-
-REMOVE_SAFE_REGEX = re.compile(r"\s*\|\s*safe\s*", re.IGNORECASE)
-
-JINJA_EXPRESSION = r"""
-\$\{
- (?=(?P(?:
- [^}\'\"\\]
- |'
- (?=(?P
- (?:[^'\\]|\\.)*
- ))(?P=singlequote)'?
- |\"
- (?=(?P
- (?:[^\"\\]|\\.)*
- ))(?P=doublequote)\"?
- |
- (?:\\.)
- )*))(?P=insidebracket)
- }
-"""
-
-JINJA_EXPRESSION_REGEX = re.compile(JINJA_EXPRESSION, flags=re.X | re.DOTALL)
-
-JINJA_REGEX = re.compile(
- rf"""
- (?=(?P
- (?:
- (?:\\\\)
- |(?:\\\$)
- |(?:(?!\$\{{).)
- )*
- ))(?P=string)
- (?:{JINJA_EXPRESSION})?
- """,
- re.X | re.DOTALL,
-)
-
-
-def _remove_safe(expression):
- return re.sub(REMOVE_SAFE_REGEX, " ", expression).strip()
-
-
-def _transform_to_t_out(expression):
- return str(Markup('').format(_remove_safe(expression)))
-
-
-def convert_jinja_to_inline(string):
- result = []
- for element in re.finditer(JINJA_REGEX, string):
- static_string = element.group("string")
- expression = element.group("insidebracket")
- if not static_string and not expression:
- continue
-
- if static_string:
- result.append(static_string)
- if expression:
- result.append("{{ %s }}" % (_remove_safe(expression),))
- return "".join(result)
-
-
-def _convert_jinja_to_t_out_text(node):
- comment = isinstance(node, lxml.etree._Comment)
- last_node = None
-
- index = 0
- for element in re.finditer(JINJA_REGEX, node.text):
- static_string = element.group("string")
- expression = element.group("insidebracket")
- if not static_string and not expression:
- continue
-
- if last_node is None:
- node.text = static_string
- last_node = node
- elif comment and static_string:
- node.text += static_string
- elif static_string:
- last_node.tail = static_string
-
- if expression:
- if comment:
- node.text += _transform_to_t_out(expression)
- else:
- new_node = lxml.html.fragment_fromstring(_transform_to_t_out(expression))
- node.insert(index, new_node)
- index += 1
- last_node = new_node
-
-
-def _convert_jinja_to_t_out_tail(node):
- last_node = node
-
- for element in re.finditer(JINJA_REGEX, node.tail):
- static_string = element.group("string")
- expression = element.group("insidebracket")
- if not static_string and not expression:
- continue
-
- last_node.tail = ""
- if expression:
- node = lxml.html.fragment_fromstring(_transform_to_t_out(expression))
- last_node.addnext(node)
- if static_string:
- last_node.tail = static_string
- if expression:
- last_node = node
-
-
-def convert_jinja_to_qweb(string):
- string = re.sub(r"""^<\?xml version=("|')1\.0\1\?>\s*""", "", string, flags=re.M)
- # Create a parent in case there is multiples root nodes
- element = lxml.html.fragment_fromstring(string, create_parent="div")
- for el in element.getiterator():
- if el.text:
- _convert_jinja_to_t_out_text(el)
- if el.tail:
- _convert_jinja_to_t_out_tail(el)
- for key, value in el.attrib.items():
- if re.search(JINJA_EXPRESSION_REGEX, value):
- del el.attrib[key]
- el.set("t-attf-" + key, convert_jinja_to_inline(value))
- result = lxml.html.tostring(element, encoding="unicode")
- # Remove the technically created parent div, otherwise the first jinja
- # instruction will not match a jinja regex.
- result = result[5:-6]
-
- for func in [
- _replace_set,
- _replace_for,
- _replace_endfor,
- _replace_if,
- _replace_elif,
- _replace_else,
- _replace_endif,
- ]:
- result = func(result)
-
- # Make sure the html is correct
- result = lxml.etree.tostring(lxml.html.fragment_fromstring(result, create_parent="div"), encoding="unicode")
-
- # Remove the parent div
- return result[5:-6]
-
-
-def _get_set(matchobj):
- return Markup("""{}""").format(
- matchobj.group(1),
- html.unescape(matchobj.group(2).strip()),
- html.unescape(matchobj.group(3).strip()),
- )
-
-
-def _replace_set(string):
- reg = re.compile(r"^(\s*)%\s*set([^=]*)=(.*)", re.IGNORECASE | re.MULTILINE)
- return reg.sub(_get_set, string)
-
-
-def _get_for(matchobj):
- return Markup("""{}""").format(
- matchobj.group(1), html.unescape(matchobj.group(3).strip()), html.unescape(matchobj.group(2).strip())
- )
-
-
-def _replace_for(string):
- reg = re.compile(r"^(\s*)%\s*for((?:(?! in ).)*?) in (.*?):?\s*$", re.IGNORECASE | re.MULTILINE)
- return reg.sub(_get_for, string)
-
-
-def _replace_endfor(string):
- reg = re.compile(r"^(\s*)%\s*endfor.*", re.IGNORECASE | re.MULTILINE)
- return reg.sub(r"\1", string)
-
-
-def _get_if(matchobj):
- return Markup("""{}""").format(matchobj.group(1), html.unescape(matchobj.group(2).strip()))
-
-
-def _replace_if(string):
- reg = re.compile(r"^(\s*)%\s*if(.*?):?\s*$", re.IGNORECASE | re.MULTILINE)
- return reg.sub(_get_if, string)
-
-
-def _get_elif(matchobj):
- return Markup("""{}\n{}""").format(
- matchobj.group(1), matchobj.group(1), html.unescape(matchobj.group(2).strip())
- )
-
-
-def _replace_elif(string):
- reg = re.compile(r"^(\s*)%\s*elif(.*?):?\s*$", re.IGNORECASE | re.MULTILINE)
- return reg.sub(_get_elif, string)
-
-
-def _replace_else(string):
- reg = re.compile(r"^(\s*)%\s*else.*", re.IGNORECASE | re.MULTILINE)
- return reg.sub(r'\1\n\1', string)
-
-
-def _replace_endif(string):
- reg = re.compile(r"^(\s*)%\s*endif.*", re.IGNORECASE | re.MULTILINE)
- return reg.sub(r"\1", string)
-
-
-def setup_templates_to_check(cr):
- if table_exists(cr, "_upgrade_jinja_to_qweb"):
- return
- cr.execute(
- """
- CREATE UNLOGGED TABLE _upgrade_jinja_to_qweb(
- table_name varchar,
- template_type varchar,
- table_id int4,
- template_field varchar,
- model_name varchar,
- template_name varchar,
- template_name_field varchar,
- template_converted varchar,
- PRIMARY KEY(table_name, template_type, table_id, template_field)
- )
- """
- )
-
-
-def insert_templates_to_check(cr, *args):
- cr.execute(
- """
- INSERT INTO _upgrade_jinja_to_qweb (
- table_name,
- template_type,
- table_id,
- template_field,
- model_name,
- template_name,
- template_name_field,
- template_converted
- ) VALUES(%s, %s, %s, %s, %s, %s, %s, %s)
- """,
- args,
- )
-
-
-def cleanup_templates_to_check(cr):
- cr.execute("DROP TABLE _upgrade_jinja_to_qweb")
-
-
-def upgrade_jinja_fields(
- cr,
- table_name,
- inline_template_fields,
- qweb_fields,
- name_field="name",
- model_name=None,
- table_model_name="model",
- fetch_model_name=False,
-):
- _validate_table(table_name)
- all_field = inline_template_fields + qweb_fields
- if not model_name:
- all_field = [table_model_name] + all_field
- sql_fields = ", ".join(all_field)
-
- sql_where_inline_fields = [field + " like '%${%'" for field in inline_template_fields]
- sql_where_qweb_fields = [field + r"~ '(\$\{|%\s*(if|for))'" for field in qweb_fields]
- sql_where_fields = " OR ".join(sql_where_inline_fields + sql_where_qweb_fields)
-
- setup_templates_to_check(cr)
- model = model_of_table(cr, table_name)
-
- cr.commit() # ease the processing for PG
- ncr = named_cursor(cr, 100)
- ncr.execute(
- f"""
- SELECT id, {name_field}, {sql_fields}
- FROM {table_name}
- WHERE {sql_where_fields}
- """
- )
- for data in ncr.iterdict():
- _logger.info("process %s(%s) %s", table_name, data["id"], data[name_field])
-
- # only for mailing.mailing
- if fetch_model_name:
- cr.execute(
- """
- SELECT model FROM ir_model WHERE id=%s
- """,
- [data[table_model_name]],
- )
- model_name = cr.fetchone()[0]
- else:
- model_name = model_name or data[table_model_name]
-
- # convert the fields
- templates_converted = {}
- for field in inline_template_fields:
- _logger.info(" `- convert inline field %s", field)
- converted = convert_jinja_to_inline(data[field]) if data[field] else ""
- templates_converted[field] = converted
- if data[field]:
- insert_templates_to_check(
- cr,
- table_name,
- "inline_template",
- data["id"],
- field,
- model_name,
- data[name_field],
- name_field,
- converted,
- )
-
- for field in qweb_fields:
- _logger.info(" `- convert qweb field %s", field)
- converted = convert_jinja_to_qweb(data[field]) if data[field] else ""
- templates_converted[field] = converted
- insert_templates_to_check(
- cr, table_name, "qweb", data["id"], field, model_name, data[name_field], name_field, converted
- )
-
- fields = [f for f in (inline_template_fields + qweb_fields) if data[f] != templates_converted[f]]
- if fields:
- sql_fields = ",".join([field + "=%s" for field in fields])
- field_values = [templates_converted[field] for field in fields]
-
- cr.execute(
- f"""
- UPDATE {table_name}
- SET {sql_fields}
- WHERE id = %s
- """,
- field_values + [data["id"]],
- )
- ncr.close()
-
- if not table_exists(cr, "ir_translation"):
- return
-
- _logger.info("process translations for model %s", model)
- # NOTE: Not all translations may not be updated.
- # Difference jinja values can give the same qweb value.
- # `${object.company_id.name|safe}` and `${object.company_id.name}` both give `{{ object.company_id.name }}`
- # Which will violates the unique constraint.
- # In this case, we just ignore the update and remove the duplicated row.
-
- inline_entries = [f"{model},{name}" for name in inline_template_fields]
- if inline_entries:
- cr.execute(
- """
- SELECT id, src, value
- FROM ir_translation
- WHERE name IN %s
- AND (src LIKE '%%${%%' OR value LIKE '%%${%%')
- """,
- [tuple(inline_entries)],
- )
- for tid, src, value in cr.fetchall():
- converted_src = convert_jinja_to_inline(src) if src else ""
- converted_value = convert_jinja_to_inline(value) if value else ""
- cr.execute(
- """
- DELETE FROM ir_translation orig
- USING ir_translation dup
- WHERE orig.id = %s
- AND dup.id != orig.id
- -- "ir_translation_unique" UNIQUE, btree (type, name, lang, res_id, md5(src))
- AND dup.type = orig.type
- AND dup.name = orig.name
- AND dup.lang = orig.lang
- AND dup.res_id = orig.res_id
- AND dup.src = %s
- RETURNING orig.id
- """,
- [tid, converted_src],
- )
- if not cr.rowcount:
- # no duplicate found, update the translation
- cr.execute(
- "UPDATE ir_translation SET src=%s, value=%s WHERE id=%s",
- [converted_src, converted_value, tid],
- )
-
- cr.commit() # commit changes for the named cursor below
- qweb_entries = [f"{model},{name}" for name in qweb_fields]
- if qweb_entries:
- ncr = named_cursor(cr, 1000)
- ncr.execute(
- r"""
- SELECT id, src, value
- FROM ir_translation
- WHERE name IN %s
- AND (src ~ '(\$\{|%%\s*(if|for))' OR value ~ '(\$\{|%%\s*(if|for))')
- """,
- [tuple(qweb_entries)],
- )
-
- for tid, src, value in ncr:
- converted_src = convert_jinja_to_qweb(src) if src else ""
- converted_value = convert_jinja_to_qweb(value) if value else ""
- cr.execute(
- """
- DELETE FROM ir_translation orig
- USING ir_translation dup
- WHERE orig.id = %s
- AND dup.id != orig.id
- -- "ir_translation_unique" UNIQUE, btree (type, name, lang, res_id, md5(src))
- AND dup.type = orig.type
- AND dup.name = orig.name
- AND dup.lang = orig.lang
- AND dup.res_id = orig.res_id
- AND dup.src = %s
- RETURNING orig.id
- """,
- [tid, converted_src],
- )
- if not cr.rowcount:
- # no duplicate found, update the translation
- cr.execute(
- "UPDATE ir_translation SET src=%s, value=%s WHERE id=%s",
- [converted_src, converted_value, tid],
- )
- ncr.close()
-
-
-def verify_upgraded_jinja_fields(cr):
- env = get_env(cr)
- cr.execute("SELECT DISTINCT(table_name) FROM _upgrade_jinja_to_qweb")
- for (table_name,) in cr.fetchall():
- field_errors = {}
- missing_records = []
- ncr = named_cursor(cr, 100)
- ncr.execute(
- """
- SELECT template_type,
- table_id,
- template_field,
- model_name,
- template_name,
- template_name_field,
- template_converted
- FROM _upgrade_jinja_to_qweb
- WHERE table_name = %s
- ORDER BY template_type,
- template_field
- """,
- [table_name],
- )
- for (
- template_type,
- table_id,
- template_field,
- model_name,
- template_name,
- template_name_field,
- template_converted,
- ) in ncr:
- if model_name not in env:
- # custom model not loaded yet. Ignore
- continue
- model = env[model_name]
- record = model.with_context({"active_test": False}).search([], limit=1, order="id")
-
- key = (table_id, template_name, template_name_field)
- field_errors[key] = []
-
- if not record:
- missing_records.append(key)
-
- is_valid = is_converted_template_valid(
- env, template_field, template_converted, model_name, record.id, engine=template_type
- )
- if not is_valid:
- field_errors[key].append(template_field)
-
- if missing_records:
- list_items = "\n".join(
- f'id: "{id}", {html_escape(name_field)}: "{html_escape(name)}" '
- for id, name, name_field in missing_records
- )
- add_to_migration_reports(
- f"""
-
-
- Some of the records for the table {html_escape(table_name)} could not be tested because there is no
- record in the database.
- The {html_escape(table_name)} records are:
-
-
-
- """,
- "Jinja upgrade",
- format="html",
- )
- field_errors = dict(filter(lambda x: bool(x[1]), field_errors.items()))
-
- if field_errors:
- string = []
- for (id, name, name_field), fields in field_errors.items():
- fields_string = "\n".join(f"{html_escape(field)}" for field in fields)
- string.append(
- f"""id: {id}, {html_escape(name_field)}: {html_escape(name)},
- fields: """
- )
-
- string = "\n".join(string)
- add_to_migration_reports(
- f"""
-
-
- Some of the fields of the table {html_escape(table_name)} does not render the same value
- before and after being converted.
- The mail.template are:
-
-
-
- """,
- "Jinja upgrade",
- format="html",
- )
- ncr.close()
- cleanup_templates_to_check(cr)
-
-
-def is_converted_template_valid(env, template_before, template_after, model_name, record_id, engine="inline_template"):
- def callback(q):
- render_before = None
- with contextlib.suppress(Exception):
- render_before = _render_template_jinja(env, template_before, model_name, record_id)
-
- render_after = None
- if render_before is not None:
- try:
- with mute_logger("odoo.addons.mail.models.mail_render_mixin"):
- render_after = env["mail.render.mixin"]._render_template(
- template_after, model_name, [record_id], engine=engine
- )[record_id]
- except Exception:
- pass
-
- # post process qweb render to remove comments from the rendered jinja in
- # order to avoid false negative because qweb never render comments.
- if render_before and render_after and engine == "qweb":
- element_before = lxml.html.fragment_fromstring(render_before, create_parent="div")
- for comment_element in element_before.xpath("//comment()"):
- comment_element.getparent().remove(comment_element)
- render_before = lxml.html.tostring(element_before, encoding="unicode")
- render_after = lxml.html.tostring(
- lxml.html.fragment_fromstring(render_after, create_parent="div"), encoding="unicode"
- )
-
- q.put(render_before is not None and render_before == render_after)
-
- # to avoid memory leaks in external C libraries (lxml/libxml2), process in a forked child
- queue = Queue()
- proc = Process(target=callback, args=[queue])
- proc.start()
- res = queue.get(timeout=60)
- if proc.is_alive():
- proc.kill()
- return res
-
-
-# jinja render
-
-
-def format_date(env, date, pattern=False, lang_code=False):
- try:
- return tools.format_date(env, date, date_format=pattern, lang_code=lang_code)
- except babel.core.UnknownLocaleError:
- return date
-
-
-def format_datetime(env, dt, tz=False, dt_format="medium", lang_code=False):
- try:
- return tools.format_datetime(env, dt, tz=tz, dt_format=dt_format, lang_code=lang_code)
- except babel.core.UnknownLocaleError:
- return dt
-
-
-def format_time(env, time, tz=False, time_format="medium", lang_code=False):
- try:
- return tools.format_time(env, time, tz=tz, time_format=time_format, lang_code=lang_code)
- except babel.core.UnknownLocaleError:
- return time
-
-
-def relativedelta_proxy(*args, **kwargs):
- # dateutil.relativedelta is an old-style class and cannot be directly
- # instantiated within a jinja2 expression, so a lambda "proxy" is
- # is needed, apparently
- return relativedelta.relativedelta(*args, **kwargs)
-
-
-template_env_globals = {
- "str": str,
- "quote": urls.url_quote,
- "urlencode": urls.url_encode,
- "datetime": safe_eval.datetime,
- "len": len,
- "abs": abs,
- "min": min,
- "max": max,
- "sum": sum,
- "filter": filter,
- "reduce": functools.reduce,
- "map": map,
- "relativedelta": relativedelta_proxy,
- "round": round,
-}
-
-jinja_template_env = SandboxedEnvironment(
- block_start_string="<%",
- block_end_string="%>",
- variable_start_string="${",
- variable_end_string="}",
- comment_start_string="<%doc>",
- comment_end_string="",
- line_statement_prefix="%",
- line_comment_prefix="##",
- trim_blocks=True, # do not output newline after blocks
- autoescape=True, # XML/HTML automatic escaping
-)
-
-jinja_template_env.globals.update(template_env_globals)
-
-
-def _render_template_jinja(env, template_txt, model, res_id):
- if not template_txt:
- return ""
-
- template = jinja_template_env.from_string(tools.ustr(template_txt))
-
- record = env[model].browse([res_id])
- variables = {
- "format_date": functools.partial(format_date, env),
- "format_datetime": functools.partial(format_datetime, env),
- "format_time": functools.partial(format_time, env),
- "format_amount": functools.partial(tools.format_amount, env),
- "format_duration": tools.format_duration,
- "user": env.user,
- "ctx": {},
- "is_html_empty": is_html_empty,
- "object": record,
- }
-
- safe_eval.check_values(variables)
- render_result = template.render(variables)
- if render_result == "False":
- render_result = ""
- return render_result
diff --git a/src/util/json.py b/src/util/json.py
deleted file mode 100644
index f710a52bb..000000000
--- a/src/util/json.py
+++ /dev/null
@@ -1,34 +0,0 @@
-from __future__ import absolute_import
-
-__all__ = ["dumps", "load", "loads"]
-
-try:
- import orjson
-except ImportError:
- import json
-
- def dumps(value, sort_keys=False):
- return json.dumps(value, sort_keys=sort_keys, separators=(",", ":"))
-
- def loads(value):
- return json.loads(value)
-
- def load(fp):
- return json.load(fp)
-else:
-
- def dumps(value, sort_keys=False):
- if isinstance(value, tuple):
- # downcast namedtuples
- value = tuple(value)
-
- option = orjson.OPT_NON_STR_KEYS
- if sort_keys:
- option |= orjson.OPT_SORT_KEYS
- return orjson.dumps(value, option=option).decode()
-
- def loads(value):
- return orjson.loads(value)
-
- def load(fp):
- return orjson.loads(fp.read())
diff --git a/src/util/logger.py b/src/util/logger.py
deleted file mode 100644
index b058087ea..000000000
--- a/src/util/logger.py
+++ /dev/null
@@ -1,29 +0,0 @@
-# -*- coding: utf-8 -*-
-import atexit
-import logging
-import os
-
-from .misc import on_CI
-
-_REGISTERED = False
-
-_logger = logging.getLogger(__name__.rpartition(".")[0])
-
-
-class CriticalHandler(logging.Handler):
- def __init__(self):
- super(CriticalHandler, self).__init__(logging.CRITICAL)
-
- def emit(self, record):
- global _REGISTERED # noqa: PLW0603
- if _REGISTERED:
- return
-
- # force exit with status_code=1 if any critical log is emit during upgrade
- atexit.register(os._exit, 1)
- _REGISTERED = True
-
-
-if on_CI(): # hopefully temporary restriction
- product = _logger.name.partition(".")[0]
- logging.getLogger(product).addHandler(CriticalHandler())
diff --git a/src/util/misc.py b/src/util/misc.py
deleted file mode 100644
index 85a0f9bd4..000000000
--- a/src/util/misc.py
+++ /dev/null
@@ -1,836 +0,0 @@
-"""Miscellaneous standalone functions."""
-
-import ast
-import collections
-import datetime
-import functools
-import hashlib
-import inspect
-import logging
-import os
-import re
-import sys
-import textwrap
-import uuid
-from contextlib import contextmanager
-from itertools import chain, islice
-
-try:
- from odoo import release
- from odoo.modules.module import get_module_path
- from odoo.tools.parse_version import parse_version
-except ImportError:
- from openerp import release
- from openerp.modules.module import get_module_path
- from openerp.tools.parse_version import parse_version
-
-from .exceptions import MigrationError, SleepyDeveloperError
-
-# python3 shim
-try:
- unicode # noqa: B018
-except NameError:
- unicode = str
-
-try:
- from ast import unparse as ast_unparse
-except ImportError:
- try:
- from astunparse import unparse as _ast_unparse
-
- def ast_unparse(n):
- # Older unparse adds weird new lines:
- # >>> unparse(ast.parse("a",mode="eval").body)
- # 'a\n'
- return _ast_unparse(n).strip()
- except ImportError:
- ast_unparse = None
-
-_logger = logging.getLogger(__name__)
-
-
-class Sentinel:
- # waiting PEP661...
- __slots__ = ("name",)
-
- def __init__(self, name):
- self.name = name
-
- def __repr__(self):
- return self.name
-
-
-AUTO = AUTOMATIC = Sentinel("AUTO")
-
-
-def _cached(func):
- sentinel = Sentinel("sentinel")
-
- @functools.wraps(func)
- def wrapper(*args, **kwargs):
- result = getattr(func, "_result", sentinel)
- if result == sentinel:
- result = func._result = func(*args, **kwargs)
- return result
-
- return wrapper
-
-
-# copied from odoo as older OpenERP versions doesn't have it
-def str2bool(s, default=None):
- s = unicode(s).lower()
- y = ["y", "yes", "1", "true", "t", "on"]
- n = ["n", "no", "0", "false", "f", "off"]
- if s not in (y + n):
- if default is None:
- raise ValueError("Use 0/1/yes/no/true/false/on/off")
- return bool(default)
- return s in y
-
-
-def version_gte(version):
- """
- Return whether currently running Odoo version is greater or equal to `version`.
-
- This function is useful for conditional execution in an upgrade script that applies to
- multiple versions, e.g. `0.0.0` scripts.
-
- :param str version: Odoo version, must follow the format `[saas~]X.Y` where `X` is the
- major Odoo version, `saas~` is necessary only when `Y` is nonzero
- :rtype: bool
- """
- if "-" in version:
- raise SleepyDeveloperError("version cannot contains dash")
- return parse_version(release.serie) >= parse_version(version)
-
-
-def version_between(a, b):
- """
- Return whether currently running Odoo version is in the range `[a,b]`.
-
- See also :func:`~odoo.upgrade.util.misc.version_gte`
-
- .. note::
- The bounds are inclusive.
-
- :param str a: Odoo version, lower bound
- :param str b: Odoo version, upper bound
- :rtype: bool
-
- """
- if "-" in a + b:
- raise SleepyDeveloperError("version cannot contains dash")
- return parse_version(a) <= parse_version(release.serie) <= parse_version(b)
-
-
-@_cached
-def has_enterprise():
- """
- Return whether the current installation has enterprise addons available.
-
- :meta private: exclude from online docs
- """
- # NOTE should always return True as customers need Enterprise to migrate or
- # they are on SaaS, which include enterpise addons.
- # This act as a sanity check for developers or in case we release the scripts.
- if os.getenv("ODOO_HAS_ENTERPRISE"):
- return True
- # XXX maybe we will need to change this for version > 9
- kw = {} if version_gte("saas~18.5") else {"downloaded": False}
- return bool(get_module_path("delivery_fedex", display_warning=False, **kw))
-
-
-@_cached
-def has_design_themes():
- """
- Return whether the current installation has theme addons available.
-
- :meta private: exclude from online docs
- """
- if os.getenv("ODOO_HAS_DESIGN_THEMES"):
- return True
- kw = {} if version_gte("saas~18.5") else {"downloaded": False}
- return bool(get_module_path("theme_yes", display_warning=False, **kw))
-
-
-@_cached
-def on_CI():
- return str2bool(os.getenv("MATT", "0")) or str2bool(os.getenv("CI", "0"))
-
-
-def splitlines(s):
- """
- Yield stripped lines of `s`. Skip empty lines Remove comments (starts with `#`).
-
- :meta private: exclude from online docs
- """
- return (
- stripped_line for line in s.splitlines() for stripped_line in [line.split("#", 1)[0].strip()] if stripped_line
- )
-
-
-def expand_braces(s):
- """
- Expand braces in the input.
-
- .. example::
-
- .. code-block:: python
-
- >>> util.expand_braces("a_{this,that}_b")
- ['a_this_b', 'a_that_b']
-
- :param str s: string to expand, must contain precisely one pair of braces.
- :return: expanded input
- """
- # expand braces (a la bash)
- r = re.compile(r"(.*){((?:[^},]*?)(?:,[^},]*?)+)}(.*)", flags=re.DOTALL)
- m = r.search(s)
- if not m:
- raise ValueError("No expansion braces found")
- head, matches, tail = m.groups()
- if re.search("[}{]", head + matches + tail):
- raise ValueError("Extra braces detected")
- return [head + x + tail for x in matches.split(",")]
-
-
-def split_osenv(name, default=""):
- value = os.getenv(name, default).strip()
- return re.split(r"\W+", value) if value else []
-
-
-try:
- import importlib.util
- from pathlib import Path
-
- try:
- import odoo.upgrade
- except ImportError:
- _search_path = [Path(__file__).parent.parent]
- else:
- _search_path = [Path(p) for p in odoo.upgrade.__path__]
-
- def import_script(path, name=None):
- """
- Import an upgrade script.
-
- This function allows to import functions from other upgrade scripts into the
- current one.
-
- .. example::
- In :file:`mymodule/15.0.1.0/pre-migrate.py`
-
- .. code-block:: python
-
- def my_util(cr):
- # do stuff
-
- In :file:`myothermodule/16.0.1.0/post-migrate.py`
-
- .. code-block:: python
-
- from odoo.upgrade import util
-
- script = util.import_script("mymodule/15.0.1.0/pre-migrate.py")
-
- def migrate(cr, version):
- script.my_util(cr) # reuse the function
-
- This function returns a Python `module`.
-
- .. code-block:: python
-
- >>> util.import_script("base/0.0.0/end-moved0.py", name="my-moved0")
-
-
- :param str path: relative path to the script to import in the form
- `$module/$version/$script-name`
-
- .. note::
- The script must be available in the upgrade path.
-
- :param str or None name: name to assign to the returned module, take the name from
- the imported file if `None`
- :return: a module created from the imported upgrade script
- """
- if not name:
- name, _ = os.path.splitext(os.path.basename(path))
- for full_path in (sp / path for sp in _search_path):
- if full_path.exists():
- break
- else:
- raise ImportError(path) # noqa: TRY301
- spec = importlib.util.spec_from_file_location(name, full_path)
- module = importlib.util.module_from_spec(spec)
- spec.loader.exec_module(module)
- return module
-
-except ImportError:
- # python2 version
- import imp
-
- def import_script(path, name=None):
- if not name:
- name, _ = os.path.splitext(os.path.basename(path))
- full_path = os.path.join(os.path.dirname(__file__), "..", path)
- with open(full_path) as fp:
- return imp.load_source(name, full_path, fp)
-
-
-@contextmanager
-def skippable_cm():
- """
- Return a context manager to allow another context manager to not yield.
-
- See :func:`~odoo.upgrade.util.records.edit_view` for an example usage.
- """
- if not hasattr(skippable_cm, "_msg"):
-
- @contextmanager
- def _():
- if 0:
- yield
-
- try:
- with _():
- pass
- except RuntimeError as r:
- skippable_cm._msg = str(r)
- try:
- yield
- except RuntimeError as r:
- if str(r) != skippable_cm._msg:
- raise
-
-
-def chunks(iterable, size, fmt=None):
- """
- Split `iterable` into chunks of `size` and wrap each chunk using `fmt` function.
-
- This function is useful for splitting huge input data into smaller chunks that can be
- processed independently.
-
- .. example::
-
- .. code-block:: python
-
- >>> list(chunks(range(10), 4, fmt=tuple))
- [(0, 1, 2, 3), (4, 5, 6, 7), (8, 9)]
- >>> ' '.join(chunks('abcdefghijklm', 3))
- 'abc def ghi jkl m'
-
- :param iterable iterable: iterable object to split
- :param int size: chunk size
- :param function fmt: function to apply to each chunk, when `None` is passed `fmt`
- becomes `"".join` if `iterable` is a string, otherwise `iter`
- :return: a generator that iterates over the result of `fmt` applied to each chunk
- """
- if fmt is None:
- # fmt:off
- fmt = {
- str: "".join,
- unicode: u"".join,
- }.get(type(iterable), iter)
- # fmt:on
-
- it = iter(iterable)
- try:
- while True:
- yield fmt(chain((next(it),), islice(it, size - 1)))
- except StopIteration:
- return
-
-
-def log_progress(it, logger, qualifier="elements", size=None, estimate=True, log_hundred_percent=False):
- if size is None:
- size = len(it)
- t0 = t1 = datetime.datetime.now()
- for i, e in enumerate(it, 1):
- yield e
- t2 = datetime.datetime.now()
- secs_last, secs_start = (t2 - t1).total_seconds(), (t2 - t0).total_seconds()
- if secs_last > 60 or (log_hundred_percent and i == size and secs_start > 10):
- t1 = datetime.datetime.now()
- tdiff = t2 - t0
- j = float(i)
- if estimate:
- tail = " (total estimated time: %s)" % (datetime.timedelta(seconds=tdiff.total_seconds() * size / j),)
- else:
- tail = ""
-
- logger.info(
- "[%.02f%%] %d/%d %s processed in %s%s",
- (j / size * 100.0),
- i,
- size,
- qualifier,
- tdiff,
- tail,
- )
-
-
-def log_chunks(it, logger, chunk_size, qualifier="items"):
- tinit = tlog = datetime.datetime.now()
-
- def log(chunk_num, size=chunk_size):
- now = datetime.datetime.now()
- logger.info(
- "Chunk #%d of %d %s processed in %s (total %s)",
- chunk_num,
- size,
- qualifier,
- now - tlog,
- now - tinit,
- )
- return now
-
- i = 0
- for i, e in enumerate(it, 1):
- yield e
- tlog = tlog if i % chunk_size else log(i // chunk_size)
-
- if i == 0:
- # empty iterator
- logger.info("No %s to process", qualifier)
- elif i % chunk_size != 0:
- # log the last partial chunk
- log(i // chunk_size + 1, i % chunk_size)
-
-
-def make_pickleable_callback(callback):
- """
- Make a callable importable.
-
- `ProcessPoolExecutor.map` arguments needs to be pickleable
- Functions can only be pickled if they are importable.
- However, the callback's file is not importable due to the dash in the filename.
- We should then put the executed function in its own importable file.
-
- :meta private: exclude from online docs
- """
- callback_filepath = inspect.getfile(callback)
- name = "_upgrade_" + hashlib.sha256(callback_filepath.encode()).hexdigest()
- if name not in sys.modules:
- sys.modules[name] = import_script(callback_filepath, name=name)
- try:
- return getattr(sys.modules[name], callback.__name__)
- except AttributeError:
- error_msg = (
- "The converter callback `{}` is a nested function in `{}`.\n"
- "Move it outside the `migrate()` function to make it top-level."
- ).format(callback.__name__, callback.__module__)
- raise MigrationError(error_msg)
-
-
-class SelfPrint(object):
- """
- Class that will return a self representing string. Used to evaluate domains.
-
- :meta private: exclude from online docs
- """
-
- def __init__(self, name):
- self.__name = name
-
- def __getattr__(self, attr):
- return SelfPrint("%r.%s" % (self, attr))
-
- def __getitem__(self, key):
- return SelfPrint("%r[%r]" % (self, key))
-
- def __call__(self, *args, **kwargs):
- s = [repr(a) for a in args]
- for k, v in kwargs.items():
- s.append("%s=%r" % (k, v))
- return SelfPrint("%r(%s)" % (self, ", ".join(s)))
-
- def __add__(self, other):
- return SelfPrint("(%r + %r)" % (self, other))
-
- def __radd__(self, other):
- return SelfPrint("(%r + %r)" % (other, self))
-
- def __sub__(self, other):
- return SelfPrint("(%r - %r)" % (self, other))
-
- def __rsub__(self, other):
- return SelfPrint("(%r - %r)" % (other, self))
-
- def __mul__(self, other):
- return SelfPrint("(%r * %r)" % (self, other))
-
- def __rmul__(self, other):
- return SelfPrint("(%r * %r)" % (other, self))
-
- def __div__(self, other):
- return SelfPrint("(%r / %r)" % (self, other))
-
- def __rdiv__(self, other):
- return SelfPrint("(%r / %r)" % (other, self))
-
- def __truediv__(self, other):
- return SelfPrint("(%r / %r)" % (self, other))
-
- def __rtruediv__(self, other):
- return SelfPrint("(%r / %r)" % (other, self))
-
- def __floordiv__(self, other):
- return SelfPrint("(%r // %r)" % (self, other))
-
- def __rfloordiv__(self, other):
- return SelfPrint("(%r // %r)" % (other, self))
-
- def __mod__(self, other):
- return SelfPrint("(%r %% %r)" % (self, other))
-
- def __rmod__(self, other):
- return SelfPrint("(%r %% %r)" % (other, self))
-
- def __pow__(self, other):
- return SelfPrint("(%r ** %r)" % (self, other))
-
- def __rpow__(self, other):
- return SelfPrint("(%r ** %r)" % (other, self))
-
- def __pos__(self):
- return SelfPrint("+(%r)" % (self,))
-
- def __neg__(self):
- return SelfPrint("-(%r)" % (self,))
-
- def __repr__(self):
- return self.__name
-
- __str__ = __repr__
-
- def __iter__(self):
- raise RuntimeError("Cannot self-print iterations")
-
-
-class SelfPrintEvalContext(collections.defaultdict):
- """
- Evaluation Context that will return a SelfPrint object for all non-literal object.
-
- :meta private: exclude from online docs
- """
-
- def __init__(self, *args, **kwargs):
- super(SelfPrintEvalContext, self).__init__(None, *args, **kwargs)
-
- def __missing__(self, key):
- return SelfPrint(key)
-
- @classmethod
- def preprocess(klass, expr):
- """
- Prepare `expr` to be evaluated in a self printed context.
-
- This is necessary to avoid skipping domains that `SelfPrint` cannot print.
- Example: [('company_id', 'in', [*company_ids, False])
-
- Returns a pair with the new expression and an evaluation context that should
- be used in :func:`~odoo.upgrade.util.misc.safe_eval`.
-
- ```
- >>> prepared_domain, context = util.SelfPrintEvalContext.preprocess(domain)
- >>> safe_eval(prepared_domain, context)
- ```
-
- :meta private: exclude from online docs
- """
- if not ast_unparse:
- return expr, SelfPrintEvalContext()
-
- class RewriteName(ast.NodeTransformer):
- def __init__(self):
- self.replaces = {}
- super(RewriteName, self).__init__()
-
- def _replace_node(self, prefix, node):
- uniq_id = prefix + uuid.uuid4().hex[:12]
- unparsed = ast_unparse(node).strip()
- self.replaces[uniq_id] = SelfPrint(unparsed)
- return ast.Name(id=uniq_id, ctx=ast.Load())
-
- def visit_Starred(self, node):
- return self._replace_node("_upg_Starred", node)
-
- def visit_BoolOp(self, node):
- return self._replace_node("_upg_BoolOp", node)
-
- def visit_UnaryOp(self, node):
- return self._replace_node("_upg_UnaryOp_Not", node) if isinstance(node.op, ast.Not) else node
-
- replacer = RewriteName()
- root = ast.parse(expr.strip(), mode="exec")
- visited = replacer.visit(root)
- return (ast_unparse(visited).strip(), SelfPrintEvalContext(replacer.replaces))
-
-
-if version_gte("saas~18.4"):
- import odoo.tools.safe_eval as _safe_eval_mod
-
- def safe_eval(expr, context=None):
- if context is None:
- context = SelfPrintEvalContext()
-
- assert isinstance(expr, (str, bytes))
- assert isinstance(context, SelfPrintEvalContext)
-
- if version_gte("saas~18.5"):
- c = _safe_eval_mod.compile_codeobj(expr, filename=None, mode="eval")
- _safe_eval_mod.assert_valid_codeobj(_safe_eval_mod._SAFE_OPCODES, c, expr)
- else:
- c = _safe_eval_mod.test_expr(expr, _safe_eval_mod._SAFE_OPCODES, mode="eval", filename=None)
-
- context["__builtins__"] = dict(_safe_eval_mod._BUILTINS)
- try:
- return _safe_eval_mod.unsafe_eval(c, context, None)
- except _safe_eval_mod._BUBBLEUP_EXCEPTIONS:
- raise
- except Exception as e:
- raise ValueError("{!r} while evaluating\n{!r}".format(e, expr))
- finally:
- del context["__builtins__"]
-else:
- try:
- from odoo.tools.safe_eval import safe_eval as _safe_eval_func
- except ImportError:
- from openerp.tools.safe_eval import safe_eval as _safe_eval_func
-
- def safe_eval(expr, context=None):
- if context is None:
- context = SelfPrintEvalContext()
- assert isinstance(context, SelfPrintEvalContext)
-
- return _safe_eval_func(expr, context, nocopy=True)
-
-
-class _Replacer(ast.NodeTransformer):
- """Replace literal nodes in an AST."""
-
- def __init__(self, mapping):
- self.mapping = collections.defaultdict(list)
- for key, value in mapping.items():
- key_ast = key if isinstance(key, ast.AST) else ast.parse(key, mode="eval").body
- self.mapping[key_ast.__class__.__name__].append((key_ast, value))
- self.updated = False
-
- def visit(self, node):
- if node.__class__.__name__ in self.mapping:
- for target_ast, new in self.mapping[node.__class__.__name__]:
- if self._match(node, target_ast):
- if callable(new):
- new_node = new(node)
- self.updated |= new_node is not node
- return new_node
- self.updated = True
- # since we use eval below, non-callable must be an expression (no assignment)
- return ast.parse(new, mode="eval").body
- return super(_Replacer, self).visit(node)
-
- def _strict_match(self, left, right):
- return left is right
-
- def _all_match(self, lefts, rights):
- return len(lefts) == len(rights) and all(self._match(left_, right_) for left_, right_ in zip(lefts, rights))
-
- def _match(self, left, right):
- # None context is assumed as a wildcard, it's simpler to write than specific context instances
- same_ctx = (
- not hasattr(right, "ctx")
- or right.ctx is None
- or getattr(left, "ctx", None).__class__ is right.ctx.__class__
- )
- lcname = left.__class__.__name__
- rcname = right.__class__.__name__
- same_type = lcname == rcname or (lcname in ["Num", "Str", "NameConstant"] and rcname == "Constant")
- matcher = getattr(self, "_match_" + lcname, self._strict_match)
- return matcher(left, right) if same_type and same_ctx else False
-
- #### Literals ####
- def _match_Constant(self, left, right):
- # We don't care about kind for u-strings
- return right.value is literal_replace.WILDCARD or (
- type(left.value) is type(right.value) and left.value == right.value
- )
-
- def _match_Num(self, left, right):
- # Dreprecated, for Python <3.8, right could be matched against modern Constant
- rn = right.n if hasattr(right, "n") else right.value
- return rn is literal_replace.WILDCARD or (type(left.n) is type(rn) and left.n == rn)
-
- def _match_Str(self, left, right):
- # Dreprecated, for Python <3.8, right could be matched against modern Constant
- rs = right.s if hasattr(right, "s") else right.value
- return rs is literal_replace.WILDCARD or left.s == rs
-
- def _match_FormattedValue(self, left, right):
- return (
- self._match(left.value, right.value)
- and self._match(left.conversion, right.conversion)
- # format_spec could be None, handled in _strict_match
- and self._match(left.format_spec, right.format_spec)
- )
-
- def _match_JoinedStr(self, left, right):
- return self._all_match(left.values, right.values)
-
- def _match_Bytes(self, left, right):
- # Deprecated, for Python <3.8
- return self._match_Str(left, right)
-
- def _match_List(self, left, right):
- return (len(right.elts) == 1 and right.elts[0] is literal_replace.WILDCARD) or self._all_match(
- left.elts, right.elts
- )
-
- def _match_Tuple(self, left, right):
- return self._match_List(left, right)
-
- def _match_Set(self, left, right):
- return self._match_List(left, right)
-
- def _match_Dict(self, left, right):
- return self._all_match(left.keys, right.keys) and self._all_match(left.values, right.values)
-
- # Ellipsis is matched by _strict_match
-
- def _match_NameConstant(self, left, right):
- # Deprecated, for Python <3.8
- return right.value is literal_replace.WILDCARD or left.value == right.value
-
- #### Variables ####
-
- def _match_Name(self, left, right):
- return right.id is literal_replace.WILDCARD or left.id == right.id
-
- def _match_Starred(self, left, right):
- return self._match(left.value, right.value)
-
- #### Expressions ####
-
- def _match_Expr(self, left, right):
- return self._match(left.value, right.value)
-
- def _match_NamedExpr(self, left, right):
- return self._match(left.target, right.target) and self._match(left.value, right.value)
-
- def _match_UnaryOp(self, left, right):
- return left.op.__class__ is right.op.__class__ and self._match(left.operand, right.operand)
-
- def _match_BinOp(self, left, right):
- return (
- left.op.__class__ is right.op.__class__
- and self._match(left.left, right.left)
- and self._match(left.right, right.right)
- )
-
- def _match_BoolOp(self, left, right):
- return left.op.__class__ is right.op.__class__ and self._all_match(left.values, right.values)
-
- def _match_Compare(self, left, right):
- return (
- len(left.ops) == len(right.ops)
- and all(lop.__class__ is rop.__class__ for lop, rop in zip(left.ops, right.ops))
- and self._match(left.left, right.left)
- and self._all_match(left.comparators, right.comparators)
- )
-
- def _match_IfExp(self, left, right):
- return (
- self._match(left.test, right.test)
- and self._match(left.body, right.body)
- and self._match(left.orelse, right.orelse)
- )
-
- def _match_Attribute(self, left, right):
- return (right.attr is literal_replace.WILDCARD or left.attr == right.attr) and self._match(
- left.value, right.value
- )
-
- #### Subscripting ####
-
- def _match_Subscript(self, left, right):
- return self._match(left.value, right.value) and (
- right.slice is literal_replace.WILDCARD or self._match(left.slice, right.slice)
- )
-
- def _match_Index(self, left, right):
- return right.value is literal_replace.WILDCARD or self._match(left.value, right.value)
-
- def _match_Slice(self, left, right):
- return (
- self._match(left.lower, right.lower)
- and self._match(left.upper, right.upper)
- and self._match(left.step, right.step)
- )
-
- def _match_ExtSlice(self, left, right):
- return self._all_match(left.dims, right.dims)
-
- #### Comprehensions ####
-
- def _match_ListComp(self, left, right):
- return self._match(left.elt, right.elt) and self._all_match(left.generators, right.generators)
-
- def _match_SetComp(self, left, right):
- return self._match_ListComp(left, right)
-
- def _match_DictComp(self, left, right):
- return (
- self._match(left.key, right.key)
- and self._match(left.value, right.value)
- and self._all_match(left.generators, right.generators)
- )
-
- def _match_comprehension(self, left, right):
- return (
- # async is not expected in our use cases, just for completeness
- getattr(left, "is_async", 0) == getattr(right, "is_async", 0)
- and self._match(left.target, right.target)
- and self._match(left.iter, right.iter)
- and self._all_match(left.ifs, right.ifs)
- )
-
-
-def literal_replace(expr, mapping):
- if ast_unparse is None:
- _logger.critical("AST unparse unavailable")
- return expr
- try:
- root = ast.parse(textwrap.dedent(expr), mode="exec")
- except SyntaxError:
- _logger.warning("Invalid Python expression `%s`", expr)
- return expr
- replacer = _Replacer(mapping)
- visited = replacer.visit(root)
- return ast_unparse(visited).strip() if replacer.updated else expr
-
-
-class _Symbol(str):
- def __init__(self, symbol):
- self.id = symbol
-
- def __repr__(self):
- return self
-
-
-_Symbol.__name__ = "Name" # for ast_unparse to render instances
-
-
-literal_replace.WILDCARD = _Symbol("*")
-"""
-Unique object for glob matching
-
-This object should be correctly rendered by `unparse`.
-It can be used in parts where we want a glob match. Example:
-```
->>> util.ast_unparse(ast.Name(util.literal_replace.WILDCARD, None))
-'*'
->>> util.ast_unparse(ast.List([util.literal_replace.WILDCARD], None))
-'[*]'
-```
-
-:meta private: exclude from online docs
-"""
diff --git a/src/util/models.py b/src/util/models.py
deleted file mode 100644
index c92d42a0b..000000000
--- a/src/util/models.py
+++ /dev/null
@@ -1,647 +0,0 @@
-"""
-Utility functions for modifying models.
-
-Model operations are best done in `pre-` script of the involved modules.
-"""
-
-import logging
-import re
-
-from psycopg2 import sql
-
-from . import json
-from .const import ENVIRON
-from .fields import IMD_FIELD_PATTERN, remove_field
-from .helpers import _ir_values_value, _validate_model, model_of_table, table_of_model
-from .indirect_references import indirect_references
-from .inherit import for_each_inherit, inherit_parents
-from .misc import _cached, chunks, log_progress, version_gte
-from .pg import (
- _get_unique_indexes_with,
- column_exists,
- column_type,
- column_updatable,
- explode_execute,
- explode_query_range,
- format_query,
- get_m2m_tables,
- get_value_or_en_translation,
- parallel_execute,
- table_exists,
- update_m2m_tables,
- view_exists,
-)
-
-# avoid namespace clash
-from .pg import rename_table as pg_rename_table
-from .records import _remove_import_export_paths, _rm_refs, remove_records, remove_view, replace_record_references_batch
-from .report import add_to_migration_reports
-
-_logger = logging.getLogger(__name__)
-
-
-@_cached
-def _unknown_model_id(cr):
- order = column_exists(cr, "ir_model", "order")
- extra_columns = ', "order"' if order else ""
- extra_values = ", 'id'" if order else ""
- name_value = (
- "jsonb_build_object('en_US', 'Unknown')" if column_type(cr, "ir_model", "name") == "jsonb" else "'Unknown'"
- )
- cr.execute(
- """
- INSERT INTO ir_model(name, model{})
- SELECT {}, '_unknown'{}
- WHERE NOT EXISTS (SELECT 1 FROM ir_model WHERE model = '_unknown')
- """.format(extra_columns, name_value, extra_values)
- )
- cr.execute("SELECT id FROM ir_model WHERE model = '_unknown'")
- return cr.fetchone()[0]
-
-
-def remove_model(cr, model, drop_table=True, ignore_m2m=()):
- """
- Remove a model and its references from the database.
-
- Some required indirect references to the model are replaced by the *unknown model* -
- an empty model that serves as placeholder for gone models.
-
- :param str model: name of the model to remove
- :param bool drop_table: whether to drop the table of this model
- :param list(str) or str ignore_m2m: list of m2m tables to ignore - not removed, use
- `"*"` to ignore (keep) all m2m tables
- """
- _validate_model(model)
- model_underscore = model.replace(".", "_")
- chunk_size = 1000
- notify = False
- unk_id = _unknown_model_id(cr)
-
- # remove references
- for ir in indirect_references(cr):
- if ir.company_dependent_comodel:
- if ir.company_dependent_comodel == "ir.model":
- # clean on delete set null
- cr.execute("SELECT id FROM ir_model WHERE model = %s", [model])
- [mod_id] = cr.fetchone() or [None]
- if mod_id:
- cr.execute(
- format_query(
- cr,
- """
- UPDATE {table}
- SET {field} = (
- SELECT jsonb_object_agg(
- key,
- CASE
- WHEN value::int4 = %s THEN NULL
- ELSE value::int4
- END)
- FROM jsonb_each_text({field})
- )
- WHERE {field} IS NOT NULL
- AND {field} @? %s
- """,
- table=ir.table,
- field=ir.res_id,
- ),
- [mod_id, "$.* ? (@ == {})".format(mod_id)],
- )
- continue
- if ir.table in ("ir_model", "ir_model_fields", "ir_model_data"):
- continue
- ref_model = model_of_table(cr, ir.table)
-
- query = cr.mogrify(
- """
- SELECT d.res_id
- FROM ir_model_data d
- JOIN "{}" r ON d.model = %s AND d.res_id = r.id
- WHERE d.module != '__export__'
- AND {}
- """.format(ir.table, ir.model_filter(prefix="r.")),
- [ref_model, model],
- ).decode()
- if not ir.set_unknown:
- # If not set_unknown, all records should be deleted.
- query = cr.mogrify(
- 'SELECT id FROM "{}" r WHERE {}'.format(ir.table, ir.model_filter(prefix="r.")), [model]
- ).decode()
-
- cr.execute(query)
- if ir.table == "ir_ui_view":
- for (view_id,) in cr.fetchall():
- remove_view(cr, view_id=view_id, silent=True)
- else:
- # remove in batch
- size = (cr.rowcount + chunk_size - 1) / chunk_size
- it = chunks([id for (id,) in cr.fetchall()], chunk_size, fmt=tuple)
- for sub_ids in log_progress(it, _logger, qualifier=ir.table, size=size):
- remove_records(cr, ref_model, sub_ids)
- _rm_refs(cr, ref_model, sub_ids)
-
- if ir.set_unknown:
- # Link remaining records not linked to a XMLID
- query = cr.mogrify(
- 'SELECT id FROM "{}" r WHERE {}'.format(ir.table, ir.model_filter(prefix="r.")), [model]
- ).decode()
- cr.execute(query)
- ids = [id for (id,) in cr.fetchall()]
- if ids:
- # link to `_unknown` model
- sets, args = zip(
- *[
- ('"{}" = %s'.format(c), v)
- for c, v in [(ir.res_model, "_unknown"), (ir.res_model_id, unk_id)]
- if c
- ]
- )
-
- query = 'UPDATE "{}" SET {} WHERE id IN %s'.format(ir.table, ",".join(sets))
- cr.execute(query, args + (tuple(ids),))
- notify = notify or bool(cr.rowcount)
-
- _remove_import_export_paths(cr, model)
-
- _rm_refs(cr, model)
-
- cr.execute(
- "SELECT id, {} FROM ir_model WHERE model=%s".format(get_value_or_en_translation(cr, "ir_model", "name")),
- [model],
- )
- [mod_id, mod_label] = cr.fetchone() or [None, model]
- if mod_id:
- # some required fk are in "ON DELETE SET NULL/RESTRICT".
- for tbl in ["base_action_rule", "base_automation", "google_drive_config"]:
- if column_exists(cr, tbl, "model_id"):
- cr.execute("DELETE FROM {0} WHERE model_id=%s".format(tbl), [mod_id])
-
- if not version_gte("17.0") and column_exists(cr, "base_automation", "action_server_id"):
- cr.execute(
- """
- DELETE FROM base_automation b_a
- USING ir_act_server action
- WHERE b_a.action_server_id = action.id
- AND action.binding_model_id = %s
- """,
- [mod_id],
- )
-
- cr.execute("DELETE FROM ir_model_relation WHERE model=%s", (mod_id,))
-
- # remove m2m tables
- if ignore_m2m != "*":
- tables = get_m2m_tables(cr, table_of_model(cr, model))
- ignore = set(ignore_m2m)
- for table_name in tables:
- if table_name in ignore:
- _logger.info("remove_model(%r): m2m table %r explicitly ignored", model, table_name)
- continue
- _logger.info("remove_model(%r): dropping m2m table %r", model, table_name)
- cr.execute('DROP TABLE "{}" CASCADE'.format(table_name))
- ENVIRON.setdefault("_gone_m2m", {})[table_name] = model
-
- cr.execute("DELETE FROM ir_model_constraint WHERE model=%s RETURNING id", (mod_id,))
- if cr.rowcount:
- ids = tuple(c for (c,) in cr.fetchall())
- cr.execute("DELETE FROM ir_model_data WHERE model = 'ir.model.constraint' AND res_id IN %s", [ids])
-
- # Drop XML IDs of ir.rule and ir.model.access records that will be cascade-dropped,
- # when the ir.model record is dropped - just in case they need to be re-created
- cr.execute(
- """
- DELETE
- FROM ir_model_data x
- USING ir_rule a
- WHERE x.res_id = a.id
- AND x.model = 'ir.rule'
- AND a.model_id = %s
- """,
- [mod_id],
- )
- cr.execute(
- """
- DELETE
- FROM ir_model_data x
- USING ir_model_access a
- WHERE x.res_id = a.id
- AND x.model = 'ir.model.access'
- AND a.model_id = %s
- """,
- [mod_id],
- )
-
- cr.execute("DELETE FROM ir_model WHERE id=%s", (mod_id,))
-
- cr.execute("DELETE FROM ir_model_data WHERE model=%s AND name=%s", ("ir.model", "model_%s" % model_underscore))
- cr.execute(
- "DELETE FROM ir_model_data WHERE model='ir.model.fields' AND name LIKE %s",
- [(IMD_FIELD_PATTERN % (model_underscore, "%")).replace("_", r"\_")],
- )
- cr.execute("DELETE FROM ir_model_data WHERE model = %s", [model])
- cr.execute("UPDATE ir_model_fields SET relation = '_unknown' WHERE relation = %s", [model])
-
- table = table_of_model(cr, model)
- if drop_table:
- if table_exists(cr, table):
- cr.execute('DROP TABLE "{0}" CASCADE'.format(table))
- elif view_exists(cr, table):
- cr.execute('DROP VIEW "{0}" CASCADE'.format(table))
-
- if notify:
- add_to_migration_reports(
- message="The model {model} ({label}) has been removed. "
- "The linked records (crons, mail templates, automated actions...)"
- " have also been removed (standard) or linked to the '_unknown' model (custom).".format(
- model=model, label=mod_label
- ),
- category="Removed Models",
- )
-
-
-# compat layer...
-delete_model = remove_model
-
-
-def _replace_model_in_computed_custom_fields(cr, source, target):
- # adapt computes of manual fields
- if column_exists(cr, "ir_model_fields", "compute"):
- cr.execute(
- r"""
- UPDATE ir_model_fields
- SET compute = regexp_replace(compute, %s, %s, 'g')
- WHERE state = 'manual'
- AND compute IS NOT NULL
- AND name LIKE 'x\_%%'
- """,
- [
- r"""\yenv\[('|"){}\1\]""".format(re.escape(source)),
- "env['{}']".format(target),
- ],
- )
-
-
-def rename_model(cr, old, new, rename_table=True, ignored_m2ms="ALL_BEFORE_18_1"):
- """
- Rename a model.
-
- Updates all references to the model name in the DB.
-
- If table rename is requested, from saas~18.1+, m2m table are updated too, unless
- ignored. In older versions, m2m tables are skipped unless an empty list is passed.
-
- :param str old: current model name
- :param str new: new model name
- :param bool rename_table: whether to also rename the table of the model
- :param ignored_m2ms: m2m tables to skip. Defaults to `"ALL_BEFORE_18_1"`, which skips
- all in Odoo 18 or below, none in saa~18.1+. For all versions, if
- the value is not the default, skip only the specified m2m tables.
- """
- _validate_model(old)
- _validate_model(new)
- if old in ENVIRON["__renamed_fields"]:
- ENVIRON["__renamed_fields"][new] = ENVIRON["__renamed_fields"].pop(old)
- if rename_table:
- old_table = table_of_model(cr, old)
- new_table = table_of_model(cr, new)
- if new_table != old_table:
- pg_rename_table(cr, old_table, new_table)
- if ignored_m2ms != "ALL_BEFORE_18_1": # explicit value, run the the update
- update_m2m_tables(cr, old_table, new_table, ignored_m2ms)
- elif version_gte("saas~18.1"): # from 18.1 we update by default
- update_m2m_tables(cr, old_table, new_table, ())
-
- updates = [("wkf", "osv")] if table_exists(cr, "wkf") else []
- updates += [(ir.table, ir.res_model) for ir in indirect_references(cr) if ir.res_model]
-
- for table, column in updates:
- cr.execute("SELECT 1 FROM {t} WHERE {c}=%s LIMIT 1".format(t=table, c=column), [old])
- if not cr.fetchone():
- continue
- query = cr.mogrify("UPDATE {t} SET {c}=%s WHERE {c}=%s".format(t=table, c=column), [new, old]).decode()
- explode_execute(cr, query, table=table)
-
- # "model-comma" fields
- cr.execute(
- """
- SELECT model, name
- FROM ir_model_fields
- WHERE ttype='reference'
- """
- )
- for model, column in cr.fetchall():
- table = table_of_model(cr, model)
- if column_updatable(cr, table, column):
- query = cr.mogrify(
- """
- UPDATE "{table}"
- SET "{column}"='{new}' || substring("{column}" FROM '%#",%#"' FOR '#')
- WHERE "{column}" LIKE '{old},%'
- """.format(table=table, column=column, new=new, old=old)
- ).decode()
- explode_execute(cr, query, table=table)
-
- # defaults
- if table_exists(cr, "ir_values"):
- column_read, cast_write = _ir_values_value(cr)
- query = """
- UPDATE ir_values
- SET value = {cast[0]}'{new}' || substring({column} FROM '%#",%#"' FOR '#'){cast[2]}
- WHERE {column} LIKE '{old},%'
- """.format(column=column_read, new=new, old=old, cast=cast_write.partition("%s"))
- cr.execute(query)
-
- # translations
- if table_exists(cr, "ir_translation"):
- cr.execute(
- """
- WITH renames AS (
- SELECT id, type, lang, res_id, src, '{new}' || substring(name FROM '%#",%#"' FOR '#') as new
- FROM ir_translation
- WHERE name LIKE '{old},%'
- )
- UPDATE ir_translation t
- SET name = r.new
- FROM renames r
- LEFT JOIN ir_translation e ON (
- e.type = r.type
- AND e.lang = r.lang
- AND e.name = r.new
- AND CASE WHEN e.type = 'model' THEN e.res_id IS NOT DISTINCT FROM r.res_id
- WHEN e.type = 'selection' THEN e.src IS NOT DISTINCT FROM r.src
- ELSE e.res_id IS NOT DISTINCT FROM r.res_id AND e.src IS NOT DISTINCT FROM r.src
- END
- )
- WHERE t.id = r.id
- AND e.id IS NULL
- """.format(new=new, old=old)
- )
- cr.execute("DELETE FROM ir_translation WHERE name LIKE '{},%'".format(old))
-
- cr.execute(
- """
- UPDATE ir_translation
- SET name=%s
- WHERE name=%s
- AND type IN ('constraint', 'sql_constraint', 'view', 'report', 'rml', 'xsl')
- """,
- [new, old],
- )
-
- old_u = old.replace(".", "_")
- new_u = new.replace(".", "_")
-
- cr.execute(
- "UPDATE ir_model_data SET name=%s WHERE model=%s AND name=%s",
- ("model_%s" % new_u, "ir.model", "model_%s" % old_u),
- )
-
- cr.execute(
- """
- UPDATE ir_model_data
- SET name=%s || substring(name from %s)
- WHERE model='ir.model.fields'
- AND name LIKE %s
- """,
- ["field_%s" % new_u, len(old_u) + 7, (IMD_FIELD_PATTERN % (old_u, "%")).replace("_", r"\_")],
- )
-
- col_prefix = ""
- if not column_exists(cr, "ir_act_server", "condition"):
- col_prefix = "--" # sql comment the line
-
- cr.execute(
- r"""
- UPDATE ir_act_server
- SET {col_prefix} condition=regexp_replace(condition, '([''"]){old}\1', '\1{new}\1', 'g'),
- code=regexp_replace(code, '([''"]){old}\1', '\1{new}\1', 'g')
- """.format(col_prefix=col_prefix, old=old.replace(".", r"\."), new=new)
- )
-
- _replace_model_in_computed_custom_fields(cr, old, new)
-
-
-def merge_model(cr, source, target, drop_table=True, fields_mapping=None, ignore_m2m=()):
- """
- Merge a model into another one.
-
- This function moves all the references from ``source`` model into ``target`` model and
- *removes* ``source`` model and its references. By default, only the fields with the
- same name in both models are moved from source to target, optionally a mapping with
- differently named fields can be provided.
-
- .. warning::
- This function does not move the records from ``source`` model to ``target`` model.
-
- :param str source: name of the model to be merged
- :param str target: name of the destination model to merge into
- :param bool drop_table: whether to drop the table of the source model
- :param dict or None fields_mapping: mapping of field names from source to target
- model, when `None` only fields with same name are
- moved
- :param list(str) or str ignore_m2m: list of m2m tables ignored to remove from source model.
- """
- _validate_model(source)
- _validate_model(target)
- cr.execute("SELECT model, id FROM ir_model WHERE model in %s", ((source, target),))
- model_ids = dict(cr.fetchall())
- mapping = {model_ids[source]: model_ids[target]}
-
- ir_model_tables = ["ir_model", "ir_model_fields", "ir_model_constraint", "ir_model_relation"]
- ignores = ir_model_tables + [ir.table for ir in indirect_references(cr, bound_only=True) if ir.res_model_id]
-
- replace_record_references_batch(cr, mapping, "ir.model", replace_xmlid=False, ignores=ignores)
-
- # remap the fields on ir_model_fields
- cr.execute(
- """
- SELECT mf1.id,
- mf2.id
- FROM ir_model_fields mf1
- JOIN ir_model_fields mf2
- ON mf1.model=%s
- AND mf2.model=%s
- AND mf1.name=mf2.name
- """,
- [source, target],
- )
- field_ids_mapping = dict(cr.fetchall())
- if fields_mapping:
- jmap = json.dumps(fields_mapping)
- cr.execute(
- """
- SELECT mf1.name,
- mf1.id,
- mf2.id
- FROM ir_model_fields mf1
- JOIN ir_model_fields mf2
- ON mf1.model=%s
- AND mf2.model=%s
- AND mf2.name=(%s::jsonb ->> mf1.name::varchar)::varchar
- """,
- [source, target, jmap],
- )
- explicit_mapping = {name: (id_s, id_t) for name, id_s, id_t in cr.fetchall()}
- field_ids_mapping.update(explicit_mapping.values())
- # The target fields may not yet exists. We simply update the source fields in this case.
- missing = {s: t for s, t in fields_mapping.items() if s not in explicit_mapping}
- if missing:
- jmap = json.dumps(missing)
- cr.execute(
- """
- UPDATE ir_model_fields
- SET model = %s,
- model_id = %s,
- name = (%s::jsonb ->> name::varchar)::varchar
- WHERE model = %s
- AND name IN %s
- RETURNING id
- """,
- [target, model_ids[target], jmap, source, tuple(missing)],
- )
- if cr.rowcount:
- # remove their xmlid (now invalid names). The ORM will recreate them.
- cr.execute(
- "DELETE FROM ir_model_data WHERE model='ir.model.fields' AND res_id IN %s",
- [sum(cr.fetchall(), ())],
- )
-
- if field_ids_mapping:
- replace_record_references_batch(
- cr,
- field_ids_mapping,
- "ir.model.fields",
- replace_xmlid=False,
- ignores=["ir_model_fields_group_rel", "ir_model_fields_selection"],
- )
-
- for ir in indirect_references(cr):
- if ir.res_model and not ir.res_id and ir.table not in ir_model_tables:
- # only update unbound references, other ones have been updated by the call to
- # `replace_record_references_batch`
- where_clauses = []
- for _, uniqs in _get_unique_indexes_with(cr, ir.table, ir.res_model):
- sub_where = " AND ".join("o.{0} = t.{0}".format(a) for a in uniqs if a != ir.res_model) or "true"
- where_clauses.append(
- "NOT EXISTS(SELECT 1 FROM {t} o WHERE {w} AND o.{c}=%(new)s)".format(
- t=ir.table, c=ir.res_model, w=sub_where
- )
- )
- where = " AND ".join(where_clauses) or "true"
- query = "UPDATE {t} t SET {c}=%(new)s WHERE {w} AND {c}=%(old)s".format(t=ir.table, c=ir.res_model, w=where)
- fmt_query = cr.mogrify(query, {"new": target, "old": source}).decode()
- if column_exists(cr, ir.table, "id"):
- parallel_execute(cr, explode_query_range(cr, fmt_query, table=ir.table, alias="t"))
- else:
- cr.execute(fmt_query)
-
- _replace_model_in_computed_custom_fields(cr, source, target)
-
- # Adapt translations
- if table_exists(cr, "ir_translation"):
- cr.execute(
- """
- UPDATE ir_translation it
- SET name = %(target)s || substr(name, %(len_source)s)
- WHERE it.name LIKE %(source_like)s
- AND NOT EXISTS (
- -- Some translations may already exist in the target model
- SELECT 1 FROM ir_translation itt
- WHERE itt.name = %(target)s || substr(it.name, %(len_source)s)
- AND itt.type = it.type
- AND itt.lang = it.lang
- AND itt.res_id = it.res_id
- )
- """,
- {
- "target": target,
- "source_like": source.replace("_", r"\_") + ",%",
- "len_source": len(source) + 1,
- },
- )
-
- remove_model(cr, source, drop_table=drop_table, ignore_m2m=ignore_m2m)
-
-
-def remove_inherit_from_model(cr, model, inherit, keep=(), skip_inherit=(), with_inherit_parents=True):
- """
- Remove ``inherit`` from ``model``.
-
- This function removes all fields inherited via ``inherit`` from a ``model`` and all
- its descendant models. All fields from the inherit, including those from its parent
- models are also removed, unless ``with_inherit_parents`` mode is unset. In such case
- only fields from ``inherit`` are removed, excluding fields in its parents. Fields
- listed in ``keep`` are never removed. If some descendants from ``model`` are listed in
- ``skip_inherit`` they will keep the fields from ``inherit``.
-
- :param str model: name of the model from which to remove the inherit
- :param str inherit: name of the inherited model (or mixin) to remove
- :param tuple(str) keep: tuple with field names to keep
- :param tuple(str) skip_inherit: list of descendant models of ``model`` to not process
- :param boolean with_inherit_parents: if unset, remove fields coming from ``inherit``
- only, keeping all fields from its parents
- """
- _validate_model(model)
- _validate_model(inherit)
-
- parents = set(inherit_parents(cr, inherit, interval="[]"))
- inherit_models = {inherit} | (parents if with_inherit_parents else set())
- keep = set(keep)
- if not with_inherit_parents and parents:
- cr.execute("SELECT name FROM ir_model_fields WHERE model IN %s GROUP BY name", [tuple(parents)])
- keep.update(r[0] for r in cr.fetchall())
-
- cr.execute(
- """
- SELECT name, ttype, relation, store
- FROM ir_model_fields
- WHERE model IN %s
- AND name NOT IN ('id',
- 'create_uid', 'write_uid',
- 'create_date', 'write_date',
- '__last_update', 'display_name')
- AND name != ALL(%s)
- """,
- [tuple(inherit_models), list(keep)],
- )
- for field, ftype, relation, store in cr.fetchall():
- if ftype == "one2many" and store:
- # for mixin, one2many are filtered by their model and res_id.
- table = table_of_model(cr, relation)
- irs = [
- ir
- for ir in indirect_references(cr)
- if ir.table == table
- and ir.table != "ir_attachment" # delegated to the `remove_field` call
- and ir.res_id is not None
- and not ir.company_dependent_comodel
- ]
- for ir in irs:
- query = cr.mogrify(
- format_query(cr, "DELETE FROM {} WHERE ({})", ir.table, sql.SQL(ir.model_filter())), [model]
- ).decode()
- explode_execute(cr, query, table=table)
- remove_field(cr, model, field, skip_inherit="*") # inherits will be removed by the recursive call.
-
- # down on inherits of `model`
- for inh in for_each_inherit(cr, model, skip_inherit):
- remove_inherit_from_model(
- cr, inh.model, inherit, keep=keep, skip_inherit=skip_inherit, with_inherit_parents=with_inherit_parents
- )
-
-
-def convert_model_to_abstract(cr, model, drop_table=True, keep=()):
- _validate_model(model)
-
- if drop_table:
- table = table_of_model(cr, model)
- if table_exists(cr, table):
- cr.execute('DROP TABLE "{0}" CASCADE'.format(table))
- elif view_exists(cr, table):
- cr.execute('DROP VIEW "{0}" CASCADE'.format(table))
-
- # Even if `__last_update` can be changed on the model definition, we hardcode the name.
- # I'm not aware of any (standard) model that modify it. The field will need to be removed explicitly if it happen.
- for field in ["id", "__last_update", "display_name", "create_uid", "create_date", "write_uid", "write_date"]:
- if field not in keep:
- remove_field(cr, model, field)
diff --git a/src/util/modules.py b/src/util/modules.py
deleted file mode 100644
index 7c06c8ed0..000000000
--- a/src/util/modules.py
+++ /dev/null
@@ -1,1230 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-Utility functions for module-level operations.
-
-In most cases module operations (rename, merge, remove, ...) should be performed in a
-`base` script. The reason is that once the `base` module is upgraded, all the information
-regarding modules should be already set in the DB for the upgrade process to function
-correctly. The command line option `--pre-upgrade-scripts` (available from Odoo 16) allows
-to run upgrade scripts *before* loading base. This is the recommended way to perform
-module operations after a major upgrade.
-"""
-
-try:
- from collections.abc import Sequence, Set
-except ImportError:
- from collections import Sequence, Set
-
-import functools
-import itertools
-import logging
-import os
-import warnings
-from inspect import currentframe
-from operator import itemgetter
-
-try:
- import odoo
- from odoo.modules.db import create_categories
- from odoo.tools.func import frame_codeinfo
- from odoo.tools.misc import topological_sort
-
- try:
- from odoo.modules import get_manifest
- except ImportError:
- from odoo.modules import load_information_from_description_file as get_manifest
-except ImportError:
- import openerp as odoo
- from openerp.modules import load_information_from_description_file as get_manifest
- from openerp.modules.db import create_categories
- from openerp.tools.func import frame_codeinfo
-
- try:
- from openerp.tools.misc import topological_sort
- except ImportError:
- from openerp.addons.web.controllers.main import module_topological_sort as topological_sort
-
-from .const import ENVIRON, NEARLYWARN
-from .exceptions import MigrationError, SleepyDeveloperError, UnknownModuleError, UpgradeWarning
-from .fields import remove_field
-from .helpers import _validate_model, table_of_model
-from .misc import on_CI, parse_version, str2bool, version_gte
-from .models import delete_model
-from .orm import env, flush
-from .pg import SQLStr, column_exists, format_query, table_exists, target_of
-from .records import ref, remove_group, remove_menus, remove_records, remove_view, replace_record_references_batch
-
-INSTALLED_MODULE_STATES = ("installed", "to install", "to upgrade")
-_logger = logging.getLogger(__name__)
-
-ENVIRON.setdefault("AUTO_DISCOVERY_RAN", False)
-
-if version_gte("15.0"):
- AUTO_INSTALL = os.getenv("UPG_AUTOINSTALL")
- _NO_AUTOINSTALL = os.getenv("UPG_NO_AUTOINSTALL")
- if AUTO_INSTALL:
- if _NO_AUTOINSTALL:
- _logger.info("Ignoring the deprecated `UPG_NO_AUTOINSTALL` env variable as `UPG_AUTOINSTALL` is also set.")
- elif _NO_AUTOINSTALL:
- AUTO_INSTALL = "only_link_modules" if str2bool(_NO_AUTOINSTALL) else "all"
- # TODO add link to doc
- _logger.warning(
- "Usage of the `UPG_NO_AUTOINSTALL` environment variable is deprecated. Please use `UPG_AUTOINSTALL=%s`.",
- AUTO_INSTALL,
- )
- else:
- AUTO_INSTALL = "all"
-else:
- AUTO_INSTALL = "all"
-
-if AUTO_INSTALL not in ("all", "only_link_modules", "none"):
- raise ValueError("Invalid value for the `UPG_AUTOINSTALL` environment variable: {!r}".format(AUTO_INSTALL))
-
-_logger.info("module auto-install strategy: %s", AUTO_INSTALL)
-
-# python3 shims
-try:
- basestring # noqa: B018
-except NameError:
- basestring = str
-
-
-if version_gte("16.0"):
- from odoo.modules.registry import Registry as _Registry
-
- def _warn_usage_outside_base(f):
- @functools.wraps(f)
- def wrapper(cr, *args, **kwargs):
- if "base" in _Registry(cr.dbname)._init_modules:
- warnings.warn(
- "Calling `{}` outside the module base can lead to unexpected results".format(f.__name__),
- UpgradeWarning,
- stacklevel=2,
- )
- return f(cr, *args, **kwargs)
-
- return wrapper
-
-else:
- # deactivated on old versions, to avoid modification of legacy upgrade scripts.
- def _warn_usage_outside_base(f):
- return f
-
-
-def modules_installed(cr, *modules):
- """
- Return whether *all* given modules are installed.
-
- :param list(str) modules: names of the modules to check
- :rtype: bool
-
- .. note::
-
- In the context of upgrades a module is considered as installed if it is marked for
- upgrade, or for installation; even if they are not yet fully installed.
- """
- assert modules
- cr.execute(
- """
- SELECT count(1)
- FROM ir_module_module
- WHERE name IN %s
- AND state IN %s
- """,
- [modules, INSTALLED_MODULE_STATES],
- )
- return cr.fetchone()[0] == len(modules)
-
-
-def module_installed(cr, module):
- """
- Return whether a module is installed.
-
- :param str module: name of the module to check
- :rtype: bool
-
- See :func:`modules_installed`.
- """
- return modules_installed(cr, module)
-
-
-@_warn_usage_outside_base
-def uninstall_module(cr, module):
- """
- Uninstall and remove all records owned by a module.
-
- :param str module: name of the module to uninstall
- """
- cr.execute("SELECT id FROM ir_module_module WHERE name=%s", (module,))
- (mod_id,) = cr.fetchone() or [None]
- if not mod_id:
- return
-
- # delete constraints only owned by this module
- cr.execute(
- """
- SELECT name
- FROM ir_model_constraint
- GROUP BY name
- HAVING array_agg(module) = %s
- """,
- ([mod_id],),
- )
-
- constraints = tuple(map(itemgetter(0), cr.fetchall()))
- if constraints:
- cr.execute(
- """
- SELECT table_name, constraint_name
- FROM information_schema.table_constraints
- WHERE constraint_name IN %s
- """,
- [constraints],
- )
- for table, constraint in cr.fetchall():
- cr.execute('ALTER TABLE "%s" DROP CONSTRAINT "%s"' % (table, constraint))
-
- cr.execute(
- """
- DELETE
- FROM ir_model_constraint
- WHERE module = %s
- """,
- [mod_id],
- )
-
- # delete data
- model_ids, field_ids, menu_ids = [], [], []
-
- # some models' data needs to be remove before or after others.
- firsts = ["ir.rule"]
- lasts = ["ir.actions.server"]
-
- cr.execute(
- """
- SELECT model, res_id
- FROM ir_model_data d
- WHERE NOT EXISTS (SELECT 1
- FROM ir_model_data
- WHERE id != d.id
- AND res_id = d.res_id
- AND model = d.model
- AND module != d.module)
- AND module = %s
- AND model != 'ir.module.module'
- ORDER BY array_position(%s::text[], model::text) NULLS LAST,
- array_position(%s::text[], model::text) NULLS FIRST,
- id DESC
- """,
- [module, firsts, lasts],
- )
- to_group = []
- for model, res_id in cr.fetchall():
- if model == "ir.model":
- model_ids.append(res_id)
- elif model == "ir.model.fields":
- field_ids.append(res_id)
- elif model == "ir.ui.menu":
- menu_ids.append(res_id)
- else:
- to_group.append((model, res_id))
-
- for model, group in itertools.groupby(to_group, lambda it: it[0]):
- if model == "ir.ui.view":
- for _, res_id in group:
- remove_view(cr, view_id=res_id, silent=True)
- elif model == "res.groups":
- for _, res_id in group:
- remove_group(cr, group_id=res_id)
- else:
- remove_records(cr, model, [it[1] for it in group])
-
- if menu_ids:
- remove_menus(cr, menu_ids)
-
- # remove relations
- cr.execute(
- """
- SELECT name
- FROM ir_model_relation
- GROUP BY name
- HAVING array_agg(module) = %s
- """,
- ([mod_id],),
- )
- relations = tuple(map(itemgetter(0), cr.fetchall()))
- cr.execute("DELETE FROM ir_model_relation WHERE module=%s", (mod_id,))
- if relations:
- cr.execute("SELECT table_name FROM information_schema.tables WHERE table_name IN %s", (relations,))
- for (rel,) in cr.fetchall():
- cr.execute('DROP TABLE "%s" CASCADE' % (rel,))
-
- if model_ids:
- cr.execute("SELECT model FROM ir_model WHERE id IN %s", [tuple(model_ids)])
- for (model,) in cr.fetchall():
- delete_model(cr, model)
-
- if field_ids:
- cr.execute("SELECT model, name FROM ir_model_fields WHERE id IN %s", [tuple(field_ids)])
- for model, name in cr.fetchall():
- if name == "id":
- delete_model(cr, model)
- else:
- remove_field(cr, model, name)
-
- cr.execute("DELETE FROM ir_model_data WHERE module=%s", (module,))
- if table_exists(cr, "ir_translation"):
- cr.execute("DELETE FROM ir_translation WHERE module=%s", [module])
- cr.execute("UPDATE ir_module_module SET state='uninstalled' WHERE name=%s", (module,))
-
-
-def uninstall_theme(cr, theme, base_theme=None):
- """
- Uninstall a theme module and remove it from websites.
-
- :param str theme: name of the theme module to uninstall
- :param str or None base_theme: if not `None`, unload first this base theme
-
- .. warning::
-
- This function can only be used in `post-` scripts of `website` module as it relies
- on the ORM.
-
- See :func:`remove_theme` and :func:`uninstall_module`.
- """
- cr.execute("SELECT id FROM ir_module_module WHERE name=%s AND state in %s", [theme, INSTALLED_MODULE_STATES])
- (theme_id,) = cr.fetchone() or [None]
- if not theme_id:
- return
-
- env_ = env(cr)
- IrModuleModule = env_["ir.module.module"]
- if base_theme:
- cr.execute("SELECT id FROM ir_module_module WHERE name=%s", (base_theme,))
- (website_theme_id,) = cr.fetchone() or [None]
- theme_extension = IrModuleModule.browse(theme_id)
- for website in env_["website"].search([("theme_id", "=", website_theme_id)]):
- theme_extension._theme_unload(website)
- else:
- websites = env_["website"].search([("theme_id", "=", theme_id)])
- for website in websites:
- IrModuleModule._theme_remove(website)
- flush(env_["base"])
- with warnings.catch_warnings():
- warnings.filterwarnings(action="http://github.com/ignore", category=UpgradeWarning)
- uninstall_module(cr, theme)
-
-
-@_warn_usage_outside_base
-def remove_module(cr, module):
- """
- Completely remove a module.
-
- This operation is equivalent to uninstall and removal of *all* references to
- the module - no trace of it is left in the database.
-
- :param str module: name of the module to remove
-
- .. warning::
- Since this function removes *all* data associated to the module. Ensure to
- reassign records before calling this function.
- """
- # NOTE: we cannot use the uninstall of module because the given
- # module need to be currently installed and running as deletions
- # are made using orm.
-
- uninstall_module(cr, module)
- cr.execute("DELETE FROM ir_module_module_dependency WHERE name=%s", (module,))
- cr.execute("DELETE FROM ir_module_module WHERE name=%s RETURNING id", (module,))
- if cr.rowcount:
- [mod_id] = cr.fetchone()
- cr.execute("DELETE FROM ir_model_data WHERE model='ir.module.module' AND res_id=%s", [mod_id])
-
- ENVIRON["__modules_auto_discovery_force_installs"].discard(module)
- ENVIRON["__modules_auto_discovery_force_upgrades"].pop(module, None)
-
-
-def remove_theme(cr, theme, base_theme=None):
- """
- Uninstall a theme module.
-
- .. warning::
- This function can only be used in `post-` scripts.
-
- See :func:`remove_module` and :func:`uninstall_theme`.
- """
- uninstall_theme(cr, theme, base_theme=base_theme)
- cr.execute("DELETE FROM ir_module_module_dependency WHERE name=%s", (theme,))
- cr.execute("DELETE FROM ir_module_module WHERE name=%s RETURNING id", (theme,))
- if cr.rowcount:
- [mod_id] = cr.fetchone()
- cr.execute("DELETE FROM ir_model_data WHERE model='ir.module.module' AND res_id=%s", [mod_id])
-
- ENVIRON["__modules_auto_discovery_force_installs"].discard(theme)
- ENVIRON["__modules_auto_discovery_force_upgrades"].pop(theme, None)
-
-
-def _update_view_key(cr, old, new):
- # update view key for renamed & merged modules, also handle multi-website
- # COWed views.
- # View key is not always equal to it's xml_id (eg when created through a
- # website.page record, the key is the page xml_id suffixed by `_view`)
- if not column_exists(cr, "ir_ui_view", "key"):
- return
- like_old = old.replace("_", r"\_").replace("%", r"\%")
- cr.execute(
- """
- UPDATE ir_ui_view
- SET key = concat('{new}', right(key, -length('{old}')))
- WHERE key LIKE '{like_old}.%'
- """.format(old=old, new=new, like_old=like_old)
- )
-
-
-@_warn_usage_outside_base
-def rename_module(cr, old, new):
- """
- Rename a module and all references to it.
-
- :param str old: current name of the module to rename
- :param str new: new name of the module to rename
- """
- cr.execute("UPDATE ir_module_module SET name=%s WHERE name=%s", (new, old))
- cr.execute("UPDATE ir_module_module_dependency SET name=%s WHERE name=%s", (new, old))
- _update_view_key(cr, old, new)
- cr.execute("UPDATE ir_model_data SET module=%s WHERE module=%s", (new, old))
- if table_exists(cr, "ir_translation"):
- cr.execute("UPDATE ir_translation SET module=%s WHERE module=%s", [new, old])
-
- mod_old = "module_" + old
- mod_new = "module_" + new
- cr.execute(
- """
- UPDATE ir_model_data
- SET name = %s
- WHERE name = %s
- AND module = %s
- AND model = %s
- """,
- [mod_new, mod_old, "base", "ir.module.module"],
- )
-
- fi = ENVIRON["__modules_auto_discovery_force_installs"]
- if old in fi:
- fi.remove(old)
- fi.add(new)
- fu = ENVIRON["__modules_auto_discovery_force_upgrades"]
- if old in fu:
- fu[new] = fu.pop(old)
-
-
-@_warn_usage_outside_base
-def merge_module(cr, old, into, update_dependers=True):
- """
- Merge a module into another.
-
- This function moves all references and records from the source module to the
- destination module.
-
- .. warning::
- This functions does not remove any record, but it removes xml_ids from the source
- module with a conflicting name in the destination module.
-
- :param str old: name of the module to be merged
- :param str into: name of the module to merge into
- :param bool update_dependers: whether the dependencies of modules that depend on `old`
- are updated
- """
- cr.execute("SELECT name, id FROM ir_module_module WHERE name IN %s", [(old, into)])
- mod_ids = dict(cr.fetchall())
-
- fi = ENVIRON["__modules_auto_discovery_force_installs"]
- if old in fi:
- fi.remove(old)
- fi.add(into)
- fu = ENVIRON["__modules_auto_discovery_force_upgrades"]
- if old in fu:
- if into not in fu:
- fu[into] = fu.pop(old)
- else:
- init = fu[old][0] or fu[into][0] # keep init flag
- # keep the min version, this controls which upgrade scripts run
- version = fu[old][1] if parse_version(fu[old][1]) < parse_version(fu[into][1]) else fu[into][1]
- del fu[old]
- fu[into] = (init, version)
-
- if old not in mod_ids:
- # this can happen in case of temp modules added after a release if the database does not
- # know about this module, i.e: account_full_reconcile in 9.0
- _logger.log(NEARLYWARN, "Unknown module %s. Skip merge into %s.", old, into)
- return
-
- if into not in mod_ids:
- raise UnknownModuleError(into)
-
- def _up(table, old, new):
- cr.execute(
- """
- UPDATE ir_model_{0} x
- SET module=%s
- WHERE module=%s
- AND NOT EXISTS(SELECT 1
- FROM ir_model_{0} y
- WHERE y.name = x.name
- AND y.module = %s)
- """.format(table),
- [new, old, new],
- )
-
- if table == "data":
- # remove xmlids pointing to the same records
- cr.execute(
- """
- DELETE
- FROM ir_model_data o
- USING ir_model_data n
- WHERE o.module = %s
- AND n.module = %s
- AND o.model = n.model
- AND o.res_id = n.res_id
- """,
- [old, new],
- )
- # merge records when they have the same model
- cr.execute(
- """
- SELECT o.model,
- jsonb_object_agg(o.res_id, n.res_id)
- FROM ir_model_data o
- JOIN ir_model_data n
- ON n.model = o.model
- AND n.name = o.name
- WHERE o.module = %s
- AND n.module = %s
- AND o.model NOT LIKE 'ir.model%%'
- AND o.model NOT LIKE 'ir.module.module%%'
- AND o.model NOT IN ('ir.ui.view', 'ir.ui.menu')
- GROUP BY o.model
- """,
- [old, new],
- )
- for model, mapping in cr.fetchall():
- replace_record_references_batch(
- cr,
- {int(f): int(t) for f, t in mapping.items()}, # jsonb keys are always string
- model,
- replace_xmlid=False,
- )
-
- # remove remaining records
- cr.execute(
- """
- SELECT model, array_agg(res_id)
- FROM ir_model_data
- WHERE module=%s
- AND model NOT LIKE 'ir.model%%'
- AND model NOT LIKE 'ir.module.module%%'
- GROUP BY model
- """,
- [old],
- )
- for model, res_ids in cr.fetchall():
- if model == "ir.ui.view":
- for v in res_ids:
- remove_view(cr, view_id=v, silent=True)
- elif model == "ir.ui.menu":
- remove_menus(cr, tuple(res_ids))
- else:
- remove_records(cr, model, res_ids)
-
- cr.execute("DELETE FROM ir_model_{0} WHERE module=%s".format(table), [old])
-
- _up("constraint", mod_ids[old], mod_ids[into])
- _up("relation", mod_ids[old], mod_ids[into])
- _up("data", old, into)
- _update_view_key(cr, old, into)
- if table_exists(cr, "ir_translation"):
- cr.execute("UPDATE ir_translation SET module=%s WHERE module=%s", [into, old])
-
- # update dependencies of modules that depends on $old
- if update_dependers:
- cr.execute(
- """
- UPDATE ir_module_module_dependency d
- SET name = %s
- FROM ir_module_module m
- WHERE m.id = d.module_id
- AND d.name = %s
- AND m.name != %s
- AND NOT EXISTS(SELECT 1
- FROM ir_module_module_dependency o
- WHERE o.module_id = d.module_id
- AND o.name = %s)
- """,
- [into, old, into, into],
- )
-
- cr.execute("DELETE FROM ir_module_module WHERE name=%s RETURNING state", [old])
- [state] = cr.fetchone()
- cr.execute("DELETE FROM ir_module_module_dependency WHERE name=%s", [old])
- cr.execute("DELETE FROM ir_model_data WHERE model='ir.module.module' AND res_id=%s", [mod_ids[old]])
- if state in INSTALLED_MODULE_STATES:
- force_install_module(cr, into, reason="installed {!r} module has been merged into it".format(old))
-
-
-def force_install_module(cr, module, if_installed=None, reason="it has been explicitly asked for"):
- """
- Force the ORM to install a module.
-
- :param str module: name of the module to install
- :param list(str) or None if_installed: only force the install when these modules are
- already installed
- :return str: the *new* state of the module
- """
- if version_gte("saas~14.5"):
- if if_installed:
- _assert_modules_exists(cr, *if_installed)
- if not if_installed or modules_installed(cr, *if_installed):
- cr.execute("SELECT 1 FROM ir_module_module WHERE name = 'base' AND state != 'to upgrade'")
- if cr.rowcount:
- if not ENVIRON.get("AUTO_DISCOVERY_RAN"):
- # run the autodiscovery once to allow to force install _new_ modules
- _trigger_auto_discovery(cr)
- elif ENVIRON.get("AUTO_DISCOVERY_UPGRADE"):
- raise MigrationError("`force_install_module` can only be called from pre/post of `base`")
- return _force_install_module(cr, module, reason="{} (done outside of a major upgrade)".format(reason))
- ENVIRON["__modules_auto_discovery_force_installs"].add(module)
- return None
- else:
- return _force_install_module(cr, module, if_installed, reason)
-
-
-def _force_install_module(cr, module, if_installed=None, reason="it has been explicitly asked for"):
- """Strict implementation of force_install: raise errors for missing modules."""
- _assert_modules_exists(cr, module)
-
- cr.execute("SELECT state FROM ir_module_module WHERE name = %s", [module])
- mod_state = cr.fetchone()[0]
- if mod_state in INSTALLED_MODULE_STATES:
- return mod_state
-
- subquery = ""
- params = [module]
- if if_installed:
- try:
- _assert_modules_exists(cr, *if_installed)
- except UnknownModuleError as e:
- # Warn about unknown modules (can help track typos in the call).
- # we don't care about missing modules, as by definition, they are not installed.
- _logger.log(
- NEARLYWARN, "Unknown 'if_installed' modules: %s; consider them as uninstalled.", ", ".join(e.args)
- )
- subquery = """AND EXISTS(SELECT 1 FROM ir_module_module
- WHERE name IN %s
- AND state IN %s)"""
- params.extend((tuple(if_installed), INSTALLED_MODULE_STATES))
-
- query = format_query(
- cr,
- """
- WITH RECURSIVE deps (mod_id, dep_name) AS (
- SELECT m.id, d.name from ir_module_module_dependency d
- JOIN ir_module_module m on (d.module_id = m.id)
- WHERE m.name = %s
- UNION
- SELECT m.id, d.name from ir_module_module m
- JOIN deps ON deps.dep_name = m.name
- JOIN ir_module_module_dependency d on (d.module_id = m.id)
- )
- UPDATE ir_module_module m
- SET state = CASE WHEN state = 'to remove' THEN 'to upgrade'
- WHEN state = 'uninstalled' THEN 'to install'
- ELSE state
- END,
- demo=(select demo from ir_module_module where name='base')
- FROM deps d
- WHERE m.id = d.mod_id
- {0}
- RETURNING m.name, m.state
- """,
- SQLStr(subquery),
- )
- cr.execute(query, params)
-
- states = dict(cr.fetchall())
- toinstall = [m for m in states if states[m] == "to install"]
-
- if module in toinstall:
- _logger.info(
- "force install of module %r (and its dependencies) because %s%s",
- module,
- reason,
- " and modules {!r} are already installed".format(if_installed) if if_installed else "",
- )
-
- # auto_install modules...
- if AUTO_INSTALL in ("all", "only_link_modules") and toinstall:
- # Same algo as ir.module.module.button_install(): https://git.io/fhCKd
- dep_match = ""
- if column_exists(cr, "ir_module_module_dependency", "auto_install_required"):
- dep_match = "AND d.auto_install_required = TRUE AND e.auto_install_required = TRUE"
-
- country_match = country_join = ""
- if table_exists(cr, "module_country"):
- country_join = """
- LEFT JOIN module_country mc
- ON mc.module_id = on_me.id
- LEFT JOIN (
- SELECT c.id, p.country_id
- FROM res_company c
- JOIN res_partner p
- ON p.id = c.partner_id
- WHERE c.active
- ) AS c
- ON c.country_id = mc.country_id
- """
- country_match = """
- AND (
- count(mc) = 0 -- no country set
- OR count(c) > 0 -- or have active companies with a matching country
- )
- """
-
- cat_match = ""
- if AUTO_INSTALL == "only_link_modules":
- # even if we skip auto installs, we still need to auto install the real link-modules.
- # those are in the "Hidden" category
- hidden = ref(cr, "base.module_category_hidden")
- cat_match = cr.mogrify("AND on_me.category_id = %s", [hidden]).decode()
-
- cr.execute(
- """
- SELECT on_me.name
- FROM ir_module_module_dependency d
- JOIN ir_module_module on_me ON on_me.id = d.module_id
- JOIN ir_module_module_dependency e ON e.module_id = on_me.id
- JOIN ir_module_module its_deps ON its_deps.name = e.name
- {}
- WHERE d.name = ANY(%s)
- AND on_me.state = 'uninstalled'
- AND on_me.auto_install = TRUE
- {}
- {}
- GROUP BY on_me.name
- HAVING
- -- are all dependencies (to be) installed?
- array_agg(its_deps.state)::text[] <@ %s
- {}
-
- """.format(country_join, dep_match, cat_match, country_match),
- [toinstall, list(INSTALLED_MODULE_STATES)],
- )
- for (mod,) in cr.fetchall():
- _force_install_module(
- cr,
- mod,
- reason=(
- "it is an auto install module that got all its auto install dependencies installed "
- "by the force install of {!r}"
- ).format(module),
- )
-
- # TODO handle module exclusions
-
- return states.get(module)
-
-
-def _assert_modules_exists(cr, *modules):
- assert modules
- cr.execute("SELECT name FROM ir_module_module WHERE name IN %s", [modules])
- existing_modules = {m[0] for m in cr.fetchall()}
- unexisting_modules = set(modules) - existing_modules
- if unexisting_modules:
- raise UnknownModuleError(*sorted(unexisting_modules))
-
-
-@_warn_usage_outside_base
-def new_module_dep(cr, module, new_dep):
- assert isinstance(new_dep, basestring)
- _assert_modules_exists(cr, module, new_dep)
- # One new dep at a time
- cr.execute(
- """
- INSERT INTO ir_module_module_dependency(name, module_id)
- SELECT %s, id
- FROM ir_module_module m
- WHERE name=%s
- AND NOT EXISTS(SELECT 1
- FROM ir_module_module_dependency
- WHERE module_id = m.id
- AND name=%s)
- """,
- [new_dep, module, new_dep],
- )
-
- if not version_gte("saas~14.5"):
- # Update new_dep state depending on module state
- cr.execute("SELECT state FROM ir_module_module WHERE name = %s", [module])
- mod_state = (cr.fetchone() or ["n/a"])[0]
- if mod_state in INSTALLED_MODULE_STATES:
- # Module was installed, need to install all its deps, recursively,
- # to make sure the new dep is installed
- _force_install_module(cr, new_dep, reason="it's a new dependency of {!r}".format(module))
-
-
-@_warn_usage_outside_base
-def remove_module_deps(cr, module, old_deps):
- assert isinstance(old_deps, (Sequence, Set)) and not isinstance(old_deps, basestring)
- # As the goal is to have dependencies removed, the objective is reached even when they don't exist.
- # Therefore, we don't need to assert their existence (at the cost of missing typos).
- cr.execute(
- """
- DELETE
- FROM ir_module_module_dependency
- WHERE module_id = (SELECT id
- FROM ir_module_module
- WHERE name = %s)
- AND name IN %s
- """,
- [module, tuple(old_deps)],
- )
-
-
-@_warn_usage_outside_base
-def module_deps_diff(cr, module, plus=(), minus=()):
- for new_dep in plus:
- new_module_dep(cr, module, new_dep)
- if minus:
- remove_module_deps(cr, module, tuple(minus))
-
-
-@_warn_usage_outside_base
-def module_auto_install(cr, module, auto_install):
- if column_exists(cr, "ir_module_module_dependency", "auto_install_required"):
- params = []
- if auto_install is True:
- value = "TRUE"
- elif auto_install:
- value = "(name = ANY(%s))"
- params = [list(auto_install)]
- else:
- value = "FALSE"
-
- cr.execute(
- """
- UPDATE ir_module_module_dependency
- SET auto_install_required = {}
- WHERE module_id = (SELECT id
- FROM ir_module_module
- WHERE name = %s)
- """.format(value),
- params + [module],
- )
-
- cr.execute("UPDATE ir_module_module SET auto_install = %s WHERE name = %s", [auto_install is not False, module])
-
-
-@_warn_usage_outside_base
-def trigger_auto_install(cr, module):
- _assert_modules_exists(cr, module)
- if AUTO_INSTALL == "none":
- return False
-
- dep_match = "true"
- if column_exists(cr, "ir_module_module_dependency", "auto_install_required"):
- dep_match = "d.auto_install_required = true"
-
- country_match = country_join = ""
- if table_exists(cr, "module_country"):
- country_join = """
- LEFT JOIN module_country mc
- ON mc.module_id = m.id
- LEFT JOIN (
- SELECT c.id, p.country_id
- FROM res_company c
- JOIN res_partner p
- ON p.id = c.partner_id
- WHERE c.active
- ) AS c
- ON c.country_id = mc.country_id
- """
- country_match = """
- AND (
- count(mc) = 0 -- no country set
- OR count(c) > 0 -- or have active companies with a matching country
- )
- """
-
- cat_match = "true"
- if AUTO_INSTALL == "only_link_modules":
- # even if we skip auto installs, we still need to auto install the real link-modules.
- # those are in the "Hidden" category
- hidden = ref(cr, "base.module_category_hidden")
- cat_match = cr.mogrify("m.category_id = %s", [hidden]).decode()
-
- query = """
- SELECT m.id
- FROM ir_module_module_dependency d
- JOIN ir_module_module m ON m.id = d.module_id
- JOIN ir_module_module md ON md.name = d.name
- {}
- WHERE m.name = %s
- AND m.state = 'uninstalled'
- AND m.auto_install = true
- AND {}
- AND {}
- GROUP BY m.id
- HAVING bool_and(md.state IN %s)
- {}
- """.format(country_join, dep_match, cat_match, country_match)
-
- cr.execute(query, [module, INSTALLED_MODULE_STATES])
- if cr.rowcount:
- force_install_module(cr, module, reason="it's an auto install module and all its dependencies are installed")
- return True
- return False
-
-
-def _set_module_category(cr, module, category):
- cid = create_categories(cr, category.split("/"))
- cr.execute("UPDATE ir_module_module SET category_id=%s WHERE name=%s", [cid, module])
-
-
-def _set_module_countries(cr, module, countries):
- if not table_exists(cr, "module_country"):
- return
- clean_query = """
- DELETE
- FROM module_country mc
- USING ir_module_module m
- WHERE m.id = mc.module_id
- AND m.name = %s
- """
- cr.execute(clean_query, [module])
-
- if not countries:
- return
-
- insert_query = """
- INSERT INTO module_country(module_id, country_id)
- SELECT m.id, c.id
- FROM ir_module_module m,
- res_country c
- WHERE m.name = %s
- AND c.code IN %s
- """
- cr.execute(insert_query, [module, tuple(c.upper() for c in countries)])
-
-
-@_warn_usage_outside_base
-def new_module(cr, module, deps=(), auto_install=False, category=None, countries=()):
- if deps:
- _assert_modules_exists(cr, *deps)
-
- cr.execute("SELECT id FROM ir_module_module WHERE name = %s", [module])
- if cr.rowcount:
- # Avoid duplicate entries for module which is already installed,
- # even before it has become standard module in new version
- # Also happen for modules added afterward, which should be added by multiple series.
- # But we should force the dependencies
- mod_id = cr.fetchone()[0]
-
- # In CI, it should not happen. Log it as critical
- level = logging.CRITICAL if on_CI() else logging.WARNING
-
- _logger.log(level, "New module %r already defined. Resetting dependencies.", module)
- cr.execute("DELETE FROM ir_module_module_dependency WHERE module_id = %s", [mod_id])
-
- else:
- state = "uninstalled"
- cr.execute(
- """
- INSERT INTO ir_module_module (name, state, demo)
- VALUES (%s, %s, (SELECT demo FROM ir_module_module WHERE name='base'))
- RETURNING id
- """,
- [module, state],
- )
- (new_id,) = cr.fetchone()
-
- cr.execute(
- """
- INSERT INTO ir_model_data (name, module, noupdate, model, res_id)
- VALUES ('module_'||%s, 'base', 't', 'ir.module.module', %s)
- """,
- [module, new_id],
- )
-
- for dep in deps:
- new_module_dep(cr, module, dep)
-
- if category is not None:
- _set_module_category(cr, module, category)
- _set_module_countries(cr, module, countries)
-
- module_auto_install(cr, module, auto_install)
- trigger_auto_install(cr, module)
-
-
-def _caller_version(depth=3):
- frame = currentframe()
- version = "util"
- while version == "util":
- filename, _ = frame_codeinfo(frame, depth)
- version = ".".join(filename.split(os.path.sep)[-2].split(".")[:2])
- depth += 1
-
- return version
-
-
-@_warn_usage_outside_base
-def force_upgrade_of_fresh_module(cr, module, init=True):
- """
- Force the execution of upgrade scripts for a module that is being installed.
-
- Standard Odoo doesn't run upgrade scripts when a module is being installed. This makes
- sense because, technically speaking, the module is not being upgraded. Still, it
- happens that a (new) module needs to perform some operations for it to be correctly
- installed, like grabbing data from another module. This is common when a module is
- functionally *split* into several ones.
-
- :param str module: name of the module for which to force the execution of upgrade
- scripts
- :param bool init: whether to set the module to be in *init* mode
-
- Being in init mode has the side effect of not respecting noupdate flags, in XML file
- nor in `ir_model_data`.
- """
- version = _caller_version()
- if version_gte("saas~14.5"):
- # We must delay until the modules actually exists. They are added by the auto discovery process.
- ENVIRON["__modules_auto_discovery_force_upgrades"][module] = (init, version)
- return None
-
- return _force_upgrade_of_fresh_module(cr, module, init, version)
-
-
-def _force_upgrade_of_fresh_module(cr, module, init, version):
- # Low level implementation
- # Force module state to be in `to upgrade`.
- # Needed for migration script execution. See http://git.io/vnF7f
- state = "to install" if init and version_gte("saas~18.2") else "to upgrade"
- cr.execute(
- """
- UPDATE ir_module_module
- SET state=%s,
- latest_version=%s
- WHERE name=%s
- AND state='to install'
- RETURNING id
- """,
- [state, version, module],
- )
- if init and cr.rowcount:
- if version_gte("saas~18.2"):
- env_ = env(cr)
- env_.registry._force_upgrade_scripts.add(module)
- else:
- # Force module in `init` mode beside its state is forced to `to upgrade`
- # See http://git.io/vnF7O
- odoo.tools.config["init"][module] = "oh yeah!"
-
-
-# for compatibility
-force_migration_of_fresh_module = force_upgrade_of_fresh_module
-"""
-:meta private: exclude from online docs
-"""
-
-
-def _trigger_auto_discovery(cr):
- # low level implementation.
- # Called by `base/0.0.0/post-modules-auto-discovery.py` script.
- # Use accumulated values for the auto_install and force_upgrade modules.
-
- ENVIRON["AUTO_DISCOVERY_RAN"] = True
-
- force_installs = ENVIRON["__modules_auto_discovery_force_installs"]
-
- cr.execute(
- """
- SELECT m.name, array_agg(d.name)
- FROM ir_module_module m
- LEFT JOIN ir_module_module_dependency d ON d.module_id = m.id
- GROUP BY m.name
- """
- )
- existing = dict(cr.fetchall())
-
- graph = {}
- for module in odoo.modules.get_modules():
- manifest = get_manifest(module)
- graph[module] = (
- set(manifest["depends"]),
- manifest["auto_install"],
- manifest["category"],
- manifest.get("countries"),
- )
-
- for module in topological_sort({k: v[0] for k, v in graph.items()}):
- deps, auto_install, category, countries = graph[module]
- if module not in existing:
- new_module(cr, module, deps=deps, auto_install=auto_install, category=category, countries=countries)
- else:
- current_deps = set(existing[module])
- plus = deps - current_deps
- minus = current_deps - deps
- if plus or minus:
- module_deps_diff(cr, module, plus=plus, minus=minus)
- _set_module_category(cr, module, category)
- _set_module_countries(cr, module, countries)
- module_auto_install(cr, module, auto_install)
-
- cr.execute(
- """
- SELECT d.name, STRING_AGG(m.name, ', ' ORDER BY m.name)
- FROM ir_module_module_dependency d
- JOIN ir_module_module m
- ON d.module_id = m.id
- JOIN ir_module_module dm
- ON dm.name = d.name
- WHERE m.state IN %(installed_state)s
- AND dm.state NOT IN %(installed_state)s
- GROUP BY d.name
- """,
- {"installed_state": INSTALLED_MODULE_STATES},
- )
- for new_dep, modules in cr.fetchall():
- _force_install_module(cr, new_dep, reason="it's a new dependency of {}".format(modules))
-
- for module in force_installs:
- _force_install_module(cr, module)
-
- for module, (init, version) in ENVIRON["__modules_auto_discovery_force_upgrades"].items():
- _force_upgrade_of_fresh_module(cr, module, init, version)
-
-
-@_warn_usage_outside_base
-def modules_auto_discovery(cr, force_installs=None, force_upgrades=None):
- # Cursor, Optional[Set[str]], Optional[Set[str]] -> None
-
- # Register the modules to force install and force upgrade
- # The actual auto discovery is delayed in `base/0.0.0/post-modules-auto-discovery.py`
-
- if force_installs:
- ENVIRON["__modules_auto_discovery_force_installs"].update(force_installs)
- if force_upgrades:
- version = _caller_version()
- ENVIRON["__modules_auto_discovery_force_upgrades"].update(dict.fromkeys(force_upgrades, (False, version)))
-
-
-def move_model(cr, model, from_module, to_module, move_data=False, keep=()):
- """
- Move a model from one module to another.
-
- :param str model: name of the model to move
- :param str from_module: name of the module where the model is originally defined
- :param str to_module: name of the destination module where the model is to be moved
- :param bool move_data: whether to also update `ir_model_data` for records of the model
- :param list(str) keep: list of XML ids to keep - not move
-
- This function can be called for moving overrides of a model to another module. As it
- cannot distinguish between source model or inherited model, it raises an exception if
- the destination module isn't installed.
- """
- _validate_model(model)
- if not module_installed(cr, to_module):
- raise MigrationError("Cannot move model {!r} to module {!r} as it isn't installed".format(model, to_module))
-
- if any("." in k for k in keep):
- raise SleepyDeveloperError("The `keep` argument must not contain fully-qualified xmlids, only the local names")
-
- def update_imd(data_model, path):
- table = table_of_model(cr, data_model)
- if not table_exists(cr, table):
- return
-
- joins = []
- where = "d.name != ALL(%(keep)s)"
- if path:
- path = path.split(".")
- column = path.pop()
-
- working_table = table
- for index, milestone in enumerate(path, 1):
- working_table, working_column, _ = target_of(cr, working_table, milestone)
-
- joins.append(
- "JOIN {t} t{i} ON t{i}.{c} = t{j}.{m}".format(
- t=working_table,
- i=index,
- c=working_column,
- j=index - 1,
- m=milestone,
- )
- )
-
- where += " AND t{}.{} = %(linked_model)s".format(len(path), column)
-
- joins = "\n".join(joins)
- params = {
- "linked_model": model,
- "from_module": from_module,
- "to_module": to_module,
- "data_model": data_model,
- "keep": list(keep),
- }
-
- query = """
- WITH dups AS (
- SELECT d.id
- FROM ir_model_data d
- JOIN ir_model_data o
- ON d.name = o.name
- JOIN {table} t0
- ON t0.id = d.res_id
- {joins}
- WHERE d.model = %(data_model)s
- AND d.module = %(from_module)s
- AND o.module = %(to_module)s
- AND {where}
- )
- DELETE FROM ir_model_data d
- USING dups
- WHERE dups.id = d.id
- """
- cr.execute(query.format(table=table, joins=joins, where=where), params)
-
- query = """
- UPDATE ir_model_data d
- SET module = %(to_module)s
- FROM {table} t0
- {joins}
- WHERE t0.id = d.res_id
- AND d.model = %(data_model)s
- AND d.module = %(from_module)s
- AND {where}
- """
- cr.execute(query.format(table=table, joins=joins, where=where), params)
-
- update_imd("ir.model", path="model")
- update_imd("ir.model.fields", path="model_id.model")
- update_imd("ir.model.fields.selection", path="field_id.model_id.model")
- update_imd("ir.model.constraint", path="model.model")
- update_imd("ir.model.relation", path="model.model")
- update_imd("ir.rule", path="model_id.model")
- update_imd("ir.model.access", path="model_id.model")
- update_imd("ir.ui.view", path="model")
- update_imd("ir.actions.act_window", path="res_model")
- update_imd("ir.actions.server", path="model_id.model")
- update_imd("ir.actions.report", path="model")
- update_imd("email.template", path="model") # OpenERP <= 8.0
- update_imd("mail.template", path="model")
- if column_exists(cr, "ir_cron", "model"):
- # < 10.saas~14
- update_imd("ir.cron", path="model")
- else:
- update_imd("ir.cron", path="ir_actions_server_id.model_id.model")
-
- if move_data:
- update_imd(model, path=None)
diff --git a/src/util/orm.py b/src/util/orm.py
deleted file mode 100644
index f43213a3d..000000000
--- a/src/util/orm.py
+++ /dev/null
@@ -1,798 +0,0 @@
-# -*- coding: utf-8 -*-
-# ruff: noqa: PLC0415
-"""
-Utility functions to perform operations via the ORM.
-
-The functions on this module allow to use the ORM safely during upgrades. They enhance or
-patch the ORM such that it can handle high volumes of data in a performant way. In some
-cases totally different alternatives to the ORM's own functions are provided. The functions
-on this module work along the ORM of *all* supported versions.
-"""
-
-import logging
-import re
-from contextlib import contextmanager
-from functools import wraps
-from itertools import chain
-from textwrap import dedent
-
-try:
- from unittest.mock import patch
-except ImportError:
- from mock import patch
-
-try:
- try:
- from odoo.api import SUPERUSER_ID
- except ImportError:
- from odoo import SUPERUSER_ID
- from odoo import fields as ofields
- from odoo import modules, release
-except ImportError:
- from openerp import SUPERUSER_ID, modules, release
-
- try:
- from openerp import fields as ofields
- except ImportError:
- # this is to allow v7.0 DBs to import this module without errors
- # note: some functions on this module will fail (like recompute_fields)
- ofields = None
-
-from .const import BIG_TABLE_THRESHOLD
-from .exceptions import MigrationError
-from .helpers import table_of_model
-from .misc import chunks, log_progress, version_between, version_gte
-from .pg import SQLStr, column_exists, format_query, get_columns, named_cursor
-
-# python3 shims
-try:
- basestring # noqa: B018
-except NameError:
- basestring = str
-
-_logger = logging.getLogger(__name__)
-
-
-def env(cr):
- """
- Create a new environment.
-
- .. warning::
- This function does *not* empty the cache maintained on the cursor for superuser
- with an empty environment. A call to `invalidate_cache` may be necessary every time
- data is modified directly in the database.
-
- :return: the new environment
- :rtype: :class:`~odoo.api.Environment`
- """
- try:
- from odoo.api import Environment
- except ImportError:
- try:
- from openerp.api import Environment
- except ImportError:
- v = release.major_version
- raise MigrationError("Hold on! There is not yet `Environment` in %s" % v)
- return Environment(cr, SUPERUSER_ID, {})
-
-
-def get_admin_channel(cr):
- """
- Retrieve the admin channel or create it if it does not exist.
-
- :meta private: exclude from online docs
- """
- e = env(cr)
- admin_channel = None
- # mail.channel was renamed to discuss.channel in 16.3
- channel_model_name = "mail.channel" if "mail.channel" in e else "discuss.channel"
- if channel_model_name in e:
- admin_channel = e.ref("mail.channel_admin", raise_if_not_found=False)
- if admin_channel:
- return admin_channel
- # search for old name
- admin_channel = e.ref("__upgrade__.channel_administrators", raise_if_not_found=False)
- if admin_channel:
- # rename it.
- from .records import rename_xmlid # imported here to avoid import cycles
-
- rename_xmlid(cr, "__upgrade__.channel_administrators", "mail.channel_admin", noupdate=True)
- return admin_channel
-
- admin_group = e.ref("base.group_system", raise_if_not_found=False)
- if admin_group:
- search_rules = [
- ("channel_type", "=", "channel"),
- ("group_public_id", "=", admin_group.id),
- ("group_ids", "in", admin_group.id),
- ]
- if "public" in e[channel_model_name]._fields:
- search_rules.append(("public", "=", "groups"))
- admin_channel = next(
- iter(e[channel_model_name].search(search_rules).sorted(lambda c: "admin" not in c.name.lower()) or []),
- None,
- )
- if not admin_channel:
- channel_values = {
- "name": "Administrators",
- "channel_type": "channel",
- "group_public_id": admin_group.id,
- "group_ids": [(6, 0, [admin_group.id])],
- }
- if "public" in e[channel_model_name]._fields:
- channel_values["public"] = "groups"
- admin_channel = e[channel_model_name].create(channel_values)
-
- e["ir.model.data"].create(
- {
- "name": "channel_admin",
- "module": "mail",
- "model": channel_model_name,
- "res_id": admin_channel.id,
- "noupdate": True,
- }
- )
-
- return admin_channel
-
-
-def guess_admin_id(cr):
- """
- Guess the admin user id of `cr` database.
-
- :meta private: exclude from online docs
- """
- if not version_gte("12.0"):
- return SUPERUSER_ID
- cr.execute(
- """
- SELECT min(r.uid)
- FROM res_groups_users_rel r
- JOIN res_users u ON r.uid = u.id
- WHERE u.active
- AND r.gid = (SELECT res_id
- FROM ir_model_data
- WHERE module = 'base'
- AND name = 'group_system')
- """
- )
- return cr.fetchone()[0] or SUPERUSER_ID
-
-
-def create_cron(cr, name, model, code, interval=(1, "hours")):
- # (Cursor, str, str, str, Tuple[int, str]) -> None
- cr.execute("SELECT id FROM ir_model WHERE model = %s", [model])
- model_id = cr.fetchone()[0]
- number, unit = interval
- # TODO handle version <=10.saas-14
- cron = {
- "name": "Post Upgrade: %s" % (name,),
- "model_id": model_id,
- "state": "code",
- "code": dedent(code),
- "interval_number": number,
- "interval_type": unit,
- }
- if not version_gte("saas~17.3"):
- cron["numbercall"] = -1
-
- xid_name = "cron_" + re.sub(r"\W+", "_", cron["name"].lower())
- xid = "__upgrade__." + xid_name
-
- e = env(cr)
- data = {"module": "__upgrade__", "xml_id": xid, "values": cron, "noupdate": True}
- if hasattr(e["ir.cron"], "_load_records"):
- e["ir.cron"]._load_records([data])
- else:
- # < saas-11.5
- e["ir.model.data"]._update("ir.cron", **data)
-
- # Exclude the related server action from CLOC with an extra xmlid
- if not column_exists(cr, "ir_cron", "ir_actions_server_id"):
- return
-
- columns = get_columns(cr, "ir_model_data", ignore=("id", "module"))
- s_columns = ["s." + c for c in columns]
- query = """
- INSERT INTO ir_model_data(module, {columns})
- SELECT '__cloc_exclude__', {s_columns}
- FROM ir_model_data x
- JOIN ir_cron c ON x.model = 'ir.cron' AND x.res_id = c.id
- JOIN ir_model_data s ON s.model = 'ir.actions.server' AND s.res_id = c.ir_actions_server_id
- WHERE x.module = '__upgrade__'
- AND x.name = %s
- AND s.module = '__upgrade__'
- ON CONFLICT DO NOTHING
- """
- cr.execute(query.format(columns=",".join(columns), s_columns=",".join(s_columns)), [xid_name])
-
-
-_TRACKING_ATTR = "tracking" if version_gte("saas~12.2") else "track_visibility"
-
-
-def flush(records):
- ((hasattr(records, "env") and getattr(records.env, "flush_all", None)) or getattr(records, "flush", lambda: None))()
-
-
-def recompute(records):
- (
- (hasattr(records, "env") and getattr(records.env, "_recompute_all", None))
- or getattr(records, "recompute", lambda: None)
- )()
-
-
-def invalidate(records, *args):
- env = getattr(records, "env", None)
- expected_args = 2 if env is None else 0
- if len(args) != expected_args:
- raise ValueError("Unexpected arguments: %r" % (args[expected_args:],))
- ((env and getattr(records.env, "invalidate_all", None)) or records.invalidate_cache)(*args)
-
-
-def no_selection_cache_validation(f=None):
- if not version_gte("8.0"):
- return f
- old_convert = ofields.Selection.convert_to_cache
-
- def _convert(self, value, record, validate=True):
- if self.model_name == "res.users" and self.name == "lang":
- return old_convert(self, value, record, validate=validate)
- return old_convert(self, value, record, validate=False)
-
- if f is None:
- return patch(ofields.__name__ + ".Selection.convert_to_cache", _convert)
-
- @wraps(f)
- def wrapper(*args, **kwargs):
- with patch(ofields.__name__ + ".Selection.convert_to_cache", _convert):
- return f(*args, **kwargs)
-
- return wrapper
-
-
-@no_selection_cache_validation
-def recompute_fields(cr, model, fields, ids=None, logger=_logger, chunk_size=256, strategy="auto", query=None):
- """
- Recompute field values.
-
- This function will recompute fields of a model restricted to a set of records - or
- all. The re-computation is not done on all records at the same time. It is split in
- batches (chunks). This avoids performance issues, and, in the worse cases, failures
- due to `MemoryError`. After each chunk is processed the data is sent back to the
- database according to one of the following strategies:
-
- - *flush*: use the `flush` method of the ORM
- - *commit*: `commit` the cursor - also flush
- - *auto*: pick the best alternative between the two above given the number of records
- to compute and the presence of tracked fields.
-
- The *commit* strategy is less prone to cause a `MemoryError` for a huge volume of data.
-
- :param str model: name of the model to recompute
- :param list(str) fields: list of the name of the fields to recompute
- :param list(int) or None ids: list of the IDs of the records to recompute, when `None`
- recompute *all* records, unless `query` is also set (see
- below)
- :param logger: logger used to report the progress
- :type logger: :class:`logging.Logger`
- :param int chunk_size: number of records per chunk - used to split the processing
- :param str strategy: strategy used to process the re-computation
- :param str query: query to get the IDs of records to recompute, it is an error to set
- both `ids` and `query`. Note that the processing will always happen
- in ascending order. If that is unwanted, you must use `ids` instead.
- """
- if strategy not in {"flush", "commit", "auto"}:
- raise ValueError("Invalid strategy {!r}".format(strategy))
- if ids is not None and query is not None:
- raise ValueError("Cannot set both `ids` and `query`")
- Model = env(cr)[model] if isinstance(model, basestring) else model
- model = Model._name
-
- if ids is None:
- query = format_query(cr, "SELECT id FROM {}", table_of_model(cr, model)) if query is None else SQLStr(query)
- cr.execute(
- format_query(cr, "CREATE UNLOGGED TABLE _upgrade_rf(id) AS (WITH query AS ({}) SELECT * FROM query)", query)
- )
- count = cr.rowcount
- cr.execute("ALTER TABLE _upgrade_rf ADD CONSTRAINT pk_upgrade_rf_id PRIMARY KEY (id)")
-
- def get_ids():
- with named_cursor(cr, itersize=2**20) as ncr:
- ncr.execute("SELECT id FROM _upgrade_rf ORDER BY id")
- for (id_,) in ncr:
- yield id_
-
- ids_ = get_ids()
- else:
- count = len(ids)
- ids_ = ids
-
- if not count:
- cr.execute("DROP TABLE IF EXISTS _upgrade_rf")
- return
-
- _logger.info("Computing fields %s of %r on %d records", fields, model, count)
-
- if strategy == "auto":
- big_table = count > BIG_TABLE_THRESHOLD
- any_tracked_field = any(getattr(Model._fields[f], _TRACKING_ATTR, False) for f in fields)
- strategy = "commit" if big_table and any_tracked_field else "flush"
-
- size = (count + chunk_size - 1) / chunk_size
- qual = "{} {:d}-bucket".format(model, chunk_size) if chunk_size != 1 else model
- for subids in log_progress(chunks(ids_, chunk_size, list), logger, qualifier=qual, size=size):
- records = Model.browse(subids)
- for field_name in fields:
- field = records._fields[field_name]
- if hasattr(records, "_recompute_todo"):
- # < 13.0
- records._recompute_todo(field)
- else:
- Model.env.add_to_compute(field, records)
-
- recompute(records)
- # trigger dependent fields recomputation
- records.modified(fields)
- if strategy == "commit":
- cr.commit()
- else:
- flush(records)
- invalidate(records)
- cr.execute("DROP TABLE IF EXISTS _upgrade_rf")
-
-
-class iter_browse(object):
- """
- Iterate over recordsets.
-
- The `callable` object returned by this class can be used as an iterator that loads
- records by chunks (into a `recordset`). After each chunk is exhausted their data is
- sent back to the database - flushed or committed - and a new chunk is loaded.
-
- This class allows to run code like:
-
- .. code-block:: python
-
- for record in env['my.model'].browse(ids):
- record.field1 = val
-
- env['my.model'].browse(ids)._compute_field2()
- env['my.model'].create(values)
-
- in a performant way while also avoiding `MemoryError`, even when `ids` or `values`
- have millions of entries. The alternative using this class would be:
-
- .. example::
- .. code-block:: python
-
- MyModel = util.env(cr)['my.model']
- for record in util.iter_browse(MyModel, ids):
- record.field1 = val
-
- util.iter_browse(MyModel, ids)._compute_field2()
- util.iter_browse(MyModel, ids).create(values)
-
- :param model: the model to iterate
- :type model: :class:`odoo.model.Model`
- :param list(int) ids: list of IDs of the records to iterate
- :param int chunk_size: number of records to load in each iteration chunk, `200` by
- default
- :param logger: logger used to report the progress, by default
- :data:`~odoo.upgrade.util.orm._logger`
- :type logger: :class:`logging.Logger`
- :param str strategy: whether to `flush` or `commit` on each chunk, default is `flush`
- :return: the object returned by this class can be used to iterate, or call any model
- method, safely on millions of records.
-
- See also :func:`~odoo.upgrade.util.orm.env`
- """
-
- __slots__ = ("_chunk_size", "_cr_uid", "_it", "_logger", "_model", "_patch", "_size", "_strategy")
-
- def __init__(self, model, *args, **kw):
- assert len(args) in [1, 3] # either (cr, uid, ids) or (ids,)
- self._model = model
- self._cr_uid = args[:-1]
- ids = args[-1]
- self._size = len(ids)
- self._chunk_size = kw.pop("chunk_size", 200) # keyword-only argument
- self._logger = kw.pop("logger", _logger)
- self._strategy = kw.pop("strategy", "flush")
- assert self._strategy in {"flush", "commit"}
- if kw:
- raise TypeError("Unknown arguments: %s" % ", ".join(kw))
-
- self._patch = None
- self._it = chunks(ids, self._chunk_size, fmt=self._browse)
-
- def _browse(self, ids):
- next(self._end(), None)
- args = self._cr_uid + (list(ids),)
- if not self._patch:
- self._patch = no_selection_cache_validation()
- self._patch.start()
-
- return self._model.browse(*args)
-
- def _end(self):
- if self._strategy == "commit":
- self._model.env.cr.commit()
- else:
- flush(self._model)
- invalidate(self._model, *self._cr_uid)
- if self._patch:
- self._patch.stop()
- if 0:
- yield
-
- def __iter__(self):
- if self._it is None:
- raise RuntimeError("%r ran twice" % (self,))
-
- it = chain.from_iterable(self._it)
- if self._logger:
- it = log_progress(it, self._logger, qualifier=self._model._name, size=self._size)
- self._it = None
- return chain(it, self._end())
-
- def __getattr__(self, attr):
- if self._it is None:
- raise RuntimeError("%r ran twice" % (self,))
-
- if not callable(getattr(self._model, attr)):
- raise TypeError("The attribute %r is not callable" % attr)
-
- it = self._it
- if self._logger:
- sz = (self._size + self._chunk_size - 1) // self._chunk_size
- qualifier = "%s[:%d]" % (self._model._name, self._chunk_size)
- it = log_progress(it, self._logger, qualifier=qualifier, size=sz)
-
- def caller(*args, **kwargs):
- args = self._cr_uid + args
- return [getattr(chnk, attr)(*args, **kwargs) for chnk in chain(it, self._end())]
-
- self._it = None
- return caller
-
- def create(self, values, **kw):
- """
- Create records.
-
- An alternative to the default `create` method of the ORM that is safe to use to
- create millions of records.
-
- :param list(dict) values: list of values of the records to create
- :param bool multi: whether to use the multi version of `create`, by default is
- `True` from Odoo 12 and above
- """
- multi = kw.pop("multi", version_gte("saas~11.5"))
- if kw:
- raise TypeError("Unknown arguments: %s" % ", ".join(kw))
-
- if not values:
- raise ValueError("`create` cannot be called with an empty `values` argument")
-
- if self._size:
- raise ValueError("`create` can only called on empty `browse_record` objects.")
-
- ids = []
- size = len(values)
- it = chunks(values, self._chunk_size, fmt=list)
- if self._logger:
- sz = (size + self._chunk_size - 1) // self._chunk_size
- qualifier = "env[%r].create([:%d])" % (self._model._name, self._chunk_size)
- it = log_progress(it, self._logger, qualifier=qualifier, size=sz)
-
- self._patch = no_selection_cache_validation()
- for sub_values in it:
- self._patch.start()
-
- if multi:
- ids += self._model.create(sub_values).ids
- elif not self._cr_uid:
- ids += [self._model.create(sub_value).id for sub_value in sub_values]
- else:
- # old API, `create` directly return the id
- ids += [self._model.create(*(self._cr_uid + (sub_value,))) for sub_value in sub_values]
-
- next(self._end(), None)
- args = self._cr_uid + (ids,)
- return iter_browse(
- self._model, *args, chunk_size=self._chunk_size, logger=self._logger, strategy=self._strategy
- )
-
-
-@contextmanager
-def custom_module_field_as_manual(env, rollback=True, do_flush=False):
- """
- Mark fields of custom modules as `manual`.
-
- Helper to be used with a Python `with` statement, to perform an operation with models and fields coming
- from Python modules acting as `manual` models/fields, while restoring back the state of these models
- and fields once the operation done.
- e.g.
- - validating views coming from custom modules, with the fields loaded as the custom source code was there,
- - crawling the menus as the models/fields coming from custom modules were available.
-
- !!! Rollback might be deactivated with the `rollback` parameter but for internal purpose ONLY !!!
-
- :meta private: exclude from online docs
- """
- # 1. Convert models which are not in the registry to `manual` models
- # and list the models that were converted, to restore them back afterwards.
- models = list(env.registry.models)
- # Get all models that seem to come from uninstalled odoo modules, they won't be loaded
- # but we cannot consider them as custom models
- env.cr.execute(
- """
- SELECT m.model
- FROM ir_model m
- JOIN ir_model_data d
- ON d.res_id = m.id
- AND d.model = 'ir.model'
- JOIN ir_module_module x
- ON x.name = d.module
- WHERE d.module IN %s
- GROUP BY m.model
- HAVING bool_and(COALESCE(x.state, 'uninstalled') = 'uninstalled')
- """,
- [tuple(modules.get_modules())],
- )
- models_uninstalled_standard_modules = [r[0] for r in env.cr.fetchall()]
- if models_uninstalled_standard_modules:
- _logger.warning("Leftover models from uninstalled standard modules: %s", models_uninstalled_standard_modules)
- env.cr.execute(
- """
- UPDATE ir_model
- SET state = 'manual'
- WHERE state = 'base'
- AND model not in %s
- RETURNING id, model
- """,
- (tuple(models + models_uninstalled_standard_modules),),
- )
- updated_models = env.cr.fetchall()
- updated_model_ids, custom_models = zip(*updated_models) if updated_models else [[], []]
-
- mock_x_email_fields = []
- if updated_model_ids and version_gte("14.0") and "mail.thread.blacklist" in env:
- # 1.1 Add a mock x_email field for custom mail.blacklist models
- env.cr.execute(
- """
- INSERT INTO ir_model_fields(name, model, model_id, field_description,
- state, compute, store, ttype)
- SELECT 'x_email', model, id, jsonb_build_object('en_US', 'Mock email field for custom model'){},
- 'manual', 'for record in records: record["x_email"] = "upgrade@example.com"', false, 'char'
- FROM ir_model
- WHERE id IN %s
- AND is_mail_blacklist
- ON CONFLICT DO NOTHING
- RETURNING id
- """.format("" if version_gte("16.0") else "->>'en_US'"),
- [tuple(updated_model_ids)],
- )
- mock_x_email_fields = [r[0] for r in env.cr.fetchall()]
-
- temp_ir_model_access_ids = []
- if version_gte("14.0") and custom_models:
- # 1.2 Version 14.0 and above requires new access rights for transient models,
- # and the rights are yet to be added post standard upgrade for custom modules.
- # To avoid access errors, create some temporary access rights for these models.
- env.cr.execute(
- """
- INSERT INTO ir_model_access (name, model_id, active, perm_read)
- SELECT 'tmp_access', ir_model.id, true, true
- FROM ir_model
- LEFT JOIN ir_model_access
- ON ir_model_access.model_id = ir_model.id
- WHERE ir_model.model IN %s
- AND ir_model.transient
- AND ir_model_access.id IS NULL
- RETURNING id
- """,
- [custom_models],
- )
- temp_ir_model_access_ids = [r[0] for r in env.cr.fetchall()]
-
- # 2. Convert fields which are not in the registry to `manual` fields
- # and list the fields that were converted, to restore them back afterwards.
- # Also temporarily disable rules (ir.rule) that come from custom modules.
- updated_field_ids = []
-
- # 2.1 Convert fields not in the registry of models already in the registry.
- # defuse constraint for manual fields, ref: odoo/odoo@3d2f766
- env.cr.execute("SELECT 1 FROM pg_constraint WHERE conname = 'ir_model_fields_name_manual_field'")
- have_constraint = bool(env.cr.rowcount)
- if have_constraint:
- env.cr.execute("ALTER TABLE ir_model_fields DROP CONSTRAINT ir_model_fields_name_manual_field")
- # In the past, some models added the reserved word `env` as field (e.g. `payment.acquirer`)
- # if the field was not correctly removed from the database during past upgrades, the field remains in the database.
- reserved_words = ["env"]
- ignores = {"ir.actions.server": ["condition"], "ir.ui.view": ["page"]}
- for model in models:
- model_fields = tuple(list(env.registry[model]._fields) + reserved_words + ignores.get(model, []))
- env.cr.execute(
- """
- UPDATE ir_model_fields
- SET state = 'manual'
- WHERE state = 'base'
- AND model = %s
- AND name not in %s
- RETURNING id
- """,
- [model, model_fields],
- )
- updated_field_ids += [r[0] for r in env.cr.fetchall()]
-
- # 2.2 Convert fields of custom models, models that were just converted to `manual` models in the previous step.
- for model in custom_models:
- env.cr.execute(
- """
- UPDATE ir_model_fields
- SET state = 'manual'
- WHERE state = 'base'
- AND model = %s
- AND name not in %s
- RETURNING id
- """,
- (model, tuple(reserved_words)),
- )
- updated_field_ids += [r[0] for r in env.cr.fetchall()]
-
- # 2.3 Temporarily disable rules that come from custom modules
- standard_modules = modules.get_modules()
-
- env.cr.execute(
- """
- WITH custom_rules AS (
- SELECT r.id
- FROM ir_rule r
- LEFT JOIN ir_model_data d ON d.model = 'ir.rule' AND d.res_id = r.id
- WHERE COALESCE(d.module, '') NOT IN %s
- AND r.active
- )
- UPDATE ir_rule r
- SET active = false
- FROM custom_rules c
- WHERE r.id = c.id
- RETURNING r.id
- """,
- (tuple(standard_modules),),
- )
- disabled_ir_rule_ids = [r[0] for r in env.cr.fetchall()]
-
- # 3. Alter fields which won't work by just changing their `state` to `manual`,
- # because information from their Python source is missing.
- # List them and what was altered, to restore them back afterwards.
-
- # 3.1. Loading of base selection fields which have been converted to manual,
- # for which we don't have the possible values.
- updated_selection_fields = []
- # For Odoo <= 12.0.
- # From 13.0, there is a model `ir.model.fields.selection` holding the values,
- # and `ir.model.fields.selection` becomes a computed field.
- if not env.registry["ir.model.fields"]._fields["selection"].compute:
- env.cr.execute(
- """
- UPDATE ir_model_fields
- SET selection = '[]'
- WHERE state = 'manual'
- AND COALESCE(selection, '') = ''
- RETURNING id, selection
- """
- )
- updated_selection_fields += env.cr.fetchall()
-
- # 3.2. Loading of base many2one fields converted to manual, set to `required` and `on_delete` to `set null` in db,
- # which is not accepted by the ORM:
- # https://github.com/odoo/odoo/blob/2a7e06663c1281f0cf75f72fc491bc2cc39ef81c/odoo/fields.py#L2404-L2405
- env.cr.execute(
- """
- UPDATE ir_model_fields
- SET on_delete = 'restrict'
- WHERE state = 'manual'
- AND ttype = 'many2one'
- AND required
- AND on_delete = 'set null'
- RETURNING id, on_delete
- """
- )
- updated_many2one_fields = env.cr.fetchall()
-
- # 3.4. models `_rec_name` are not reloaded correctly.
- # If the model has no `_rec_name` and there is a manual field `name` or `x_name`,
- # the `_rec_name` becomes this name field. But then, when we convert back the manual fields to base field,
- # and we reload the registry, the `_rec_name` is not reset by the ORM,
- # and there is an assert raising in `models.py`: `assert cls._rec_name in cls._fields`.
- rec_names = {key: model._rec_name for key, model in env.registry.models.items()}
-
- # 3.5 patches
- patches = [] # Ideally a contextlib.ExitStack but we need to support here Python2.7
-
- @contextmanager
- def all_patches():
- for p in patches:
- p.start()
- yield
- for p in reversed(patches):
- p.stop()
-
- if version_gte("9.0"):
- # 3.5.1 `_build_model` calls `check_pg_name` even if the table is not created/altered, and in some cases
- # models that have been converted to manual have a too long name, and we dont have the `_table` info.
- patches.append(patch("odoo.models.check_pg_name", lambda name: None))
-
- if version_between("9.0", "saas~14.3"):
- # 3.5.2: `display_name` is added automatically, as a base field, and depends on the field `name`
- # Sometimes, a custom model has no `name` field or it couldn't be loaded (e.g. an invalid `related`)
- # Mark it as manual so its skipped on loading fail.
- from odoo.models import BaseModel
-
- # Since saas-14.4, _add_magic_fields() no longer exists. Moreover,
- # '_rec_name' is automatically fixed when the field it refers to is
- # dropped from the model's class. Therefore, 'display_name' no
- # longer needs to become manual.
- origin_add_magic_fields = BaseModel._add_magic_fields
-
- def _add_magic_fields(self):
- res = origin_add_magic_fields(self)
- if self._custom and "display_name" in self._fields:
- self._fields["display_name"].manual = True
- return res
-
- patches.append(patch.object(BaseModel, "_add_magic_fields", _add_magic_fields))
-
- if version_gte("saas~16.4"):
- # 3.5.3 allow loading manual fields
- patches.append(patch("odoo.addons.base.models.ir_model.IrModelFields._is_manual_name", lambda self, name: True))
-
- with all_patches():
- # 4. Reload the registry with the models and fields converted to manual.
- setup_models = (
- env.registry._setup_models__ if hasattr(env.registry, "_setup_models__") else env.registry.setup_models
- )
- setup_models(env.cr)
-
- # 5. Do the operation.
- yield
-
- if rollback:
- if do_flush:
- flush(env["base"])
-
- # 6. Restore back models and fields converted from `base` to `manual`.
- if updated_model_ids:
- env.cr.execute("UPDATE ir_model SET state = 'base' WHERE id IN %s", (tuple(updated_model_ids),))
- if mock_x_email_fields:
- # Remove mock x_email field for custom mail.blacklist models
- env.cr.execute("DELETE FROM ir_model_fields WHERE id IN %s", [tuple(mock_x_email_fields)])
-
- if updated_field_ids:
- env.cr.execute("UPDATE ir_model_fields SET state = 'base' WHERE id IN %s", (tuple(updated_field_ids),))
- if have_constraint:
- # restore constraint for manual fields, ref: odoo/odoo@3d2f766
- env.cr.execute(
- r"""
- ALTER TABLE ir_model_fields
- ADD CONSTRAINT ir_model_fields_name_manual_field
- CHECK (state != 'manual' OR name LIKE 'x\_%')
- """
- )
- if disabled_ir_rule_ids:
- env.cr.execute("UPDATE ir_rule SET active = 't' WHERE id IN %s", (tuple(disabled_ir_rule_ids),))
- if temp_ir_model_access_ids:
- env.cr.execute("DELETE FROM ir_model_access WHERE id IN %s", [tuple(temp_ir_model_access_ids)])
- for field_id, selection in updated_selection_fields:
- env.cr.execute("UPDATE ir_model_fields SET selection = %s WHERE id = %s", (selection, field_id))
- for field_id, on_delete in updated_many2one_fields:
- env.cr.execute("UPDATE ir_model_fields SET on_delete = %s WHERE id = %s", [on_delete, field_id])
- for model, rec_name in rec_names.items():
- env.registry[model]._rec_name = rec_name
-
- # 7. Reload the registry as before
- env.clear()
- setup_models = (
- env.registry._setup_models__ if hasattr(env.registry, "_setup_models__") else env.registry.setup_models
- )
- setup_models(env.cr)
diff --git a/src/util/pg.py b/src/util/pg.py
deleted file mode 100644
index 7a5873123..000000000
--- a/src/util/pg.py
+++ /dev/null
@@ -1,1898 +0,0 @@
-# -*- coding: utf-8 -*-
-"""Utility functions for interacting with PostgreSQL."""
-
-import collections
-import logging
-import os
-import re
-import string
-import threading
-import time
-import uuid
-import warnings
-from contextlib import contextmanager
-from functools import partial, reduce
-from multiprocessing import cpu_count
-
-try:
- from concurrent.futures import ThreadPoolExecutor # noqa: I001
- import concurrent
-except ImportError:
- ThreadPoolExecutor = None
-
-try:
- from collections import UserList
-except ImportError:
- from UserList import UserList
-
-try: # noqa: SIM105
- range = xrange # noqa: A001
-except NameError:
- pass
-
-import psycopg2
-from psycopg2 import errorcodes, sql
-from psycopg2.extensions import quote_ident
-from psycopg2.extras import Json
-
-try:
- from odoo.modules import module as odoo_module
- from odoo.sql_db import db_connect
-except ImportError:
- from openerp.sql_db import db_connect
-
- odoo_module = None
-
-from .exceptions import MigrationError, SleepyDeveloperError
-from .helpers import _validate_table, model_of_table
-from .misc import AUTO, Sentinel, log_progress, on_CI, version_gte
-
-_logger = logging.getLogger(__name__)
-
-ON_DELETE_ACTIONS = frozenset(("SET NULL", "CASCADE", "RESTRICT", "NO ACTION", "SET DEFAULT"))
-MAX_BUCKETS = int(os.getenv("MAX_BUCKETS", "150000"))
-DEFAULT_BUCKET_SIZE = int(os.getenv("BUCKET_SIZE", "10000"))
-
-
-class PGRegexp(str):
- """
- Wrapper for semantic meaning of parameters: this string is a Postgres regular expression.
-
- See :func:`~odoo.upgrade.util.records.replace_in_all_jsonb_values`
- """
-
-
-class SQLStr(str):
- """
- Wrapper for semantic meaning of parameters: this string is a valid SQL snippet.
-
- See :func:`~odoo.upgrade.util.pg.format_query`
- """
-
-
-def get_max_workers():
- force_max_worker = os.getenv("MAX_WORKER")
- if force_max_worker:
- if not force_max_worker.isdigit():
- raise MigrationError("wrong parameter: MAX_WORKER should be an integer")
- return int(force_max_worker)
- return min(8, cpu_count())
-
-
-@contextmanager
-def savepoint(cr):
- # NOTE: the `savepoint` method on Cursor only appear in `saas-3`, which mean this function
- # can't be called when upgrading to saas~1 or saas~2.
- # I take the bet it won't be problematic...
- with cr.savepoint():
- yield
-
-
-def _parallel_execute_serial(cr, queries, logger=_logger):
- cnt = 0
- for query in log_progress(queries, logger, qualifier="queries", size=len(queries)):
- cr.execute(query)
- cnt += cr.rowcount
- return cnt
-
-
-if ThreadPoolExecutor is not None:
-
- def _parallel_execute_threaded(cr, queries, logger=_logger):
- if not queries:
- return None
-
- if len(queries) == 1:
- # No need to spawn other threads
- cr.execute(queries[0])
- return cr.rowcount
-
- max_workers = min(get_max_workers(), len(queries))
- cursor = db_connect(cr.dbname).cursor
-
- def execute(query):
- with cursor() as tcr:
- tcr.execute(query)
- return tcr.rowcount
-
- cr.commit()
-
- CONCURRENCY_ERRORCODES = {
- errorcodes.DEADLOCK_DETECTED,
- errorcodes.SERIALIZATION_FAILURE,
- }
- failed_queries = []
- tot_cnt = 0
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
- future_queries = {executor.submit(execute, q): q for q in queries}
- for future in log_progress(
- concurrent.futures.as_completed(future_queries),
- logger,
- qualifier="queries",
- size=len(queries),
- estimate=False,
- log_hundred_percent=True,
- ):
- try:
- tot_cnt += future.result() or 0
- except psycopg2.OperationalError as exc:
- if exc.pgcode not in CONCURRENCY_ERRORCODES:
- raise
-
- # to be retried without concurrency
- failed_queries.append(future_queries[future])
-
- if failed_queries:
- logger.warning("Serialize queries that failed due to concurrency issues")
- tot_cnt += _parallel_execute_serial(cr, failed_queries, logger=logger)
- cr.commit()
-
- return tot_cnt
-
-else:
- _parallel_execute_threaded = _parallel_execute_serial
-
-
-def parallel_execute(cr, queries, logger=_logger):
- """
- Execute queries in parallel.
-
- .. example::
- .. code-block:: python
-
- util.parallel_execute(cr, [util.format_query(cr, "REINDEX TABLE {}", t) for t in tables])
-
- .. tip::
- If looking to speedup a single query, see :func:`~odoo.upgrade.util.pg.explode_execute`.
-
- :param list(str) queries: list of queries to execute concurrently
- :param `~logging.Logger` logger: logger used to report the progress
- :return: the sum of `cr.rowcount` for each query run
- :rtype: int
-
- .. warning::
- - Due to the nature of `cr.rowcount`, the return value of this function may represent an
- underestimate of the real number of affected records. For instance, when some records
- are deleted/updated as a result of an `ondelete` clause, they won't be taken into account.
-
- - As a side effect, the cursor will be committed.
-
- .. note::
- If a concurrency issue occurs, the *failing* queries will be retried sequentially.
- """
- parallel_execute_impl = (
- _parallel_execute_serial
- if getattr(threading.current_thread(), "testing", False)
- or (odoo_module is not None and getattr(odoo_module, "current_test", False))
- else _parallel_execute_threaded
- )
- return parallel_execute_impl(cr, queries, logger=_logger)
-
-
-def format_query(cr, query, *args, **kwargs):
- """
- Safely format a query.
-
- The `str` arguments to this function are assumed to be SQL identifiers. They are
- wrapped in double quotes before being expanded using :meth:`str.format`. Any other
- `psycopg2.sql.Composable `_
- are also allowed. This includes :class:`~odoo.upgrade.util.pg.ColumnList`, see also
- :func:`~odoo.upgrade.util.pg.get_columns`
-
- .. example::
- .. code-block:: python
-
- >>> util.format_query(cr, "SELECT {0} FROM {table}", "id", table="res_users")
- SELECT "id" FROM "res_users"
-
- :param str query: query to format, can use brackets `{}` as in :func:`str.format`
- """
-
- def wrap(arg):
- if isinstance(arg, sql.Composable):
- return arg
- elif isinstance(arg, SQLStr):
- return sql.SQL(arg)
- else:
- return sql.Identifier(arg)
-
- args = tuple(wrap(a) for a in args)
- kwargs = {k: wrap(v) for k, v in kwargs.items()}
- return SQLStr(sql.SQL(query).format(*args, **kwargs).as_string(cr._cnx))
-
-
-class _ExplodeFormatter(string.Formatter):
- """
- Retro-compatible parallel filter formatter.
-
- Any input that didn't fail before satisfies:
- 1. There is no replacement in the string other than `{parallel_filter}`.
- 2. Any literal brace was escaped --by doubling them.
-
- For any input that didn't fail before this new implementation returns the same output
- as `str.format`.
-
- The main change here, and the goal of this class, is to now make former invalid input
- valid. Thus this formatter will _only_ replace `{parallel_filter}` while keeping any
- other `{str}` or `{int}` elements. Double braces will still be formatted into
- single ones.
-
- :meta private: exclude from online docs
- """
-
- def parse(self, format_string):
- for literal_text, field_name, format_spec, conversion in super(_ExplodeFormatter, self).parse(format_string):
- if field_name is not None and field_name != "parallel_filter":
- yield literal_text + "{", None, None, None
- composed = (
- field_name
- + (("!" + conversion) if conversion else "")
- + ((":" + format_spec) if format_spec else "")
- + "}"
- )
- yield composed, None, None, None
- else:
- yield literal_text, field_name, format_spec, conversion
-
-
-_explode_format = _ExplodeFormatter().format
-
-
-def explode_query(cr, query, alias=None, num_buckets=8, prefix=None):
- """
- Explode a query to multiple queries that can be executed in parallel.
-
- Use modulo strategy to separate queries in buckets
-
- :meta private: exclude from online docs
- """
- warnings.warn(
- "`explode_query` has been deprecated in favor of `explode_query_range`. Consider also `explode_execute`.",
- category=DeprecationWarning,
- stacklevel=2,
- )
- if prefix is not None:
- if alias is not None:
- raise ValueError("Cannot use both `alias` and deprecated `prefix` arguments.")
- _logger.getChild("explode_query").warning(
- "The `prefix` argument is deprecated. Use the `alias` argument instead."
- )
- elif alias is not None:
- prefix = alias + "."
- else:
- prefix = ""
-
- if "{parallel_filter}" not in query:
- if re.search(r"\bOR\b", query, re.I):
- _logger.getChild("explode_query").warning(
- "`OR` found in the query. Please explicitly include the `{parallel_filter}` placeholder in the query."
- )
- sep_kw = " AND " if re.search(r"\sWHERE\s", query, re.M | re.I) else " WHERE "
- query += sep_kw + "{parallel_filter}"
-
- num_buckets = int(num_buckets)
- if num_buckets < 1:
- raise ValueError("num_buckets should be greater than zero")
- parallel_filter = "mod(abs({prefix}id), %s) = %s".format(prefix=prefix)
- query = _explode_format(query.replace("%", "%%"), parallel_filter=parallel_filter)
- return [cr.mogrify(query, [num_buckets, index]).decode() for index in range(num_buckets)]
-
-
-def explode_query_range(cr, query, table, alias=None, bucket_size=DEFAULT_BUCKET_SIZE, prefix=None):
- """
- Explode a query to multiple queries that can be executed in parallel.
-
- Use between strategy to separate queries in buckets
-
- :meta private: exclude from online docs
- """
- if prefix is not None:
- if alias is not None:
- raise ValueError("Cannot use both `alias` and deprecated `prefix` arguments.")
- _logger.getChild("explode_query_range").warning(
- "The `prefix` argument is deprecated. Use the `alias` argument instead."
- )
- alias = prefix.rstrip(".")
-
- alias = alias or table
-
- if "{parallel_filter}" not in query:
- if re.search(r"\bOR\b", query, re.I):
- _logger.getChild("explode_query_range").warning(
- "`OR` found in the query. Please explicitly include the `{parallel_filter}` placeholder in the query."
- )
- sep_kw = " AND " if re.search(r"\sWHERE\s", query, re.M | re.I) else " WHERE "
- query += sep_kw + "{parallel_filter}"
-
- cr.execute(format_query(cr, "SELECT min(id), max(id) FROM {}", table))
- min_id, max_id = cr.fetchone()
- if min_id is None:
- # empty table
- if on_CI():
- # Even if there are any records, return one query to be executed to validate its correctness and avoid
- # scripts that pass the CI but fail in production.
- parallel_filter = "{alias}.id IS NOT NULL".format(alias=alias)
- return [_explode_format(query, parallel_filter=parallel_filter)]
- else:
- return []
-
- count = (max_id + 1 - min_id) // bucket_size
- if count > MAX_BUCKETS:
- _logger.getChild("explode_query_range").warning(
- "High number of queries generated (%s); switching to a precise bucketing strategy", count
- )
- cr.execute(
- format_query(
- cr,
- """
- WITH t AS (
- SELECT id,
- mod(row_number() OVER(ORDER BY id) - 1, %s) AS g
- FROM {table}
- ORDER BY id
- ) SELECT array_agg(id ORDER BY id) FILTER (WHERE g=0),
- min(id),
- max(id)
- FROM t
- """,
- table=table,
- ),
- [bucket_size],
- )
- ids, min_id, max_id = cr.fetchone()
- else:
- ids = list(range(min_id, max_id + 1, bucket_size))
-
- assert min_id == ids[0] and max_id + 1 != ids[-1] # sanity checks
- ids.append(max_id + 1) # ensure last bucket covers whole range
- # `ids` holds a list of values marking the interval boundaries for all buckets
-
- if (max_id - min_id + 1) <= 1.1 * bucket_size or (len(ids) == 3 and ids[2] - ids[1] <= 0.1 * bucket_size):
- # If we return one query `parallel_execute` skip spawning new threads. Thus we return only one query if we have
- # only two buckets and the second would have at most 10% of bucket_size records.
- # Still, since the query may only be valid if there is no split, we force the usage of `prefix` in the query to
- # validate its correctness and avoid scripts that pass the CI but fail in production.
- parallel_filter = "{alias}.id IS NOT NULL".format(alias=alias)
- return [_explode_format(query, parallel_filter=parallel_filter)]
-
- parallel_filter = "{alias}.id BETWEEN %(lower-bound)s AND %(upper-bound)s".format(alias=alias)
- query = _explode_format(query.replace("%", "%%"), parallel_filter=parallel_filter)
-
- return [
- cr.mogrify(query, {"lower-bound": ids[i], "upper-bound": ids[i + 1] - 1}).decode() for i in range(len(ids) - 1)
- ]
-
-
-def explode_execute(cr, query, table, alias=None, bucket_size=DEFAULT_BUCKET_SIZE, logger=_logger):
- """
- Execute a query in parallel.
-
- The query is split by buckets of ids, then processed in parallel by workers. If the
- query does not include the special `{parallel_filter}` value, it is added to the last
- `WHERE` clause, possibly also adding it if none found. When the query already has the
- filter nothing is done. The filter always expands to the splitting strategy. The split
- is done into buckets where no more than `bucket_size` IDs are updated on each
- individual query.
-
- .. example::
- .. code-block:: python
-
- util.explode_execute(
- cr,
- '''
- UPDATE res_users u
- SET active = False
- WHERE (u.login LIKE 'dummy' OR u.login = 'bob')
- AND {parallel_filter}
- ''',
- table="res_users"
- alias="u",
- )
-
- :param str query: the query to execute.
- :param str table: name of the *main* table of the query, used to split the processing
- :param str alias: alias used for the main table in the query
- :param int bucket_size: size of the buckets of ids to split the processing
- :param logger: logger used to report the progress
- :type logger: :class:`logging.Logger`
- :return: the sum of `cr.rowcount` for each query run
- :rtype: int
-
- .. warning::
- It's up to the caller to ensure the queries do not update the same records in
- different buckets. It is advised to never use this function for `DELETE` queries on
- tables with self references due to the potential `ON DELETE` effects.
- For more details see :func:`~odoo.upgrade.util.pg.parallel_execute`.
- """
- return parallel_execute(
- cr,
- explode_query_range(cr, query, table, alias=alias, bucket_size=bucket_size),
- logger=logger,
- )
-
-
-def pg_array_uniq(a, drop_null=False):
- dn = "WHERE x IS NOT NULL" if drop_null else ""
- return SQLStr("ARRAY(SELECT x FROM unnest({0}) x {1} GROUP BY x)".format(a, dn))
-
-
-def pg_replace(s, replacements):
- q = lambda s: psycopg2.extensions.QuotedString(s).getquoted().decode("utf-8")
-
- def replace(s, r):
- func = "regexp_replace({}, {}, {}, 'g')" if isinstance(r[0], PGRegexp) else "replace({}, {}, {})"
- return func.format(s, q(r[0]), q(r[1]))
-
- return SQLStr(reduce(replace, replacements, s))
-
-
-def pg_html_escape(s, quote=True):
- """
- Generate the SQL expression to HTML escape a string.
-
- SQL version of `html.escape`.
-
- :meta private: exclude from online docs
- """
- replacements = [
- ("&", "&"), # Must be done first!
- ("<", "<"),
- (">", ">"),
- ]
- if quote:
- replacements += [
- ('"', """),
- ("'", "'"),
- ]
- return pg_replace(s, replacements)
-
-
-def pg_text2html(s, wrap="p"):
- return SQLStr(
- r"""
- CASE WHEN TRIM(COALESCE({src}, '')) ~ '^<.+$' THEN {src}
- ELSE CONCAT(
- '{opening_tag}',
- replace(
- replace(REGEXP_REPLACE({esc},
- -- regex from https://blog.codinghorror.com/the-problem-with-urls/
- -- double the %% to allow this code chunk to be used in parameterized queries
- 'https?://[-A-Za-z0-9+&@#/%%?=~_()|!:,.;]*[-A-Za-z0-9+&@#/%%=~_()|]',
- '\&',
- 'g'),
- E'\n',
- '
'),
- E'\t',
- ' '),
- '{closing_tag}')
- END
- """.format(
- opening_tag="<{}>".format(wrap) if wrap else "",
- closing_tag="".format(wrap) if wrap else "",
- src=s,
- esc=pg_html_escape(s, quote=False),
- )
- )
-
-
-def get_value_or_en_translation(cr, table, column):
- fmt = "{}->>'en_US'" if column_type(cr, table, column) == "jsonb" else "{}"
- return format_query(cr, fmt, column)
-
-
-def _column_info(cr, table, column):
- # -> tuple[str, int | None, bool, bool] | None
- _validate_table(table)
- cr.execute(
- """
- SELECT COALESCE(bt.typname, t.typname) AS udt_name,
- information_schema._pg_char_max_length(
- information_schema._pg_truetypid(a.*, t.*),
- information_schema._pg_truetypmod(a.*, t.*)
- ) AS char_max_length,
- NOT (a.attnotnull OR t.typtype = 'd' AND t.typnotnull) AS is_nullable,
- ( c.relkind IN ('r','p','v','f')
- AND pg_column_is_updatable(c.oid::regclass, a.attnum, false)
- ) AS is_updatable
- FROM pg_attribute a
- JOIN pg_class c
- ON a.attrelid = c.oid
- JOIN pg_type t
- ON a.atttypid = t.oid
- LEFT JOIN pg_type bt
- ON t.typtype = 'd'
- AND t.typbasetype = bt.oid
- WHERE c.relname = %s
- AND a.attname = %s
- """,
- [table, column],
- )
- return cr.fetchone()
-
-
-_COLUMNS = {}
-if version_gte("10.0"): # Since at least 9.0
- _COLUMNS[("ir_model_fields", "depends")] = True
- _COLUMNS[("ir_model_fields", "domain")] = True
- _COLUMNS[("ir_model_fields", "relation_table")] = True
-
- _COLUMNS[("ir_ui_view", "arch_db")] = True
- _COLUMNS[("ir_ui_view", "active")] = True
-
- _COLUMNS[("ir_filters", "sort")] = True
- _COLUMNS[("ir_filters", "domain")] = True
- _COLUMNS[("ir_filters", "model_id")] = True
-
- _COLUMNS[("ir_model_rule", "domain_force")] = True
-
- _COLUMNS[("ir_model_data", "res_id")] = True
- _COLUMNS[("ir_model_data", "model")] = True
- _COLUMNS[("ir_model_data", "name")] = True
-
- _COLUMNS[("ir_attachment", "res_field")] = True
- _COLUMNS[("ir_attachment", "res_model")] = True
- _COLUMNS[("ir_attachment", "res_id")] = True
-
- _COLUMNS[("ir_ui_menu", "action")] = True
-
- _COLUMNS[("ir_act_window", "res_model")] = True
- _COLUMNS[("ir_act_window", "res_id")] = True
- _COLUMNS[("ir_act_window", "domain")] = True
-
- _COLUMNS[("ir_value", "res_id")] = True
-
-if version_gte("14.0"): # Since at least 13.0
- _COLUMNS[("ir_module_module_dependency", "auto_install_required")] = True
- _COLUMNS[("ir_act_window", "src_model")] = False
-
-if version_gte("saas~16.1"): # Since 16.0
- _COLUMNS[("ir_translation", "name")] = False
-
-if version_gte("saas~17.1"): # Since 17.0
- _COLUMNS[("ir_cron", "resource_ref")] = True
- _COLUMNS[("ir_act_server", "resource_ref")] = True
-
-if version_gte("saas~18.1"): # Since 18.0
- _COLUMNS[("ir_model_fields", "company_dependent")] = True
- _COLUMNS[("ir_filters", "embedded_parent_res_id")] = True
- _COLUMNS[("ir_act_report_xml", "domain")] = True
-
-
-def column_exists(cr, table, column):
- """
- Return whether a column exists.
-
- :param str table: table to check
- :param str column: column to check
- :rtype: bool
- """
- return _COLUMNS[(table, column)] if (table, column) in _COLUMNS else (_column_info(cr, table, column) is not None)
-
-
-def column_type(cr, table, column, sized=False):
- """
- Return the type of a column, if it exists.
-
- :param str table: table to check
- :param str column: column to check
- :rtype: SQL type of the column
- """
- nfo = _column_info(cr, table, column)
- if not nfo:
- return None
- if sized and nfo[1]:
- return "{}({})".format(nfo[0], nfo[1])
- return nfo[0] if nfo else None
-
-
-def column_nullable(cr, table, column):
- nfo = _column_info(cr, table, column)
- return nfo and nfo[2]
-
-
-def column_updatable(cr, table, column):
- nfo = _column_info(cr, table, column)
- return nfo and nfo[3]
-
-
-def _normalize_pg_type(type_):
- main_type, suffix = re.match(r"(.+?)((?:\[\]|\([0-9]+\))*)$", type_).groups()
- aliases = {
- "boolean": "bool",
- "smallint": "int2",
- "integer": "int4",
- "bigint": "int8",
- "real": "float4",
- "double precision": "float8",
- "character varying": "varchar",
- "timestamp with time zone": "timestamptz",
- "timestamp without time zone": "timestamp",
- }
- return aliases.get(main_type.strip().lower(), main_type) + suffix
-
-
-def create_column(cr, table, column, definition, **kwargs):
- """
- Create a column.
-
- This function will create the column only if it *doesn't* exist. It will log an error
- if the existing column has different type. If `fk_table` is set, it will ensure the
- foreign key is setup, updating if necessary, with the right `on_delete_action` if any
- is set.
-
- :param str table: table of the new column
- :param str column: name of the new column
- :param str definition: column type of the new column
- :param bool default: default value to set on the new column
- :param bool fk_table: if the new column if a foreign key, name of the foreign table
- :param str on_delete_action: `ON DELETE` clause, default `NO ACTION`, only valid if
- the column is a foreign key.
- :return: whether the column was created
- :rtype: bool
- """
- # Manual PEP 3102
- no_def = Sentinel("no_def")
- default = kwargs.pop("default", no_def)
- fk_table = kwargs.pop("fk_table", no_def)
- on_delete_action = kwargs.pop("on_delete_action", no_def)
- if kwargs:
- raise TypeError("create_column() got an unexpected keyword argument %r" % kwargs.popitem()[0])
-
- fk = ""
- if fk_table is not no_def:
- if on_delete_action is no_def:
- on_delete_action = "NO ACTION"
- elif on_delete_action not in ON_DELETE_ACTIONS:
- raise ValueError("unexpected value for the `on_delete_action` argument: %r" % (on_delete_action,))
- fk = (
- sql.SQL("REFERENCES {}(id) ON DELETE {}")
- .format(sql.Identifier(fk_table), sql.SQL(on_delete_action))
- .as_string(cr._cnx)
- )
- elif on_delete_action is not no_def:
- raise ValueError("`on_delete_action` argument can only be used if `fk_table` argument is set.")
-
- definition = _normalize_pg_type(definition)
-
- if definition == "bool" and default is no_def:
- default = False
-
- curtype = column_type(cr, table, column, sized=True)
- if curtype:
- if curtype != definition:
- _logger.error("%s.%s already exists but is %r instead of %r", table, column, curtype, definition)
- if fk_table is not no_def:
- create_fk(cr, table, column, fk_table, on_delete_action)
- if default is not no_def:
- query = 'UPDATE "{0}" SET "{1}" = %s WHERE "{1}" IS NULL'.format(table, column)
- query = cr.mogrify(query, [default]).decode()
- parallel_execute(cr, explode_query_range(cr, query, table=table))
- return False
-
- create_query = """ALTER TABLE "%s" ADD COLUMN "%s" %s %s""" % (table, column, definition, fk)
- if default is no_def:
- cr.execute(create_query)
- else:
- cr.execute(create_query + " DEFAULT %s", [default])
- cr.execute("""ALTER TABLE "%s" ALTER COLUMN "%s" DROP DEFAULT""" % (table, column))
- return True
-
-
-def create_fk(cr, table, column, fk_table, on_delete_action="http://github.com/NO%20ACTION"):
- assert on_delete_action in ON_DELETE_ACTIONS
- current_target = target_of(cr, table, column)
- if current_target:
- if current_target[:2] == (fk_table, "id"):
- # assume the `on_delete_action` is correct
- return
- cr.execute(
- sql.SQL("ALTER TABLE {} DROP CONSTRAINT {}").format(
- sql.Identifier(table), sql.Identifier(current_target[2])
- )
- )
-
- query = sql.SQL("ALTER TABLE {} ADD FOREIGN KEY ({}) REFERENCES {}(id) ON DELETE {}").format(
- sql.Identifier(table), sql.Identifier(column), sql.Identifier(fk_table), sql.SQL(on_delete_action)
- )
- cr.execute(query)
-
-
-def remove_column(cr, table, column, cascade=False):
- if column_exists(cr, table, column):
- drop_depending_views(cr, table, column)
- drop_cascade = " CASCADE" if cascade else ""
- cr.execute('ALTER TABLE "{0}" DROP COLUMN "{1}"{2}'.format(table, column, drop_cascade))
-
-
-def alter_column_type(cr, table, column, type, using=None, where=None, logger=_logger):
- """
- Alter the type of a column.
-
- Do it efficiently using a temporary column and parallel UPDATE queries.
-
- :param str table: name of the affected table.
- :param str column: name of the column to alter type.
- :param str type: new type of the column.
- :param str using: SQL expression that define how to convert the value to the new type.
- The `{0}` placeholder will be replaced by the column name.
- :param str where: used to restrict the values converted via `using`.
- :param logger: logger used to report the progress.
- :type logger: :class:`logging.Logger`
- """
- if where and not using:
- raise ValueError("`where` parameter is only relevant with a non-default `using` parameter")
-
- if not using:
- current_type = column_type(cr, table, column, sized=True)
- if current_type and current_type == _normalize_pg_type(type):
- logger.info("Column %r of table %r is already defined as %r", column, table, type)
- return
-
- # remove the existing linked `ir_model_fields_selection` records in case it was a selection field
- if table_exists(cr, "ir_model_fields_selection"):
- cr.execute(
- """
- DELETE FROM ir_model_fields_selection s
- USING ir_model_fields f
- WHERE f.id = s.field_id
- AND f.model = %s
- AND f.name = %s
- """,
- [model_of_table(cr, table), column],
- )
-
- drop_depending_views(cr, table, column)
-
- if not using:
- # if there is a high number of NULL entries, it will be faster to just ignore those
- cr.execute(format_query(cr, "ANALYZE {}({})", table, column))
- cr.execute(
- "SELECT null_frac FROM pg_stats WHERE schemaname = current_schema() AND tablename = %s AND attname = %s",
- [table, column],
- )
- [null_frac] = cr.fetchone() or (0.0,)
- if null_frac <= 0.70:
- # Simple case. Use general SQL syntax
- cr.execute(format_query(cr, "ALTER TABLE {} ALTER COLUMN {} TYPE {}", table, column, sql.SQL(type)))
- return
-
- using = "{{0}}::{}".format(type)
-
- # else, create a new column and parallel update queries.
- tmp_column = "_{}_upg".format(column)
- cr.execute(format_query(cr, "ALTER TABLE {} RENAME COLUMN {} TO {}", table, column, tmp_column))
- cr.execute(format_query(cr, "ALTER TABLE {} ADD COLUMN {} {}", table, column, sql.SQL(type)))
-
- using = format_query(cr, using, tmp_column)
- if where is None:
- where_clause = sql.SQL("")
- if column_type(cr, table, tmp_column) != "bool":
- where_clause = format_query(cr, "WHERE {} IS NOT NULL", tmp_column)
- else:
- where_clause = format_query(cr, "WHERE " + where, tmp_column)
-
- explode_execute(
- cr,
- format_query(cr, "UPDATE {} SET {} = {} {}", table, column, using, where_clause),
- table=table,
- logger=logger,
- )
-
- cr.execute(format_query(cr, "ALTER TABLE {} DROP COLUMN {} CASCADE", table, tmp_column))
-
-
-def table_exists(cr, table):
- _validate_table(table)
- cr.execute(
- """
- SELECT 1
- FROM information_schema.tables
- WHERE table_name = %s
- AND table_type = 'BASE TABLE'
- """,
- [table],
- )
- return cr.fetchone() is not None
-
-
-def sequence_exists(cr, sequence):
- cr.execute("SELECT 1 FROM information_schema.sequences WHERE sequence_name = %s", [sequence])
- return bool(cr.rowcount)
-
-
-def view_exists(cr, view):
- _validate_table(view)
- cr.execute("SELECT 1 FROM information_schema.views WHERE table_name=%s", [view])
- return bool(cr.rowcount)
-
-
-def remove_constraint(cr, table, name, cascade=False, warn=True):
- """
- Remove a table constraint.
-
- This function removes the constraint `name` from `table`. It also removes records from
- `ir_model_constraint` and its xml_ids. It logs not found constraints.
-
- .. note::
-
- If there is no constraint `name`, this function will attempt to remove
- ``{table}_{name}``, the latter is the default name the ORM uses for constraints
- created from `_sql_constraints`.
-
- :param str table: table from where to remove the constraint
- :param str name: name of the constraint to remove
- :param bool cascade: cascade the constraint removal
- :param bool warn: use warning level when logging not found constraints, otherwise use
- info level
- :return: whether the constraint was removed
- :rtype: bool
- """
- _validate_table(table)
- log = _logger.warning if warn else _logger.info
- cascade = SQLStr("CASCADE" if cascade else "")
- cr.execute(format_query(cr, "ALTER TABLE {} DROP CONSTRAINT IF EXISTS {} {}", table, name, cascade))
- # Exceptionally remove Odoo records, even if we are in PG land on this file. This is somehow
- # valid because ir.model.constraint are ORM low-level objects that relate directly to table
- # constraints.
- cr.execute("DELETE FROM ir_model_constraint WHERE name = %s RETURNING id", [name])
- if cr.rowcount:
- ids = tuple(c for (c,) in cr.fetchall())
- cr.execute("DELETE FROM ir_model_data WHERE model = 'ir.model.constraint' AND res_id IN %s", [ids])
- # The constraint was found remove any index with same name
- cr.execute(format_query(cr, "DROP INDEX IF EXISTS {}", name))
- return True
- if name.startswith(table + "_"):
- log("%r not found in ir_model_constraint, table=%r", name, table)
- return False
- log("%r not found in ir_model_constraint, attempting to remove with table %r prefix", name, table)
- return remove_constraint(cr, table, "{}_{}".format(table, name), cascade, warn)
-
-
-def get_fk(cr, table, quote_ident=True):
- """
- Return the list of foreign keys pointing to `table`.
-
- returns a 4 tuple: (foreign_table, foreign_column, constraint_name, on_delete_action)
-
- Foreign key deletion action code:
- a = no action, r = restrict, c = cascade, n = set null, d = set default
-
- :meta private: exclude from online docs
- """
- _validate_table(table)
- funk = "quote_ident" if quote_ident else "concat"
- q = """SELECT {funk}(cl1.relname) as table,
- {funk}(att1.attname) as column,
- {funk}(con.conname) as conname,
- con.confdeltype
- FROM pg_constraint as con, pg_class as cl1, pg_class as cl2,
- pg_attribute as att1, pg_attribute as att2
- WHERE con.conrelid = cl1.oid
- AND con.confrelid = cl2.oid
- AND array_lower(con.conkey, 1) = 1
- AND con.conkey[1] = att1.attnum
- AND att1.attrelid = cl1.oid
- AND cl2.relname = %s
- AND att2.attname = 'id'
- AND array_lower(con.confkey, 1) = 1
- AND con.confkey[1] = att2.attnum
- AND att2.attrelid = cl2.oid
- AND con.contype = 'f'
- """.format(funk=funk)
- cr.execute(q, (table,))
- return cr.fetchall()
-
-
-def target_of(cr, table, column):
- """
- Return the target of a foreign key.
-
- Returns None if there is not foreign key on given column.
- returns a 3-tuple (foreign_table, foreign_column, constraint_name)
-
- :meta private: exclude from online docs
- """
- cr.execute(
- """
- SELECT quote_ident(cl2.relname) as table,
- quote_ident(att2.attname) as column,
- quote_ident(con.conname) as conname
- FROM pg_constraint con
- JOIN pg_class cl1 ON (con.conrelid = cl1.oid)
- JOIN pg_attribute att1 ON ( array_lower(con.conkey, 1) = 1
- AND con.conkey[1] = att1.attnum
- AND att1.attrelid = cl1.oid)
- JOIN pg_class cl2 ON (con.confrelid = cl2.oid)
- JOIN pg_attribute att2 ON ( array_lower(con.confkey, 1) = 1
- AND con.confkey[1] = att2.attnum
- AND att2.attrelid = cl2.oid)
- WHERE cl1.relname = %s
- AND att1.attname = %s
- AND con.contype = 'f'
- """,
- [table, column],
- )
- return cr.fetchone()
-
-
-class IndexInfo(collections.namedtuple("IndexInfo", "name on isunique isconstraint ispk")):
- """:meta private: exclude from online docs."""
-
- def drop(self, cr):
- if self.isconstraint:
- remove_constraint(cr, self.on, self.name, warn=False)
- else:
- cr.execute('DROP INDEX "%s"' % self.name)
-
-
-def get_index_on(cr, table, *columns):
- """
- Return an optional IndexInfo records.
-
- NOTE: column order is respected
-
- Prefer primary keys over other indexes as the caller may want to verify PK existence before create one
-
- :meta private: exclude from online docs
- """
- _validate_table(table)
-
- if cr._cnx.server_version >= 90500:
- position = "array_position(x.indkey, x.unnest_indkey)"
- else:
- # array_position does not exists prior postgresql 9.5
- position = "strpos(array_to_string(x.indkey::int4[] || 0, ','), x.unnest_indkey::varchar || ',')"
- cr.execute(
- """
- SELECT name, on_, indisunique, indisconstraint, indisprimary
- FROM (SELECT i.relname as name,
- c.relname as on_,
- x.indisunique,
- t.conname IS NOT NULL as indisconstraint,
- x.indisprimary,
- array_agg(a.attname::text order by {}) as attrs
- FROM (select *, unnest(indkey) as unnest_indkey from pg_index) x
- JOIN pg_class c ON c.oid = x.indrelid
- JOIN pg_class i ON i.oid = x.indexrelid
- JOIN pg_attribute a ON (a.attrelid=c.oid AND a.attnum=x.unnest_indkey)
- LEFT JOIN pg_constraint t ON ( t.connamespace = i.relnamespace
- AND t.conname = i.relname
- AND t.contype IN ('u'::"char", 'p'::"char")
- AND x.indisunique AND t.conrelid = c.oid)
- WHERE (c.relkind = ANY (ARRAY['r'::"char", 'm'::"char"]))
- AND i.relkind = 'i'::"char"
- AND c.relname = %s
- GROUP BY 1, 2, 3, 4, 5
- ) idx
- WHERE attrs = %s
- ORDER BY indisprimary DESC
- FETCH FIRST ROW ONLY
- """.format(position),
- [table, list(columns)],
- )
- return IndexInfo(*cr.fetchone()) if cr.rowcount else None
-
-
-def _get_unique_indexes_with(cr, table, *columns):
- # (Cursor, str, *str) -> List[Tuple[str, List[str]]
- """
- Return all unique indexes on at least `columns`.
-
- return a list of tuple [index_name, list_of_column]
- """
- _validate_table(table)
- assert columns
- cr.execute(
- """
- SELECT name, attrs
- FROM (SELECT quote_ident(i.relname) as name,
- array_agg(a.attname::text) as attrs
- FROM (select *, unnest(indkey) as unnest_indkey from pg_index) x
- JOIN pg_class c ON c.oid = x.indrelid
- JOIN pg_class i ON i.oid = x.indexrelid
- JOIN pg_attribute a ON (a.attrelid=c.oid AND a.attnum=x.unnest_indkey)
- WHERE (c.relkind = ANY (ARRAY['r'::"char", 'm'::"char"]))
- AND i.relkind = 'i'::"char"
- AND c.relname = %s
- AND x.indisunique
- GROUP BY 1
- ) idx
- WHERE attrs @> %s
- """,
- [table, list(columns)],
- )
- return cr.fetchall()
-
-
-def create_index(cr, name, table_name, *columns):
- # create index if table and columns exists and index don't already exists
- _validate_table(table_name)
- if not columns:
- raise SleepyDeveloperError("Missing `columns` for index")
- if all(column_exists(cr, table_name, c) for c in columns) and get_index_on(cr, table_name, *columns) is None:
- cr.execute(
- "CREATE INDEX {index_name} ON {table_name}({columns})".format(
- index_name=name, table_name=table_name, columns=",".join(columns)
- )
- )
- return True
- return False
-
-
-@contextmanager
-def temp_index(cr, table, *columns):
- # create a temporary index that will be removed at the end of the contextmanager
- assert columns
- _validate_table(table)
- name = "_".join(("_upg", table) + columns + (hex(int(time.time() * 1000))[2:],))
- create_index(cr, name, table, *columns)
- try:
- yield
- finally:
- cr.execute('DROP INDEX IF EXISTS "{}"'.format(name))
-
-
-def get_depending_views(cr, table, column):
- # http://stackoverflow.com/a/11773226/75349
- _validate_table(table)
- q = """
- SELECT distinct quote_ident(dependee.relname), dependee.relkind
- FROM pg_depend
- JOIN pg_rewrite ON pg_depend.objid = pg_rewrite.oid
- JOIN pg_class as dependee ON pg_rewrite.ev_class = dependee.oid
- JOIN pg_class as dependent ON pg_depend.refobjid = dependent.oid
- JOIN pg_attribute ON pg_depend.refobjid = pg_attribute.attrelid
- AND pg_depend.refobjsubid = pg_attribute.attnum
- WHERE dependent.relname = %s
- AND pg_attribute.attnum > 0
- AND pg_attribute.attname = %s
- AND dependee.relkind in ('v', 'm')
- """
- cr.execute(q, [table, column])
- return cr.fetchall()
-
-
-# sentinel object for function parameters to not alter.
-KEEP_CURRENT = Sentinel("KEEP_CURRENT")
-
-
-class ColumnList(UserList, sql.Composable):
- """
- Encapsulate a list of elements that represent column names.
-
- The resulting object can be rendered as string with leading/trailing comma or an alias.
-
- :param list(str) list_: list of column names
- :param list(str) quoted: list of quoted column names, it must correspond with the
- `list_` parameter
-
- .. example::
- >>> columns = ColumnList(["id", "field_Yx"], ["id", '"field_Yx"'])
-
- >>> list(columns)
- ['id', '"field_Yx"']
-
- >>> columns.using(alias="t").as_string(cr._cnx)
- '"t"."id", "t"."field_Yx"'
-
- >>> columns.using(leading_comma=True).as_string(cr._cnx)
- ', "id", "field_Yx"'
-
- >>> util.format_query(cr, "SELECT {} t.name FROM table t", columns.using(alias="t", trailing_comma=True))
- 'SELECT "t"."id", "t"."field_Yx", t.name FROM table t'
-
- >>> columns = ColumnList.from_unquoted(cr, ["foo", "BAR"])
-
- >>> list(columns)
- ['"foo"', '"BAR"']
-
- >>> list(columns.iter_unquoted())
- ['foo', 'BAR']
-
- .. note::
- This class is better used via :func:`~odoo.upgrade.util.pg.get_columns`
- """
-
- def __init__(self, list_=(), quoted=()):
- assert len(list_) == len(quoted)
- self._unquoted_columns = list(list_)
- super(ColumnList, self).__init__(quoted)
- self._leading_comma = False
- self._trailing_comma = False
- self._alias = None
-
- def __hash__(self):
- return hash((tuple(self._unquoted_columns), self._leading_comma, self._trailing_comma, self._alias))
-
- def __eq__(self, other):
- return (
- isinstance(other, ColumnList)
- and other.data == self.data
- and other._unquoted_columns == self._unquoted_columns
- and other._leading_comma == self._leading_comma
- and other._trailing_comma == self._trailing_comma
- and other._alias == self._alias
- )
-
- @classmethod
- def from_unquoted(cls, cr, list_):
- """
- Build a ColumnList from a list of column names that may need quoting.
-
- :param list(str) list_: list of unquoted column names
- """
- quoted = [quote_ident(c, cr._cnx) for c in list_]
- return cls(list_, quoted)
-
- def using(self, leading_comma=KEEP_CURRENT, trailing_comma=KEEP_CURRENT, alias=KEEP_CURRENT):
- """
- Set up parameters to render this list as a string.
-
- :param bool leading_comma: whether to render a leading comma in front of this list
- :param bool trailing_comma: whether to render a trailing comma
- :param str or None alias: alias of the table of the columns, no alias is added if
- set to `None`
- :return: a copy of the list with the parameters set
- :rtype: :class:`~odoo.upgrade.util.pg.ColumnList`
- """
- if (
- (leading_comma is KEEP_CURRENT or self._leading_comma is leading_comma)
- and (trailing_comma is KEEP_CURRENT or self._trailing_comma is trailing_comma)
- and (alias is KEEP_CURRENT or self._alias == alias)
- ):
- return self
- new = ColumnList(self._unquoted_columns, self.data)
- new._leading_comma = self._leading_comma if leading_comma is KEEP_CURRENT else bool(leading_comma)
- new._trailing_comma = self._trailing_comma if trailing_comma is KEEP_CURRENT else bool(trailing_comma)
- new._alias = self._alias if alias is KEEP_CURRENT else str(alias) if alias is not None else None
- return new
-
- def as_string(self, context):
- """:meta private: exclude from online docs."""
- head = sql.SQL(", " if self._leading_comma and self else "")
- tail = sql.SQL("," if self._trailing_comma and self else "")
-
- if not self._alias:
- builder = sql.Identifier
- elif hasattr(sql.Identifier, "strings"):
- builder = partial(sql.Identifier, self._alias)
- else:
- # older psycopg2 versions, doesn't support passing multiple strings to the constructor
- builder = lambda elem: sql.SQL(".").join(sql.Identifier(self._alias) + sql.Identifier(elem))
-
- body = sql.SQL(", ").join(builder(elem) for elem in self._unquoted_columns)
- return sql.Composed([head, body, tail]).as_string(context)
-
- def iter_unquoted(self):
- """
- Iterate over the raw column names, non quoted.
-
- This is useful if the quoting is done outside this object. Also to get access to
- raw column names as in Postgres catalog.
-
- :return: an iterator for the raw column names
- """
- return iter(self._unquoted_columns)
-
-
-def get_columns(cr, table, ignore=("id",)):
- """
- Return a list of columns in a table.
-
- :param str table: table name whose columns are retrieved
- :param list(str) ignore: list of column names to ignore in the returning list
- :return: a list of column names present in the table
- :rtype: :class:`~odoo.upgrade.util.pg.ColumnList`
- """
- _validate_table(table)
-
- cr.execute(
- """
- SELECT coalesce(array_agg(column_name::varchar ORDER BY column_name), ARRAY[]::varchar[]),
- coalesce(array_agg(quote_ident(column_name) ORDER BY column_name), ARRAY[]::varchar[])
- FROM information_schema.columns
- WHERE table_schema = 'public'
- AND table_name = %s
- AND column_name != ALL(%s)
- """,
- [table, list(ignore)],
- )
- return ColumnList(*cr.fetchone())
-
-
-def get_common_columns(cr, table1, table2, ignore=("id",)):
- """
- Return a list of columns present in both tables.
-
- :param str table1: first table name whose columns are retrieved
- :param str table2: second table name whose columns are retrieved
- :param list(str) ignore: list of column names to ignore in the returning list
- :return: a list of column names present in both tables
- :rtype: :class:`~odoo.upgrade.util.pg.ColumnList`
- """
- _validate_table(table1)
- _validate_table(table2)
-
- cr.execute(
- """
- WITH _common AS (
- SELECT column_name
- FROM information_schema.columns
- WHERE table_schema = 'public'
- AND table_name IN %s
- AND column_name != ALL(%s)
- GROUP BY column_name
- HAVING count(table_name) = 2
- )
- SELECT coalesce(array_agg(column_name::varchar ORDER BY column_name), ARRAY[]::varchar[]),
- coalesce(array_agg(quote_ident(column_name) ORDER BY column_name), ARRAY[]::varchar[])
- FROM _common
- """,
- [(table1, table2), list(ignore)],
- )
- return ColumnList(*cr.fetchone())
-
-
-def rename_table(cr, old_table, new_table, remove_constraints=True):
- """
- Rename a table.
-
- This function renames the table `old_table` into `new_table`, as well as its primary
- key (and sequence), indexes, and foreign keys.
-
- :param str old_table: name of the table to rename
- :param str new_table: new name of the table
- :para bool remove_constraints: whether to remove the table constraints
- """
- if not table_exists(cr, old_table):
- return
-
- if table_exists(cr, new_table):
- raise MigrationError(
- "Table {new_table} already exists. Can't rename table {old_table} to {new_table}.".format(**locals())
- )
-
- cr.execute(sql.SQL("ALTER TABLE {} RENAME TO {}").format(sql.Identifier(old_table), sql.Identifier(new_table)))
-
- # rename pkey sequence
- cr.execute(
- sql.SQL("ALTER SEQUENCE IF EXISTS {} RENAME TO {}").format(
- sql.Identifier(old_table + "_id_seq"), sql.Identifier(new_table + "_id_seq")
- )
- )
-
- # track renamed table
- if table_exists(cr, "upgrade_test_data"):
- cr.execute(
- """
- UPDATE upgrade_test_data
- SET value = replace(value::text, %s, %s)::jsonb
- WHERE key = 'base.tests.test_moved0.TestMoved0'
- """,
- ['["{}",'.format(old_table), '["{}",'.format(new_table)],
- )
-
- # find and rename pkey, may still use an old name from a former migration
- cr.execute(
- """
- SELECT pgc.conname
- FROM pg_constraint pgc
- JOIN pg_index pgi
- ON pgi.indrelid = pgc.conrelid
- AND pgi.indexrelid = pgc.conindid
- AND pgi.indrelid = %s::regclass
- AND pgi.indisprimary
- WHERE pgc.confrelid = 0
- """,
- [new_table],
- )
- if cr.rowcount:
- (old_pkey,) = cr.fetchone()
- new_pkey = new_table + "_pkey"
- cr.execute(sql.SQL("ALTER INDEX {} RENAME TO {}").format(sql.Identifier(old_pkey), sql.Identifier(new_pkey)))
- else:
- new_pkey = "" # no PK renamed
-
- # rename indexes (except the pkey's one and those not containing $old_table)
- cr.execute(
- """
- SELECT indexname
- FROM pg_indexes
- WHERE tablename = %s
- AND indexname != %s
- AND indexname LIKE %s
- """,
- [new_table, new_pkey, "%" + old_table.replace("_", r"\_") + r"\_%"],
- )
- for (idx,) in cr.fetchall():
- cr.execute(
- sql.SQL("ALTER INDEX {} RENAME TO {}").format(
- sql.Identifier(idx),
- sql.Identifier(idx.replace(old_table, new_table)),
- )
- )
-
- if remove_constraints:
- # DELETE all constraints, except Primary/Foreign keys, they will be re-created by the ORM
- # NOTE: Custom constraints will instead be lost
- cr.execute(
- """
- SELECT constraint_name
- FROM information_schema.table_constraints
- WHERE table_name = %s
- AND constraint_name !~ '^[0-9_]+_not_null$'
- AND ( constraint_type NOT IN ('PRIMARY KEY', 'FOREIGN KEY')
- -- For long table names the constraint name is shortened by PG to fit 63 chars, in such cases
- -- it's better to drop the constraint, even if it's a foreign key, and let the ORM re-create it.
- OR (constraint_type = 'FOREIGN KEY' AND constraint_name NOT LIKE %s)
- )
- """,
- [new_table, old_table.replace("_", r"\_") + r"\_%"],
- )
- for (const,) in cr.fetchall():
- _logger.info("Dropping constraint %s on table %s", const, new_table)
- remove_constraint(cr, new_table, const, warn=False)
-
- # rename fkeys
- cr.execute(
- """
- SELECT constraint_name
- FROM information_schema.table_constraints
- WHERE table_name = %s
- AND constraint_type = 'FOREIGN KEY'
- AND constraint_name LIKE %s
- """,
- [new_table, old_table.replace("_", r"\_") + r"\_%"],
- )
- old_table_length = len(old_table)
- for (old_fkey,) in cr.fetchall():
- new_fkey = new_table + old_fkey[old_table_length:]
- _logger.info("Renaming FK %r to %r", old_fkey, new_fkey)
- cr.execute(
- sql.SQL("ALTER TABLE {} RENAME CONSTRAINT {} TO {}").format(
- sql.Identifier(new_table), sql.Identifier(old_fkey), sql.Identifier(new_fkey)
- )
- )
-
-
-def find_new_table_column_name(cr, table, name):
- columns = get_columns(cr, table)
- i = 0
- while name in columns:
- i += 1
- name = "%s_%s" % (name, i)
- return name
-
-
-def drop_depending_views(cr, table, column):
- """
- Drop views depending on a field to allow the ORM to resize it in-place.
-
- :meta private: exclude from online docs
- """
- for v, k in get_depending_views(cr, table, column):
- cr.execute("DROP {0} VIEW IF EXISTS {1} CASCADE".format("MATERIALIZED" if k == "m" else "", v))
-
-
-def create_m2m(cr, m2m, fk1, fk2, col1=None, col2=None):
- """
- Ensure a m2m table exists or is created.
-
- This function creates the table associated to a m2m field.
- If the table already exists, :func:`~odoo.upgrade.util.pg.fixup_m2m` is run on it.
- The table name can be generated automatically, applying the same logic of the ORM.
- In order to do so, use the value "auto" for the `m2m` parameter.
-
- :param str m2m: table name to create,
- if :const:`~odoo.upgrade.util.misc.AUTO` it is auto-generated
- :param str fk1: first foreign key table name
- :param str fk2: second foreign key table name
- :param str col1: column referencing `fk1`, defaults to `"{fk1}_id"`
- :param str col2: column referencing `fk2`, defaults to `"{fk2}_id"`
- :return: the name of the table just created/fixed-up
- :rtype: str
- """
- if col1 is None:
- col1 = "%s_id" % fk1
- if col2 is None:
- col2 = "%s_id" % fk2
-
- if m2m is AUTO:
- m2m = "{}_{}_rel".format(*sorted([fk1, fk2]))
-
- if table_exists(cr, m2m):
- fixup_m2m(cr, m2m, fk1, fk2, col1, col2)
- return m2m
-
- query = format_query(
- cr,
- """
- CREATE TABLE {m2m}(
- {col1} integer NOT NULL REFERENCES {fk1}(id) ON DELETE CASCADE,
- {col2} integer NOT NULL REFERENCES {fk2}(id) ON DELETE CASCADE,
- PRIMARY KEY ({col1}, {col2})
- );
- CREATE INDEX ON {m2m}({col2}, {col1});
- """,
- m2m=m2m,
- col1=col1,
- col2=col2,
- fk1=fk1,
- fk2=fk2,
- )
- cr.execute(query)
-
- return m2m
-
-
-def update_m2m_tables(cr, old_table, new_table, ignored_m2ms=()):
- """
- Update m2m table names and columns.
-
- This function renames m2m tables still referring to `old_table`. It also updates
- column names and constraints of those tables.
-
- :param str old_table: former table name
- :param str new_table: new table name
- :param list(str) ignored_m2ms: explicit list of m2m tables to ignore
-
- :meta private: exclude from online docs
- """
- assert isinstance(ignored_m2ms, (list, tuple))
- if old_table == new_table or not version_gte("10.0"):
- return
- ignored_m2ms = set(ignored_m2ms)
- for orig_m2m_table in get_m2m_tables(cr, new_table):
- if orig_m2m_table in ignored_m2ms:
- continue
- m = re.match(r"^(x_|)(?:(\w+)_{0}|{0}_(\w+))_rel$".format(re.escape(old_table)), orig_m2m_table)
- if m:
- m2m_table = "{}{}_{}_rel".format(m.group(1), *sorted([m.group(2) or m.group(3), new_table]))
- # Due to the 63 chars limit in generated constraint names, for long table names the FK
- # constraint is dropped when renaming the table. We need the constraint to correctly
- # identify the FK targets. The FK constraints will be dropped and recreated below.
- rename_table(cr, orig_m2m_table, m2m_table, remove_constraints=False)
- cr.execute(
- """
- UPDATE ir_model_fields
- SET relation_table = %s
- WHERE relation_table = %s
- AND state = 'manual'
- """,
- [m2m_table, orig_m2m_table],
- )
- _logger.info("Renamed m2m table %s to %s", orig_m2m_table, m2m_table)
- else:
- m2m_table = orig_m2m_table
- for m2m_col in get_columns(cr, m2m_table).iter_unquoted():
- col_info = target_of(cr, m2m_table, m2m_col)
- if not col_info or col_info[0] != new_table or col_info[1] != "id":
- continue
- old_col, new_col = map("{}_id".format, [old_table, new_table])
- if m2m_col != old_col:
- _logger.warning(
- "Possibly missing rename: the column %s of m2m table %s references the table %s",
- m2m_col,
- m2m_table,
- new_table,
- )
- continue
- old_constraint = col_info[2]
- cr.execute(
- """
- SELECT c.confdeltype
- FROM pg_constraint c
- JOIN pg_class t
- ON c.conrelid = t.oid
- WHERE t.relname = %s
- AND c.conname = %s
- """,
- [m2m_table, old_constraint],
- )
- on_delete = cr.fetchone()[0]
- query = format_query(
- cr,
- """
- ALTER TABLE {m2m_table}
- RENAME COLUMN {old_col} TO {new_col};
-
- ALTER TABLE {m2m_table}
- DROP CONSTRAINT {old_constraint},
- ADD FOREIGN KEY ({new_col}) REFERENCES {new_table} (id) ON DELETE {del_action}
- """,
- m2m_table=m2m_table,
- old_col=old_col,
- new_col=new_col,
- old_constraint=old_constraint,
- new_table=new_table,
- del_action=SQLStr("RESTRICT") if on_delete == "r" else SQLStr("CASCADE"),
- )
- cr.execute(query)
-
- cr.execute(
- """
- UPDATE ir_model_fields
- SET column1 = %s
- WHERE relation_table = %s
- AND column1 = %s
- AND state = 'manual'
- """,
- [new_col, m2m_table, old_col],
- )
- cr.execute(
- """
- UPDATE ir_model_fields
- SET column2 = %s
- WHERE relation_table = %s
- AND column2 = %s
- AND state = 'manual'
- """,
- [new_col, m2m_table, old_col],
- )
-
- _logger.info("Renamed m2m column of table %s from %s to %s", m2m_table, old_col, new_col)
-
-
-def fixup_m2m(cr, m2m, fk1, fk2, col1=None, col2=None):
- if col1 is None:
- col1 = "%s_id" % fk1
- if col2 is None:
- col2 = "%s_id" % fk2
-
- if not table_exists(cr, m2m):
- return
-
- extra_columns = get_columns(cr, m2m, ignore=(col1, col2))
- if extra_columns:
- raise MigrationError("The m2m %r has extra columns: %s" % (m2m, ", ".join(extra_columns)))
-
- # cleanup
- fixup_m2m_cleanup(cr, m2m, col1, col2)
- cr.execute(
- """
- DELETE FROM "{m2m}" t
- WHERE NOT EXISTS (SELECT id FROM "{fk1}" WHERE id=t."{col1}")
- OR NOT EXISTS (SELECT id FROM "{fk2}" WHERE id=t."{col2}")
- """.format(**locals())
- )
- deleted = cr.rowcount
- if deleted:
- _logger.debug("%(m2m)s: removed %(deleted)d invalid rows", locals())
-
- # set not null
- cr.execute('ALTER TABLE "{m2m}" ALTER COLUMN "{col1}" SET NOT NULL'.format(**locals()))
- cr.execute('ALTER TABLE "{m2m}" ALTER COLUMN "{col2}" SET NOT NULL'.format(**locals()))
-
- # create missing or bad fk
- target = target_of(cr, m2m, col1)
- if target and target[:2] != (fk1, "id"):
- cr.execute('ALTER TABLE "{m2m}" DROP CONSTRAINT {con}'.format(m2m=m2m, con=target[2]))
- target = None
- if not target:
- _logger.debug("%(m2m)s: add FK %(col1)s -> %(fk1)s", locals())
- cr.execute(
- 'ALTER TABLE "{m2m}" ADD FOREIGN KEY ("{col1}") REFERENCES "{fk1}" ON DELETE CASCADE'.format(**locals())
- )
-
- target = target_of(cr, m2m, col2)
- if target and target[:2] != (fk2, "id"):
- cr.execute('ALTER TABLE "{m2m}" DROP CONSTRAINT {con}'.format(m2m=m2m, con=target[2]))
- target = None
- if not target:
- _logger.debug("%(m2m)s: add FK %(col2)s -> %(fk2)s", locals())
- cr.execute(
- 'ALTER TABLE "{m2m}" ADD FOREIGN KEY ("{col2}") REFERENCES "{fk2}" ON DELETE CASCADE'.format(**locals())
- )
-
- # create indexes
- fixup_m2m_indexes(cr, m2m, col1, col2)
-
-
-def fixup_m2m_cleanup(cr, m2m, col1, col2):
- cr.execute(
- """
- DELETE FROM "{m2m}" t
- WHERE "{col1}" IS NULL
- OR "{col2}" IS NULL
- """.format(**locals())
- )
- deleted = cr.rowcount
- if deleted:
- _logger.debug("%(m2m)s: removed %(deleted)d rows with NULL values", locals())
-
- # remove duplicated rows
- cr.execute(
- """
- DELETE FROM "{m2m}"
- WHERE ctid IN (SELECT ctid
- FROM (SELECT ctid,
- ROW_NUMBER() OVER (PARTITION BY "{col1}", "{col2}"
- ORDER BY ctid) as rnum
- FROM "{m2m}") t
- WHERE t.rnum > 1)
- """.format(**locals())
- )
- deleted = cr.rowcount
- if deleted:
- _logger.debug("%(m2m)s: removed %(deleted)d duplicated rows", locals())
-
-
-def fixup_m2m_indexes(cr, m2m, col1, col2):
- idx1 = get_index_on(cr, m2m, col1, col2)
- idx2 = get_index_on(cr, m2m, col2, col1)
-
- if not idx1 and not idx2:
- # No index at all
- cr.execute('ALTER TABLE "%s" ADD PRIMARY KEY("%s", "%s")' % (m2m, col1, col2))
- cr.execute('CREATE INDEX ON "%s" ("%s", "%s")' % (m2m, col2, col1))
- elif idx1 and idx2:
- if not idx1.ispk and not idx2.ispk:
- # None is the PK. Create one
- idx1.drop(cr)
- cr.execute('ALTER TABLE "%s" ADD PRIMARY KEY("%s", "%s")' % (m2m, col1, col2))
- else:
- # only 1 index exist, create the second one
- # determine which one is missing
- fmt = (m2m, col2, col1) if idx1 else (m2m, col1, col2)
- existing = idx1 or idx2
- if existing.ispk:
- # the existing index is the PK, create a normal index
- cr.execute('CREATE INDEX ON "%s" ("%s", "%s")' % fmt)
- elif existing.isunique:
- # it's a unique index. Remove it and recreate a PK and a normal index
- existing.drop(cr)
- cr.execute('ALTER TABLE "%s" ADD PRIMARY KEY("%s", "%s")' % (m2m, col1, col2))
- cr.execute('CREATE INDEX ON "%s" ("%s", "%s")' % (m2m, col2, col1))
- else:
- # create a PK (unique index)
- cr.execute('ALTER TABLE "%s" ADD PRIMARY KEY("%s", "%s")' % fmt)
-
- # remove indexes on 1 column only
- idx = get_index_on(cr, m2m, col1)
- if idx:
- idx.drop(cr)
- idx = get_index_on(cr, m2m, col2)
- if idx:
- idx.drop(cr)
-
-
-def fix_wrong_m2o(cr, table, column, target, value=None):
- cr.execute(
- """
- WITH wrongs_m2o AS (
- SELECT s.id
- FROM {table} s
- LEFT JOIN {target} t
- ON s.{column} = t.id
- WHERE s.{column} IS NOT NULL
- AND t.id IS NULL
- )
- UPDATE {table} s
- SET {column}=%s
- FROM wrongs_m2o w
- WHERE s.id = w.id
- """.format(table=table, column=column, target=target),
- [value],
- )
-
-
-def get_m2m_tables(cr, table):
- """
- Return a list of m2m tables associated with `table`.
-
- We identify as m2m table all tables that have only two columns, both of which are FKs.
- This function will return m2m tables for which one Fk points to `table`
-
- :meta private: exclude from online docs
- """
- _validate_table(table)
- query = """
- WITH two_cols AS (
- SELECT t.oid
- FROM pg_class t
- JOIN pg_attribute a ON a.attrelid=t.oid
- WHERE t.relkind='r' AND a.attnum>0
- GROUP BY t.oid
- HAVING count(*)=2
- )
- SELECT DISTINCT t.relname
- FROM pg_class t
- JOIN two_cols tc ON t.oid=tc.oid
- JOIN pg_attribute a1 ON a1.attrelid=t.oid AND a1.attnum>0
- JOIN pg_constraint c1 ON c1.conrelid=t.oid AND c1.contype='f' AND a1.attnum=any(c1.conkey)
- AND array_length(c1.conkey, 1)=1
- JOIN pg_attribute a2 ON a2.attrelid=t.oid AND a2.attnum>0 AND a1.attnum!=a2.attnum
- JOIN pg_constraint c2 ON c2.conrelid=t.oid AND c2.contype='f' AND a2.attnum=any(c2.conkey)
- AND array_length(c1.conkey, 1)=1
- JOIN pg_class the_table ON c1.confrelid=the_table.oid
- WHERE the_table.relkind='r' AND the_table.relname=%s
- """
-
- cr.execute(query, [table])
- return [row[0] for row in cr.fetchall()]
-
-
-class named_cursor(object):
- def __init__(self, cr, itersize=None):
- self._ncr = cr._cnx.cursor("upg_nc_" + uuid.uuid4().hex, withhold=True)
- if itersize:
- self._ncr.itersize = itersize
-
- def __dictrow(self, row):
- return {d.name: v for d, v in zip(self._ncr.description, row)}
-
- def dictfetchall(self):
- return list(map(self.__dictrow, self._ncr.fetchall()))
-
- def dictfetchmany(self, size):
- return list(map(self.__dictrow, self._ncr.fetchmany(size)))
-
- def dictfetchone(self):
- row = self._ncr.fetchone()
- return None if row is None else self.__dictrow(row)
-
- def iterdict(self):
- return map(self.__dictrow, self._ncr)
-
- def __iter__(self):
- return self._ncr.__iter__()
-
- def __enter__(self):
- self._ncr.__enter__()
- return self
-
- def __exit__(self, exc_type, exc_value, traceback):
- return self._ncr.__exit__(exc_type, exc_value, traceback)
-
- def __getattr__(self, name):
- return getattr(self._ncr, name)
-
-
-def create_id_sequence(cr, table, set_as_default=True):
- if not table_exists(cr, table):
- raise MigrationError("The table `%s` doesn't exist, sequence can't be created." % table)
-
- # inheritance lookup
- cr.execute(
- """
- WITH RECURSIVE recursive_lookup AS (
- SELECT oid AS table_oid,
- 0 AS level
- FROM pg_class
- WHERE relname = %s
- UNION
- -- add parents with id column recursively
- SELECT i.inhparent AS table_oid,
- recursive_lookup.level + 1 AS level
- FROM pg_inherits i
- JOIN recursive_lookup
- ON i.inhrelid = recursive_lookup.table_oid
- JOIN pg_attribute c
- ON i.inhparent = c.attrelid
- WHERE c.attname = 'id'
- ) SELECT t.relname
- FROM pg_class t
- JOIN recursive_lookup
- ON t.oid = recursive_lookup.table_oid
- ORDER BY level DESC LIMIT 1
- """,
- [table],
- )
- table = cr.fetchone()[0] # will be different from param if inherited
- sequence = table + "_id_seq"
- sequence_sql, table_sql = sql.Identifier(sequence), sql.Identifier(table)
-
- if not sequence_exists(cr, sequence):
- cr.execute(
- sql.SQL("CREATE SEQUENCE {sequence} OWNED BY {table}.id").format(
- sequence=sequence_sql,
- table=table_sql,
- )
- )
-
- cr.execute(
- sql.SQL("SELECT setval('{sequence}', (SELECT COALESCE(max(id), 0) FROM {table}) + 1, false)").format(
- sequence=sequence_sql,
- table=table_sql,
- )
- )
- if set_as_default:
- cr.execute(
- sql.SQL("ALTER TABLE ONLY {table} ALTER COLUMN id SET DEFAULT nextval('{sequence}'::regclass)").format(
- sequence=sequence_sql,
- table=table_sql,
- )
- )
-
-
-def bulk_update_table(cr, table, columns, mapping, key_col="id"):
- """
- Update table based on mapping.
-
- Each `mapping` entry defines the new values for the specified `columns` for the row(s)
- whose `key_col` value matches the key.
-
- .. example::
-
- .. code-block:: python
-
- # single column update
- util.bulk_update_table(cr, "res_users", "active", {42: False, 27: True})
-
- # multi-column update
- util.bulk_update_table(
- cr,
- "res_users",
- ["active", "password"],
- {
- "admin": [True, "1234"],
- "demo": [True, "5678"],
- },
- key_col="login",
- )
-
- :param str table: table to update.
- :param str | list(str) columns: columns spec for the update. It could be a single
- column name or a list of column names. The `mapping`
- must match the spec.
- :param dict mapping: values to set, which must match the spec in `columns`,
- following the **same** order
- :param str key_col: column used as key to get the values from `mapping` during the
- update.
-
- .. warning::
-
- The values in the mapping will be casted to the type of the target column.
- This function is designed to update scalar values, avoid setting arrays or json
- data via the mapping.
- """
- _validate_table(table)
- if not columns or not mapping:
- return
-
- assert isinstance(mapping, dict)
- if isinstance(columns, str):
- columns = [columns]
- else:
- n_columns = len(columns)
- assert all(isinstance(value, (list, tuple)) and len(value) == n_columns for value in mapping.values())
-
- query = format_query(
- cr,
- """
- UPDATE {table} t
- SET ({cols}) = ROW({cols_values})
- FROM JSONB_EACH(%s) m
- WHERE t.{key_col}::text = m.key
- """,
- table=table,
- cols=ColumnList.from_unquoted(cr, columns),
- cols_values=SQLStr(
- ", ".join(
- "(m.value->>{:d})::{}".format(col_idx, column_type(cr, table, col_name, sized=True))
- for col_idx, col_name in enumerate(columns)
- )
- ),
- key_col=key_col,
- )
- cr.execute(query, [Json(mapping)])
diff --git a/src/util/records.py b/src/util/records.py
deleted file mode 100644
index bd19ff92e..000000000
--- a/src/util/records.py
+++ /dev/null
@@ -1,1998 +0,0 @@
-"""Utility functions for record-level operations."""
-
-import logging
-import os
-import re
-import uuid
-from contextlib import contextmanager
-from operator import itemgetter
-
-import lxml
-from psycopg2 import sql
-from psycopg2.extras import Json, execute_values
-
-try:
- from odoo import modules, release
- from odoo.tools.convert import xml_import
- from odoo.tools.misc import file_open
- from odoo.tools.translate import xml_translate
-except ImportError:
- from openerp import modules, release
- from openerp.tools.convert import xml_import
- from openerp.tools.misc import file_open
-
-from .const import NEARLYWARN
-from .exceptions import MigrationError
-from .helpers import (
- _get_theme_models,
- _ir_values_value,
- _validate_model,
- model_of_table,
- resolve_model_fields_path,
- table_of_model,
-)
-from .inconsistencies import break_recursive_loops
-from .indirect_references import indirect_references
-from .inherit import direct_inherit_parents, for_each_inherit
-from .misc import AUTOMATIC, chunks, parse_version, version_gte
-from .orm import env, flush
-from .pg import (
- PGRegexp,
- SQLStr,
- _get_unique_indexes_with,
- _validate_table,
- column_exists,
- column_type,
- column_updatable,
- explode_execute,
- explode_query_range,
- format_query,
- get_columns,
- get_fk,
- get_value_or_en_translation,
- parallel_execute,
- table_exists,
- target_of,
-)
-from .report import add_to_migration_reports
-
-_logger = logging.getLogger(__name__)
-
-# python3 shims
-try:
- basestring # noqa: B018
-except NameError:
- basestring = unicode = str
-
-
-def remove_view(cr, xml_id=None, view_id=None, silent=False, key=None):
- """
- Remove a view and all its descendants.
-
- This function recursively deletes the given view and its inherited views, as long as
- they are part of a module. It will fail as soon as a custom view exists anywhere in
- the hierarchy. It also removes multi-website COWed views.
-
- :param str xml_id: optional, the xml_id of the view to remove
- :param int view_id: optional, the ID of the view to remove
- :param bool silent: whether to show in the logs disabled custom views
- :param str or None key: key used to detect multi-website COWed views, if `None` then
- set to `xml_id` if provided, otherwise set to the xml_id
- referencing the view with ID `view_id` if any
-
- .. warning::
- Either `xml_id` or `view_id` must be set. Specifying both will raise an error.
- """
- assert bool(xml_id) ^ bool(view_id)
- if xml_id:
- view_id = ref(cr, xml_id)
- if view_id:
- module, _, name = xml_id.partition(".")
- cr.execute("SELECT model FROM ir_model_data WHERE module=%s AND name=%s", [module, name])
-
- [model] = cr.fetchone()
- if model != "ir.ui.view":
- raise ValueError("%r should point to a 'ir.ui.view', not a %r" % (xml_id, model))
- else:
- # search matching xmlid for logging or renaming of custom views
- xml_id = "?"
- if not key:
- cr.execute("SELECT module, name FROM ir_model_data WHERE model='ir.ui.view' AND res_id=%s", [view_id])
- if cr.rowcount:
- xml_id = "%s.%s" % cr.fetchone()
-
- # From given or determined xml_id, the views duplicated in a multi-website
- # context are to be found and removed.
- if xml_id != "?" and column_exists(cr, "ir_ui_view", "key"):
- cr.execute("SELECT id FROM ir_ui_view WHERE key = %s AND id != %s", [xml_id, view_id])
- for [v_id] in cr.fetchall():
- remove_view(cr, view_id=v_id, silent=silent, key=xml_id)
-
- if not key and column_exists(cr, "ir_ui_view", "key"):
- cr.execute("SELECT key FROM ir_ui_view WHERE id = %s and key != %s", [view_id, xml_id])
- [key] = cr.fetchone() or [None]
-
- # Occurrences of xml_id and key in the t-call of views are to be found and removed.
- if xml_id != "?":
- _remove_redundant_tcalls(cr, xml_id)
- if key and key != xml_id:
- _remove_redundant_tcalls(cr, key)
-
- if not view_id:
- return
-
- cr.execute(
- """
- SELECT v.id, x.module || '.' || x.name, v.name
- FROM ir_ui_view v LEFT JOIN
- ir_model_data x ON (v.id = x.res_id AND x.model = 'ir.ui.view' AND x.module !~ '^_')
- WHERE v.inherit_id = %s;
- """,
- [view_id],
- )
- for child_id, child_xml_id, child_name in cr.fetchall():
- if child_xml_id:
- if not silent:
- _logger.info(
- "remove deprecated built-in view %s (ID %s) as parent view %s (ID %s) is going to be removed",
- child_xml_id,
- child_id,
- xml_id,
- view_id,
- )
- remove_view(cr, child_xml_id, silent=True)
- else:
- if not silent:
- _logger.warning(
- "deactivate deprecated custom view with ID %s as parent view %s (ID %s) is going to be removed",
- child_id,
- xml_id,
- view_id,
- )
- disable_view_query = """
- UPDATE ir_ui_view
- SET name = (name || ' - old view, inherited from ' || %%s),
- inherit_id = NULL
- %s
- WHERE id = %%s
- """
- # In 8.0, disabling requires setting mode to 'primary'
- extra_set_sql = ""
- if column_exists(cr, "ir_ui_view", "mode"):
- extra_set_sql = ", mode = 'primary' "
-
- # Column was not present in v7 and it's older version
- if column_exists(cr, "ir_ui_view", "active"):
- extra_set_sql += ", active = false "
-
- disable_view_query = disable_view_query % extra_set_sql
- cr.execute(disable_view_query, (key or xml_id, child_id))
- add_to_migration_reports(
- {"id": child_id, "name": child_name},
- "Disabled views",
- )
- if not silent:
- _logger.info("remove deprecated %s view %s (ID %s)", (key and "COWed") or "built-in", key or xml_id, view_id)
-
- remove_records(cr, "ir.ui.view", [view_id])
-
-
-@contextmanager
-def edit_view(cr, xmlid=None, view_id=None, skip_if_not_noupdate=True, active="auto"):
- """
- Context manager to edit a view's arch.
-
- This function returns a context manager that may yield a parsed arch of a view as an
- `etree Element `_. Any changes done
- in the returned object will be written back to the database upon exit of the context
- manager, updating also the translated versions of the arch. Since the function may not
- yield, use :func:`~odoo.upgrade.util.misc.skippable_cm` to avoid errors.
-
- .. code-block:: python
-
- with util.skippable_cm(), util.edit_view(cr, "xml.id") as arch:
- arch.attrib["string"] = "My Form"
-
- To select the target view to edit use either `xmlid` or `view_id`, not both.
-
- When the view is identified by `view_id`, the arch is always yielded if the view
- exists, with disregard to any `noupdate` flag it may have associated. When `xmlid` is
- set, if the view `noupdate` flag is `False` then the arch will not be yielded *unless*
- `skip_if_not_noupdate` is set to `False`. If `noupdate` is `True`, the view will be
- yielded for edit.
-
- If the `active` argument is `True` or `False`, the `active` flag of the view will be set
- accordingly.
-
- .. note::
- If `active` is "auto" (default value), the view will be activated if selected
- via `xmlid` and left untouched if selected via `view_id`.
-
- :param str xmlid: optional, xml_id of the view edit
- :param int view_id: optional, ID of the view to edit
- :param bool skip_if_not_noupdate: whether to force the edit of views requested via
- `xmlid` parameter even if they are flagged as
- `noupdate=True`, ignored if `view_id` is set
- :param bool or None or "auto" active: active flag value to set. Unchanged when `None`.
- :return: a context manager that yields the parsed arch, upon exit the context manager
- writes back the changes.
- """
- assert bool(xmlid) ^ bool(view_id), "You Must specify either xmlid or view_id"
- if active not in (True, False, None, "auto"):
- raise ValueError("Invalid `active` value: {!r}".format(active))
- if active == "auto":
- active = True if xmlid else None
-
- noupdate = True
- if xmlid:
- if "." not in xmlid:
- raise ValueError("Please use fully qualified name .")
-
- module, _, name = xmlid.partition(".")
- cr.execute(
- """
- SELECT res_id, noupdate
- FROM ir_model_data
- WHERE module = %s
- AND name = %s
- """,
- [module, name],
- )
- data = cr.fetchone()
- if data:
- view_id, noupdate = data
-
- if view_id and not (skip_if_not_noupdate and not noupdate):
- arch_col = "arch_db" if column_exists(cr, "ir_ui_view", "arch_db") else "arch"
- jsonb_column = column_type(cr, "ir_ui_view", arch_col) == "jsonb"
- cr.execute(
- """
- SELECT {arch}
- FROM ir_ui_view
- WHERE id=%s
- """.format(
- arch=arch_col,
- ),
- [view_id],
- )
- [arch] = cr.fetchone() or [None]
- if arch:
-
- def parse(arch):
- arch = arch.encode("utf-8") if isinstance(arch, unicode) else arch
- return lxml.etree.fromstring(arch.replace(b"
\n", b"\n").strip())
-
- if jsonb_column:
-
- def get_trans_terms(value):
- terms = []
- xml_translate(terms.append, value)
- return terms
-
- translation_terms = {lang: get_trans_terms(value) for lang, value in arch.items()}
- arch_etree = parse(arch["en_US"])
- yield arch_etree
- new_arch = lxml.etree.tostring(arch_etree, encoding="unicode")
- terms_en = translation_terms["en_US"]
- arch_column_value = Json(
- {
- lang: xml_translate(dict(zip(terms_en, terms)).get, new_arch)
- for lang, terms in translation_terms.items()
- }
- )
- else:
- arch_etree = parse(arch)
- yield arch_etree
- arch_column_value = lxml.etree.tostring(arch_etree, encoding="unicode")
-
- set_active = ", active={}".format(bool(active)) if active is not None else ""
- cr.execute(
- "UPDATE ir_ui_view SET {arch}=%s{set_active} WHERE id=%s".format(arch=arch_col, set_active=set_active),
- [arch_column_value, view_id],
- )
-
-
-def add_view(cr, name, model, view_type, arch_db, inherit_xml_id=None, priority=16, key=None):
- inherit_id = None
- if inherit_xml_id:
- inherit_id = ref(cr, inherit_xml_id)
- if not inherit_id:
- raise ValueError(
- "Unable to add view '%s' because its inherited view '%s' cannot be found!" % (name, inherit_xml_id)
- )
- # Odoo <= 8.0 doesn't have the `key`
- key_exist = column_exists(cr, "ir_ui_view", "key")
- if key_exist and view_type == "qweb" and not key:
- key = "gen_key.%s" % str(uuid.uuid4())[:6]
- arch_col = "arch_db" if column_exists(cr, "ir_ui_view", "arch_db") else "arch"
- jsonb_column = column_type(cr, "ir_ui_view", arch_col) == "jsonb"
- arch_column_value = Json({"en_US": arch_db}) if jsonb_column else arch_db
- cr.execute(
- """
- INSERT INTO ir_ui_view(name, "type", model, inherit_id, mode, active, priority, %s)
- VALUES(%%(name)s, %%(view_type)s, %%(model)s, %%(inherit_id)s, %%(mode)s, 't', %%(priority)s, %%(arch_db)s %s)
- RETURNING id
- """
- % (arch_col + (", key" if key_exist else ""), ", %(key)s" if key_exist else ""),
- {
- "name": name,
- "view_type": view_type,
- "model": model,
- "inherit_id": inherit_id,
- "mode": "extension" if inherit_id else "primary",
- "priority": priority,
- "key": key,
- "arch_db": arch_column_value,
- },
- )
- return cr.fetchone()[0]
-
-
-# fmt:off
-if version_gte("saas~14.3"):
- def remove_asset(cr, name):
- cr.execute("SELECT id FROM ir_asset WHERE bundle = %s", [name])
- if cr.rowcount:
- remove_records(cr, "ir.asset", [aid for aid, in cr.fetchall()])
-else:
- def remove_asset(cr, name):
- remove_view(cr, name, silent=True)
-# fmt:on
-
-
-def remove_record(cr, name):
- """
- Remove a record and its references corresponding to the given :term:`xml_id `.
-
- :param str name: record xml_id, under the format `module.name`
- """
- if isinstance(name, basestring):
- if "." not in name:
- raise ValueError("Please use fully qualified name .")
- module, _, name = name.partition(".")
- cr.execute(
- """
- SELECT model, res_id
- FROM ir_model_data
- WHERE module = %s
- AND name = %s
- """,
- [module, name],
- )
- if not cr.rowcount:
- return None
- model, res_id = cr.fetchone()
- elif isinstance(name, tuple):
- if len(name) != 2:
- raise ValueError("Please use a 2-tuple (, )")
- model, res_id = name
- else:
- raise TypeError("Either use a fully qualified xmlid string . or a 2-tuple (, )")
-
- # deleguate to the right method
- if model == "ir.ui.view":
- _logger.log(NEARLYWARN, "Removing view %r", name)
- return remove_view(cr, view_id=res_id)
-
- if model == "ir.ui.menu":
- _logger.log(NEARLYWARN, "Removing menu %r", name)
- return remove_menus(cr, [res_id])
-
- if model == "res.groups":
- _logger.log(NEARLYWARN, "Removing group %r", name)
- return remove_group(cr, group_id=res_id)
-
- return remove_records(cr, model, [res_id])
-
-
-def remove_records(cr, model, ids):
- if not ids:
- return
-
- ids = tuple(ids)
-
- # remove theme model's copy_ids
- theme_copy_model = _get_theme_models().get(model)
- if theme_copy_model:
- cr.execute(
- 'SELECT id FROM "{}" WHERE theme_template_id IN %s'.format(table_of_model(cr, theme_copy_model)),
- [ids],
- )
- if theme_copy_model == "ir.ui.view":
- for (view_id,) in cr.fetchall():
- remove_view(cr, view_id=view_id)
- else:
- remove_records(cr, theme_copy_model, [rid for (rid,) in cr.fetchall()])
-
- for inh in for_each_inherit(cr, model, skip=()):
- if inh.via:
- table = table_of_model(cr, inh.model)
- if not column_exists(cr, table, inh.via):
- # column may not exists in case of a partially unintalled module that left only *magic columns* in tables
- continue
- cr.execute('SELECT id FROM "{}" WHERE "{}" IN %s'.format(table, inh.via), [ids])
- if inh.model == "ir.ui.menu":
- remove_menus(cr, [menu_id for (menu_id,) in cr.fetchall()])
- elif inh.model == "ir.ui.view":
- for (view_id,) in cr.fetchall():
- remove_view(cr, view_id=view_id)
- else:
- remove_records(cr, inh.model, [rid for (rid,) in cr.fetchall()])
-
- table = table_of_model(cr, model)
- base_query = format_query(cr, "DELETE FROM {} WHERE id IN %s", table)
- parallel_execute(
- cr,
- [cr.mogrify(base_query, [chunk_ids]).decode() for chunk_ids in chunks(ids, 1000, fmt=tuple)],
- )
- for ir in indirect_references(cr, bound_only=True):
- if not ir.company_dependent_comodel:
- query = 'DELETE FROM "{}" WHERE {} AND "{}" IN %s'.format(ir.table, ir.model_filter(), ir.res_id)
- cr.execute(query, [model, ids])
- elif ir.company_dependent_comodel == model:
- json_path = cr.mogrify(
- "$.* ? ({})".format(" || ".join(["@ == %s"] * len(ids))),
- ids,
- ).decode()
-
- query = cr.mogrify(
- format_query(
- cr,
- """
- UPDATE {table}
- SET {column} = (
- SELECT jsonb_object_agg(
- key,
- CASE
- WHEN value::int4 IN %s THEN NULL
- ELSE value::int4
- END)
- FROM jsonb_each_text({column})
- )
- WHERE {column} IS NOT NULL
- AND {column} @? {json_path}
- """,
- table=ir.table,
- column=ir.res_id,
- json_path=sql.Literal(json_path),
- ),
- [ids],
- ).decode()
- explode_execute(cr, query, table=ir.table)
- _rm_refs(cr, model, ids)
-
- if model == "res.groups":
- # A group is gone, the auto-generated view `base.user_groups_view` is outdated.
- # Create a shim. It will be re-generated later by creating/updating groups or
- # explicitly in `base/0.0.0/end-user_groups_view.py`.
- arch_col = "arch_db" if column_exists(cr, "ir_ui_view", "arch_db") else "arch"
- jsonb_column = column_type(cr, "ir_ui_view", arch_col) == "jsonb"
- arch_value = "json_build_object('en_US', '')" if jsonb_column else "''"
- cr.execute(
- "UPDATE ir_ui_view SET {} = {} WHERE id = %s".format(arch_col, arch_value),
- [ref(cr, "base.user_groups_view")],
- )
-
-
-def _rm_refs(cr, model, ids=None):
- if ids is None:
- match = "like %s"
- needle = model + ",%"
- else:
- if not ids:
- return
- match = "in %s"
- needle = tuple("{0},{1}".format(model, i) for i in ids)
-
- # "model-comma" fields
- cr.execute(
- """
- SELECT model, name
- FROM ir_model_fields
- WHERE ttype='reference'
- UNION
- SELECT 'ir.translation', 'name'
- """
- )
-
- for ref_model, ref_column in cr.fetchall():
- table = table_of_model(cr, ref_model)
- if column_updatable(cr, table, ref_column):
- query_tail = ' FROM "{0}" WHERE "{1}" {2}'.format(table, ref_column, match)
- if ref_model == "ir.ui.view":
- cr.execute("SELECT id" + query_tail, [needle])
- for (view_id,) in cr.fetchall():
- remove_view(cr, view_id=view_id, silent=True)
- elif ref_model == "ir.ui.menu":
- cr.execute("SELECT id" + query_tail, [needle])
- menu_ids = tuple(m[0] for m in cr.fetchall())
- remove_menus(cr, menu_ids)
- else:
- cr.execute("SELECT id" + query_tail, [needle])
- for (record_id,) in cr.fetchall():
- remove_record(cr, (ref_model, record_id))
-
- if table_exists(cr, "ir_values"):
- column, _ = _ir_values_value(cr)
- query = "DELETE FROM ir_values WHERE {0} {1}".format(column, match)
- cr.execute(query, [needle])
-
- if ids is None and table_exists(cr, "ir_translation"):
- cr.execute(
- """
- DELETE FROM ir_translation
- WHERE name=%s
- AND type IN ('constraint', 'sql_constraint', 'view', 'report', 'rml', 'xsl')
- """,
- [model],
- )
-
-
-def _remove_import_export_paths(cr, model, field=None):
- export_q = """
- SELECT el.id,
- e.resource,
- STRING_TO_ARRAY(el.name, '/')
- FROM ir_exports_line el
- JOIN ir_exports e
- ON el.export_id = e.id
- """
- if field:
- export_q = cr.mogrify(export_q + " WHERE el.name ~ %s ", [r"\y{}\y".format(field)]).decode()
- else:
- export_q += " WHERE el.name IS NOT NULL"
-
- impex_data = [(export_q, "ir_exports_line")]
- if table_exists(cr, "base_import_mapping"):
- import_q = """
- SELECT id,
- res_model,
- STRING_TO_ARRAY(field_name, '/')
- FROM base_import_mapping
- """
- if field:
- import_q = cr.mogrify(import_q + " WHERE field_name ~ %s ", [r"\y{}\y".format(field)]).decode()
- else:
- import_q += " WHERE field_name IS NOT NULL "
- impex_data.append((import_q, "base_import_mapping"))
-
- for query, impex_model in impex_data:
- cr.execute(query)
- to_rem = [
- path_id
- for path_id, related_model, path in cr.fetchall()
- if any(
- x.field_model == model and (field is None or x.field_name == field)
- for x in resolve_model_fields_path(cr, related_model, path)
- )
- ]
- if to_rem:
- remove_records(cr, impex_model, to_rem)
-
-
-def is_changed(cr, xmlid, interval="1 minute"):
- """
- Return whether a record was changed.
-
- This function checks if a record was changed before the current upgrade start time.
- See :file:`upgrade-util/src/base/0.0.0/pre-00-upgrade-start.py`
-
- This utility will return a false positive on xmlids of records that match the
- following conditions:
-
- * Have been updated in an upgrade preceding the current one
- * Have not been updated in the current upgrade
-
- If the `xmlid` doesn't exist in the DB this function returns ``None``.
-
- :param str xmlid: `xmlid` of the record to check
- :param str interval: SQL interval, a record is considered as changed if
- `write_date > create_date + interval`
- :rtype: bool or None
- """
- assert "." in xmlid
- module, _, name = xmlid.partition(".")
- cr.execute("SELECT model, res_id FROM ir_model_data WHERE module=%s AND name=%s", [module, name])
- data = cr.fetchone()
- if not data:
- return None
- model, res_id = data
- table = table_of_model(cr, model)
- cr.execute(
- """
- SELECT 1
- FROM {} r
- LEFT JOIN ir_config_parameter p
- ON p.key = 'upgrade.start.time'
- WHERE r.id = %s
- -- Note: use a negative search to handle the case of NULL values in write/create_date
- AND COALESCE(r.write_date < p.value::timestamp, True)
- AND r.write_date - r.create_date > interval %s
- """.format(table),
- [res_id, interval],
- )
- return bool(cr.rowcount)
-
-
-def if_unchanged(cr, xmlid, callback, interval="1 minute", **kwargs):
- """
- Run `callback` if a record is unchanged.
-
- This function will run a `callback` when the referred record is unchanged. The `xmlid`
- and any extra parameter, but not `interval`, will be passed to the `callback`. In case
- the record was changed it will be marked as `noupdate=True`. See also
- :func:`~odoo.upgrade.util.records.is_changed` and
- :func:`~odoo.upgrade.util.records.force_noupdate`.
-
- This function is useful to take an action *only* when a record hasn't been updated, a
- common example is to force an update from XML even if the record was `noupdate=True`
-
- .. code-block:: python
-
- util.if_unchanged(cr, "mymodule.myrecord", util.update_record_from_xml)
-
- :param str xmlid: xml_id of the record to check
- :param function callback: callback to execute in case the record was *not* changed, all
- extra parameters to this function are passed to the callback
- :param str interval: interval after `create_date` on which a record is considered as
- _changed_, see :func:`~odoo.upgrade.util.misc.is_changed`
- """
- changed = is_changed(cr, xmlid, interval=interval)
- if changed is None:
- return
- if changed is False:
- callback(cr, xmlid, **kwargs)
- else:
- force_noupdate(cr, xmlid, noupdate=True)
-
-
-def remove_menus(cr, menu_ids):
- if not menu_ids:
- return
- cr.execute(
- """
- WITH RECURSIVE tree(id) AS (
- SELECT id
- FROM ir_ui_menu
- WHERE id IN %s
- UNION
- SELECT m.id
- FROM ir_ui_menu m
- JOIN tree t ON (m.parent_id = t.id)
- ),
- removed_menus AS (
- DELETE
- FROM ir_ui_menu m
- USING tree t
- WHERE m.id = t.id
- RETURNING m.id
- )
- DELETE
- FROM ir_model_data imd
- USING removed_menus m
- WHERE imd.res_id = m.id
- AND imd.model = 'ir.ui.menu'
- """,
- [tuple(menu_ids)],
- )
-
-
-def remove_group(cr, xml_id=None, group_id=None):
- assert bool(xml_id) ^ bool(group_id)
- if xml_id:
- group_id = ref(cr, xml_id)
- if group_id:
- module, _, name = xml_id.partition(".")
- cr.execute("SELECT model FROM ir_model_data WHERE module=%s AND name=%s", [module, name])
- [model] = cr.fetchone()
- if model != "res.groups":
- raise ValueError("%r should point to a 'res.groups', not a %r" % (xml_id, model))
-
- if not group_id:
- return
-
- # Get all fks from table res_groups
- fks = get_fk(cr, "res_groups", quote_ident=False)
-
- # Remove records referencing the group_id from the referencing tables (restrict fks)
- standard_tables = ["ir_model_access", "rule_group_rel"]
- custom_tables = []
- for foreign_table, foreign_column, _, on_delete_action in fks:
- if on_delete_action == "r":
- if foreign_table not in standard_tables:
- cr.execute(
- 'SELECT COUNT(*) FROM "{}" WHERE "{}" = %s'.format(foreign_table, foreign_column),
- (group_id,),
- )
- count = cr.fetchone()[0]
- if count:
- custom_tables.append((foreign_table, foreign_column, count))
- continue
-
- query = 'DELETE FROM "{}" WHERE "{}" = %s'.format(foreign_table, foreign_column)
- query = cr.mogrify(query, (group_id,)).decode()
-
- if column_exists(cr, foreign_table, "id"):
- parallel_execute(cr, explode_query_range(cr, query, table=foreign_table))
- else:
- cr.execute(query)
-
- if custom_tables:
- col_name = get_value_or_en_translation(cr, "res_groups", "name")
- cr.execute("SELECT {} FROM res_groups WHERE id = %s".format(col_name), [group_id])
- group_name = cr.fetchone()[0]
- raise MigrationError(
- "\nThe following 'table (column) - records count' are referencing the group '{}'".format(group_name)
- + " and cannot be removed automatically:\n"
- + "\n".join(
- " - {} ({}) - {} record(s)".format(table, column, count) for table, column, count in custom_tables
- )
- + "\nPlease remove them manually or remove the foreign key constraints set as RESTRICT."
- )
-
- remove_records(cr, "res.groups", [group_id])
-
-
-def rename_xmlid(cr, old, new, noupdate=None, on_collision="fail"):
- """
- Rename an :term:`external identifier` (`xml_id`) of a record.
-
- A rename cannot happen when the target name already exists on the database. In such
- cases there are two options to control how this function behaves:
-
- - `fail`: raise an exception and prevent renaming
- - `merge`: rename the external identifier, remove the old one, and *replace*
- references. See :func:`~odoo.upgrade.util.records.replace_record_references_batch`
-
- .. note::
- This function does not remove records, it only updates xml_ids.
-
- :param str old: current xml_id of the record, under the format `module.name`
- :param str new: new xml_id of the record, under the format `module.name`
- :param bool or None noupdate: value to set on the `noupdate` flag of the xml_id,
- ignored if `None`
- :param str on_collision: how to proceed if the xml_id already exists, the options are
- `merge` or `fail` (default)
- :return: the ID of the record referenced by the *new* xml_id, `None` when the record
- doesn't exist
- :rtype: int or None
- """
- if "." not in old or "." not in new:
- raise ValueError("Please use fully qualified name .")
- if on_collision not in {"fail", "merge"}:
- raise ValueError("Invalid value for the `on_collision` argument: {0!r}".format(on_collision))
- if old == new:
- raise ValueError("Cannot rename an XMLID to itself")
-
- old_module, _, old_name = old.partition(".")
- new_module, _, new_name = new.partition(".")
- cr.execute("SELECT model, res_id FROM ir_model_data WHERE module = %s AND name = %s", [new_module, new_name])
- new_model, new_id = cr.fetchone() or (None, None)
- cr.execute("SELECT model, res_id FROM ir_model_data WHERE module = %s AND name = %s", [old_module, old_name])
- model, old_id = cr.fetchone() or (None, None)
-
- if new_id and old_id:
- if (model, old_id) != (new_model, new_id):
- if on_collision == "fail":
- raise MigrationError("Can't rename {} to {} as it already exists".format(old, new))
-
- if model != new_model:
- raise MigrationError("Model mismatch while renaming xmlid {}. {} to {}".format(old, model, new_model))
-
- replace_record_references(cr, (model, old_id), (model, new_id), replace_xmlid=False)
-
- if noupdate is not None:
- force_noupdate(cr, new, bool(noupdate))
- cr.execute("DELETE FROM ir_model_data WHERE module=%s AND name=%s", [old_module, old_name])
- else:
- nu = "" if noupdate is None else (", noupdate=" + str(bool(noupdate)).lower())
-
- cr.execute(
- """UPDATE ir_model_data
- SET module=%s, name=%s
- {}
- WHERE module=%s AND name=%s
- RETURNING model, res_id
- """.format(nu),
- (new_module, new_name, old_module, old_name),
- )
- model, new_id = cr.fetchone() or (None, None)
-
- if model and new_id:
- if model == "ir.ui.view" and column_exists(cr, "ir_ui_view", "key"):
- cr.execute("UPDATE ir_ui_view SET key=%s WHERE id=%s AND key=%s", [new, new_id, old])
- if cr.rowcount:
- # iff the key has been updated for this view, also update it for all other cowed views.
- # Don't change the view keys unconditionally to avoid changing unrelated views.
- cr.execute("UPDATE ir_ui_view SET key = %s WHERE key = %s", [new, old])
-
- # Adapting t-call and t-name references in views
- search_pattern = r"""\yt-(call|name)=(["']){}\2""".format(re.escape(old))
- replace_pattern = r"t-\1=\2{}\2".format(new)
- if column_type(cr, "ir_ui_view", "arch_db") == "jsonb":
- replace_in_all_jsonb_values(
- cr,
- "ir_ui_view",
- "arch_db",
- PGRegexp(search_pattern),
- replace_pattern,
- extra_filter=cr.mogrify("arch_db->>'en_US' ~ %s", [search_pattern]).decode(),
- )
- else:
- arch_col = "arch_db" if column_exists(cr, "ir_ui_view", "arch_db") else "arch"
- cr.execute(
- format_query(
- cr,
- """
- UPDATE ir_ui_view
- SET {arch} = regexp_replace({arch}, %s, %s, 'g')
- WHERE {arch} ~ %s
- """,
- arch=arch_col,
- ),
- [search_pattern, replace_pattern, search_pattern],
- )
-
- if model == "ir.ui.menu" and column_exists(cr, "res_users_settings", "homemenu_config"):
- query = """
- UPDATE res_users_settings
- SET homemenu_config = regexp_replace(homemenu_config::text, %(old)s, %(new)s)::jsonb
- WHERE homemenu_config::text ~ %(old)s
- """
- cr.execute(query, {"old": r"\y{}\y".format(re.escape(old)), "new": new})
-
- for parent_model, inh in direct_inherit_parents(cr, model):
- if inh.via:
- parent = parent_model.replace(".", "_")
- rename_xmlid(
- cr,
- "{}_{}".format(old, parent),
- "{}_{}".format(new, parent),
- noupdate=noupdate,
- on_collision=on_collision,
- )
- return new_id
- return None
-
-
-def ref(cr, xmlid):
- """
- Return the id corresponding to the given :term:`xml_id `.
-
- :param str xml_id: record xml_id, under the format `module.name`
- :return: ID of the referenced record, `None` if not found.
- :rtype: int or None
- """
- if "." not in xmlid:
- raise ValueError("Please use fully qualified name .")
-
- module, _, name = xmlid.partition(".")
- cr.execute(
- """
- SELECT res_id
- FROM ir_model_data
- WHERE module = %s
- AND name = %s
- """,
- [module, name],
- )
- data = cr.fetchone()
- if data:
- return data[0]
- return None
-
-
-def force_noupdate(cr, xmlid, noupdate=True, warn=False):
- """
- Update the `noupdate` flag of a record.
-
- :param str xmlid: record xml_id, under the format `module.name`
- :param bool noupdate: value to set on the `noupdate` flag
- :param warn: whether to output a warning in the logs when the flag was switched from
- `True` to `False`
- """
- if "." not in xmlid:
- raise ValueError("Please use fully qualified name .")
-
- module, _, name = xmlid.partition(".")
- cr.execute(
- """
- UPDATE ir_model_data
- SET noupdate = %s
- WHERE module = %s
- AND name = %s
- AND noupdate != %s
- """,
- [noupdate, module, name, noupdate],
- )
- if noupdate is False and cr.rowcount and warn:
- _logger.warning("Customizations on `%s` might be lost!", xmlid)
- return cr.rowcount
-
-
-def ensure_xmlid_match_record(cr, xmlid, model, values):
- """
- Ensure an xml_id references a record with specific values.
-
- This function ensures the record pointed by an xml_id matches the values for the
- fields specified in the `values` parameter. When the xmlid exist but it points to a
- record that doesn't match the values, the xmlid is updated to point to a record that
- matches the values if one is found. If the xmlid doesn't exist, it is created with the
- found record. When no matching record is found, nothing is done. In all cases the
- referenced record, after a possible update, of the xml_id is returned.
-
- :param str xmlid: record xml_id, under the format `module.name`
- :param str model: model name of the record
- :param dict values: mapping of field names to values the record must fulfill
-
- .. example::
-
- .. code-block:: python
-
- values = {"id": 123}
- values = {"name": "INV/2024/0001", "company_id": 1}
-
- :return: the ID of the matched record, `None` if no record found
- :rtype: int or None
-
- .. tip::
- This function is useful when migrating in-database records into a custom module, to
- create the xml_ids before the module is updated and avoid duplication.
- """
- if "." not in xmlid:
- raise ValueError("Please use fully qualified name .")
-
- logger = _logger.getChild("ensure_xmlid_match_record")
-
- module, _, name = xmlid.partition(".")
- cr.execute(
- """
- SELECT id, res_id
- FROM ir_model_data
- WHERE module = %s
- AND name = %s
- """,
- [module, name],
- )
- data_id, res_id = cr.fetchone() or (None, None)
-
- table = table_of_model(cr, model)
-
- # search for existing records matching values
- where = []
- data = ()
- for k, v in values.items():
- if v is not None:
- where += ["%s = %%s" % (get_value_or_en_translation(cr, table, k),)]
- data += (v,)
- else:
- where += ["%s IS NULL" % (k,)]
- data += ()
-
- query = ("SELECT id FROM %s WHERE " % table) + " AND ".join(where)
- cr.execute(query, data)
- records = [id for (id,) in cr.fetchall()]
- if res_id and res_id in records:
- return res_id
- if not records:
- if res_id:
- logger.debug("`%s` refers %s(%s); values differ %r; no other match found.", xmlid, model, res_id, values)
- return res_id
-
- logger.debug("`%s` doesn't exist; no match found for values %r", xmlid, values)
- return None
- new_res_id = records[0]
-
- if data_id:
- logger.info("update `%s` from %s(%s) to %s(%s); values %r", xmlid, model, new_res_id, model, res_id, values)
- cr.execute(
- """
- UPDATE ir_model_data
- SET res_id=%s
- WHERE id=%s
- """,
- [new_res_id, data_id],
- )
- else:
- logger.info("create `%s` that point to %s(%s); matching values %r", xmlid, model, new_res_id, values)
- cr.execute(
- """
- INSERT INTO ir_model_data(module, name, model, res_id, noupdate)
- VALUES (%s, %s, %s, %s, %s)
- """,
- [module, name, model, new_res_id, True],
- )
-
- return new_res_id
-
-
-def update_record_from_xml(
- cr,
- xmlid,
- reset_write_metadata=True,
- force_create=AUTOMATIC,
- from_module=None,
- reset_translations=(),
- ensure_references=False,
- fields=None,
-):
- """
- Update a record based on its definition in the :doc:`/developer/reference/backend/data`.
-
- This function ignores the `noupdate` flag on the record. It searches in all XML files
- from the manifest of the module in the xmlid, or the `from_module` parameter if set,
- for a matching definition. When found, it forces the ORM to update the record as in the
- specs in the data file.
-
- Optionally this function can reset the translations of some fields.
-
- :param str xmlid: record xml_id, under the format `module.name`
- :param bool reset_write_metadata: whether to update the `write_date` of the record
- :param bool force_create: whether the record is created if it does not exist.
- `True` by default, unless `fields` is not None.
- :param str from_module: name of the module from which to update the record, necessary
- only when the specs are in a different module than the one in
- the xml_id
- :param set(str) reset_translations: field names whose translations get reset
- :param bool ensure_references: whether referred records via `ref` XML attributes
- should also be updated.
- :param set(str) or None fields: optional list of fields to include in the XML declaration.
- If set, all other fields will be ignored. When set, record
- won't be created if missing.
-
- .. warning::
- This functions uses the ORM, therefore it can only be used after **all** models
- referenced in the data specs of the record are already **loaded**. In practice this
- means that this function should be used in `post-` or `end-` scripts.
-
- .. note::
- The standard behavior of ORM is to create the record if it doesn't exist, including
- its xml_id. That will happen on this function as well, unless `force_create` is set
- to `False`.
- """
- __update_record_from_xml(
- cr,
- xmlid,
- reset_write_metadata=reset_write_metadata,
- force_create=force_create,
- from_module=from_module,
- reset_translations=reset_translations,
- ensure_references=ensure_references,
- fields=fields,
- done_refs=set(),
- )
-
-
-def __update_record_from_xml(
- cr,
- xmlid,
- reset_write_metadata,
- force_create,
- from_module,
- reset_translations,
- ensure_references,
- fields,
- done_refs,
-):
- from .modules import get_manifest # noqa: PLC0415
-
- if "." not in xmlid:
- raise ValueError("Please use fully qualified name .")
-
- if force_create is AUTOMATIC:
- force_create = fields is None # don't force record creation if we are filtering fields
-
- module, _, name = xmlid.partition(".")
-
- cr.execute(
- """
- UPDATE ir_model_data d
- SET noupdate = false
- FROM ir_model_data o
- WHERE o.id = d.id
- AND d.module = %s
- AND d.name = %s
- RETURNING d.model, d.res_id, o.noupdate
- """,
- [module, name],
- )
- if cr.rowcount:
- model, res_id, noupdate = cr.fetchone()
- if model == "ir.model":
- return
- elif not force_create:
- _logger.warning("Record %r not found in database. Skip update.", xmlid)
- return
- else:
- # The xmlid doesn't already exists, nothing to reset
- reset_write_metadata = noupdate = reset_translations = False
- fields = None
-
- write_data = None
- if reset_write_metadata:
- table = table_of_model(cr, model)
- cr.execute("SELECT write_uid, write_date, id FROM {} WHERE id=%s".format(table), [res_id])
- write_data = cr.fetchone()
-
- from_module = from_module or module
-
- id_match = (
- "@id='{module}.{name}' or @id='{name}'".format(module=module, name=name)
- if module == from_module
- else "@id='{module}.{name}'".format(module=module, name=name)
- )
- xpath = "//*[self::act_window or self::menuitem or self::record or self::report or self::template][{}]".format(id_match) # fmt: skip
-
- # use a data tag inside openerp tag to be compatible with all supported versions
- new_root = lxml.etree.fromstring("")
-
- manifest = get_manifest(from_module)
- template = False
- found = False
- extra_references = []
-
- def add_ref(ref):
- if "." not in ref:
- extra_references.append(from_module + "." + ref)
- elif ref.split(".")[0] == from_module:
- extra_references.append(ref)
-
- for f in manifest.get("data", []):
- if not f.endswith(".xml"):
- continue
- with file_open(os.path.join(from_module, f)) as fp:
- doc = lxml.etree.parse(fp)
- for node in doc.xpath(xpath):
- found = True
- parent = node.getparent()
- if node.tag == "record" and fields is not None:
- for fn in node.xpath("./field[@name]"):
- if fn.attrib["name"] not in fields:
- node.remove(fn)
- new_root[0].append(node)
-
- if node.tag == "menuitem" and parent.tag == "menuitem" and "parent_id" not in node.attrib:
- new_root[0].append(
- lxml.builder.E.record(
- lxml.builder.E.field(name="parent_id", ref=parent.attrib["id"]),
- model="ir.ui.menu",
- id=node.attrib["id"],
- )
- )
-
- if node.tag == "template":
- template = True
- if ensure_references:
- for ref_node in node.xpath("//field[@ref]"):
- if fields is not None and ref_node.attrib["name"] not in fields:
- continue
- add_ref(ref_node.get("ref"))
- for eval_node in node.xpath("//field[@eval]"):
- if fields is not None and eval_node.attrib["name"] not in fields:
- continue
- for ref_match in re.finditer(r"\bref\((['\"])(.*?)\1\)", eval_node.get("eval")):
- add_ref(ref_match.group(2))
-
- if not found:
- if noupdate:
- force_noupdate(cr, xmlid, noupdate=True)
- suffix = " in %r module" % from_module if from_module != module else ""
- raise ValueError("Cannot find %r%s" % (xmlid, suffix))
-
- done_refs.add(xmlid)
- for ref in extra_references:
- if ref in done_refs:
- continue
- _logger.info("Update of %s - ensuring the reference %s exists", xmlid, ref)
- __update_record_from_xml(
- cr,
- ref,
- reset_write_metadata=reset_write_metadata,
- force_create=True,
- from_module=from_module,
- reset_translations=reset_translations,
- ensure_references=True,
- fields=None,
- done_refs=done_refs,
- )
-
- cr_or_env = env(cr) if version_gte("saas~16.2") else cr
- importer = xml_import(cr_or_env, from_module, idref={}, mode="update")
- kw = {"mode": "update"} if parse_version("8.0") <= parse_version(release.series) <= parse_version("12.0") else {}
- importer.parse(new_root, **kw)
- if version_gte("13.0"):
- flush(env(cr)["base"])
-
- if noupdate:
- force_noupdate(cr, xmlid, noupdate=True)
- if reset_write_metadata and write_data:
- cr.execute("UPDATE {} SET write_uid=%s, write_date=%s WHERE id=%s".format(table), write_data)
-
- if reset_translations:
- if reset_translations is True:
- if fields is None:
- fields_with_values_from_xml = {elem.attrib["name"] for elem in node.xpath("//record/field")}
- if template:
- fields_with_values_from_xml |= {"arch_db", "name"}
- else:
- fields_with_values_from_xml = fields
- if version_gte("saas~18.5"): # translate is varchar
- sql_code = "SELECT name FROM ir_model_fields WHERE model = %s AND translate IS NOT NULL AND name IN %s"
- else: # translate is boolean
- sql_code = "SELECT name FROM ir_model_fields WHERE model = %s AND translate = true AND name IN %s"
- cr.execute(
- sql_code,
- [model, tuple(fields_with_values_from_xml)],
- )
- reset_translations = [fname for [fname] in cr.fetchall()]
-
- if table_exists(cr, "ir_translation"):
- cr.execute(
- """
- DELETE FROM ir_translation
- WHERE name IN %s
- AND res_id = %s
- """,
- [tuple("{},{}".format(model, f) for f in reset_translations), res_id],
- )
- else:
- query = """
- UPDATE {}
- SET {}
- WHERE id = %s
- """.format(
- table,
- ",".join(
- [
- """%s = NULLIF(jsonb_build_object('en_US', %s->>'en_US'), '{"en_US": null}'::jsonb)"""
- % (fname, fname)
- for fname in reset_translations
- ]
- ),
- )
- cr.execute(query, [res_id])
-
- env_ = env(cr)
- module_to_reload_from = env_["ir.module.module"].search(
- [("name", "=", from_module), ("state", "=", "installed")]
- )
- if module_to_reload_from:
- if hasattr(module_to_reload_from, "_update_translations"):
- module_to_reload_from._update_translations()
- else:
- # < 9.0
- module_to_reload_from.update_translations()
-
-
-def delete_unused(cr, *xmlids, **kwargs):
- """
- Remove unused records.
-
- This function will remove records pointed by `xmlids` only if they are not referenced
- from any table. For hierarchical records (like product categories), it verifies
- if the children marked as cascade removal are also not referenced. In which case
- the record and its children are all removed.
-
- .. note::
- The records that cannot be removed are set as `noupdate=True`.
-
- :param list(str) xmlids: list of xml_ids to check for removal
- :param bool deactivate: whether to deactivate records that cannot be removed because
- they are referenced, `False` by default
- :param bool keep_xmlids: whether to keep the xml_ids of records that cannot be
- removed. By default `True` for versions up to 18.0,
- `False` from `saas~18.1` on.
- :return: list of ids of removed records, if any
- :rtype: list(int)
- """
- deactivate = kwargs.pop("deactivate", False)
- keep_xmlids = kwargs.pop("keep_xmlids", not version_gte("saas~18.1"))
- if kwargs:
- raise TypeError("delete_unused() got an unexpected keyword argument %r" % kwargs.popitem()[0])
-
- select_xids = " UNION ".join(
- [
- cr.mogrify("SELECT %s::varchar as module, %s::varchar as name", [module, name]).decode()
- for xmlid in xmlids
- for module, _, name in [xmlid.partition(".")]
- ]
- )
-
- cr.execute(
- """
- WITH xids AS (
- {}
- ),
- _upd AS (
- UPDATE ir_model_data d
- SET noupdate = true
- FROM xids x
- WHERE d.module = x.module
- AND d.name = x.name
- RETURNING d.id, d.model, d.res_id, d.module || '.' || d.name as xmlid
- )
- SELECT model, array_agg(res_id ORDER BY id), array_agg(xmlid ORDER BY id)
- FROM _upd
- GROUP BY model
- """.format(select_xids)
- )
-
- deleted = []
- for model, ids, xids in cr.fetchall():
- table = table_of_model(cr, model)
- res_id_to_xmlid = dict(zip(ids, xids))
-
- cascade_children = [
- fk_col
- for fk_tbl, fk_col, _, fk_act in get_fk(cr, table, quote_ident=False)
- if fk_tbl == table and fk_act == "c"
- ]
- if cascade_children:
- if len(cascade_children) == 1:
- join = format_query(cr, "t.{}", cascade_children[0])
- else:
- join = sql.SQL("ANY(ARRAY[{}])").format(
- sql.SQL(", ").join(sql.Composed([sql.SQL("t."), sql.Identifier(cc)]) for cc in cascade_children)
- )
-
- kids_query = format_query(
- cr,
- """
- WITH RECURSIVE _child AS (
- SELECT id AS root, id AS child
- FROM {0}
- WHERE id = ANY(%(ids)s)
- UNION -- don't use UNION ALL in case we have a loop
- SELECT c.root, t.id as child
- FROM {0} t
- JOIN _child c
- ON c.child = {1}
- )
- SELECT root AS id, array_agg(child) AS children
- FROM _child
- GROUP BY root
- """,
- table,
- join,
- )
- else:
- kids_query = format_query(cr, "SELECT id, ARRAY[id] AS children FROM {0} WHERE id = ANY(%(ids)s)", table)
-
- sub = " UNION ALL ".join(
- [
- format_query(cr, "SELECT 1 FROM {} x WHERE x.{} = ANY(s.children)", fk_tbl, fk_col)
- for fk_tbl, fk_col, _, fk_act in get_fk(cr, table, quote_ident=False)
- # ignore "on delete cascade" fk (they are indirect dependencies (lines or m2m))
- if fk_act != "c"
- # ignore children records unless the deletion is restricted
- if not (fk_tbl == table and fk_act != "r")
- ]
- )
- if sub:
- query = format_query(
- cr,
- r"""
- WITH _kids AS (
- {}
- )
- SELECT t.id
- FROM {} t
- JOIN _kids s
- ON s.id = t.id
- WHERE NOT EXISTS({})
- """,
- kids_query,
- table,
- sql.SQL(sub),
- )
- cr.execute(query, {"ids": list(ids)})
- sub_ids = list(map(itemgetter(0), cr.fetchall()))
- else:
- sub_ids = list(ids)
-
- if model == "res.lang" and table_exists(cr, "ir_translation"):
- cr.execute(
- """
- DELETE FROM ir_translation t
- USING res_lang l
- WHERE t.lang = l.code
- AND l.id = ANY(%s)
- """,
- [sub_ids],
- )
-
- remove_records(cr, model, sub_ids)
- deleted.extend(res_id_to_xmlid[r] for r in sub_ids if r in res_id_to_xmlid)
-
- if deactivate:
- deactivate_ids = tuple(set(sub_ids) - set(ids))
- if deactivate_ids:
- cr.execute(format_query(cr, "UPDATE {} SET active = false WHERE id IN %s", table), [deactivate_ids])
-
- if not keep_xmlids:
- query = """
- WITH xids AS (
- {}
- )
- DELETE
- FROM ir_model_data d
- USING xids x
- WHERE d.module = x.module
- AND d.name = x.name
- """
- cr.execute(query.format(select_xids))
-
- return deleted
-
-
-def replace_record_references(cr, old, new, replace_xmlid=True, parent_field="parent_id"):
- """
- Replace all (in)direct references of a record by another.
-
- :meta private: exclude from online docs
- """
- # TODO update workflow instances?
- assert isinstance(old, tuple) and len(old) == 2
- assert isinstance(new, tuple) and len(new) == 2
-
- if not old[1]:
- return None
-
- return replace_record_references_batch(cr, {old[1]: new[1]}, old[0], new[0], replace_xmlid, parent_field)
-
-
-def replace_record_references_batch(
- cr, id_mapping, model_src, model_dst=None, replace_xmlid=True, ignores=(), parent_field="parent_id"
-):
- """
- Replace all references to records.
-
- This functions replaces *all* references, direct or indirect to records of `model_src`
- by the corresponding records in the mapping. If the target model of the mapping is not
- the same as the source one, then `model_dst` parameter must be set.
-
- :param dict(int, int) id_mapping: mapping of IDs to replace, key value is replaced by
- the mapped value
- :param str model_src: name of the source model of the records to replace
- :param str model_dst: name of the target model of the records to replace, if `None`
- the target is assumed the same as the source
- :param bool replace_xmlid: whether to replace the references in xml_ids pointing to
- the source records
- :param list(str) ignores: list of **table** names to skip when updating the referenced
- values
- :paream str parent_field: when the target and source model are the same, and the model
- table has `parent_path` column, this field will be used to
- update it.
- """
- _validate_model(model_src)
- if model_dst is None:
- model_dst = model_src
- else:
- _validate_model(model_dst)
-
- if model_src == model_dst:
- same_ids = {k: v for k, v in id_mapping.items() if k == v}
- if same_ids:
- _logger.warning("Replace references in model `%s`, ignoring same-id mapping `%s`", model_src, same_ids)
- id_mapping = {k: v for k, v in id_mapping.items() if k != v}
-
- assert id_mapping
- assert all(isinstance(v, int) and isinstance(k, int) for k, v in id_mapping.items())
-
- id_update = any(k != v for k, v in id_mapping.items())
-
- ignores = [_validate_table(table) for table in ignores]
- if not replace_xmlid:
- ignores.append("ir_model_data")
-
- cr.execute("CREATE UNLOGGED TABLE _upgrade_rrr(old int PRIMARY KEY, new int)")
- execute_values(cr, "INSERT INTO _upgrade_rrr (old, new) VALUES %s", id_mapping.items())
-
- if model_src == model_dst:
- fk_def = []
-
- model_src_table = table_of_model(cr, model_src)
- for table, fk, _, _ in get_fk(cr, model_src_table, quote_ident=False):
- if table in ignores:
- continue
- query = """
- UPDATE {table} t
- SET {fk} = r.new
- FROM _upgrade_rrr r
- WHERE r.old = t.{fk}
- """
- unique_indexes = _get_unique_indexes_with(cr, table, fk)
- if unique_indexes:
- conditions = [""]
- for _, uniq_cols in unique_indexes:
- uniq_cols = set(uniq_cols) - {fk} # noqa: PLW2901
- where_clause = (
- " AND ".join(format_query(cr, "u.{col} = t.{col}", col=col) for col in uniq_cols)
- if uniq_cols
- else "true"
- )
- conditions.append("NOT EXISTS(SELECT 1 FROM {table} u WHERE u.{fk} = r.new AND %s)" % where_clause)
-
- query += " AND ".join(conditions)
-
- if not column_exists(cr, table, "id"):
- # seems to be a m2m table. Avoid duplicated entries
- (col2,) = get_columns(cr, table, ignore=(fk,)).iter_unquoted()
-
- # handle possible duplicates after update of m2m table
- cr.execute(
- format_query(
- cr,
- """
- WITH rr2 AS (
- SELECT array_agg(t.{fk} ORDER BY t.{fk}) AS olds,
- r.new AS new,
- t.{col2} AS col2
- FROM {table} t
- JOIN _upgrade_rrr r
- ON r.old = t.{fk}
- GROUP BY r.new, t.{col2}
- )
- DELETE
- FROM {table} t
- USING rr2 r
- WHERE t.{col2} = r.col2
- AND t.{fk} = ANY(r.olds[2:])
- """,
- table=table,
- fk=fk,
- col2=col2,
- )
- )
- query += " AND NOT EXISTS(SELECT 1 FROM {table} e WHERE e.{col2} = t.{col2} AND e.{fk} = r.new)"
-
- col2_info = target_of(cr, table, col2) # col2 may not be a FK
- if col2_info and col2_info[:2] == (model_src_table, "id"):
- # a m2m on itself, remove the self referencing entries
- # It only handle 1-level recursions. For multi-level recursions, it should be handled manually.
- # We can't decide which link to break.
- # XXX: add a warning?
- query += """;
- DELETE
- FROM {table} t
- USING _upgrade_rrr r
- WHERE t.{fk} = r.new
- AND t.{fk} = t.{col2};
- """
-
- cr.execute(format_query(cr, query, table=table, fk=fk, col2=col2))
-
- else: # it's a model
- fmt_query = format_query(cr, query, table=table, fk=fk)
- parallel_execute(cr, explode_query_range(cr, fmt_query, table=table, alias="t"))
-
- # track default values to update
- model = model_of_table(cr, table)
- fk_def.append((model, fk))
-
- delete_query = """
- DELETE
- FROM {table} t
- USING _upgrade_rrr r
- WHERE t.{fk} = r.old
- """
- cr.execute(format_query(cr, delete_query, table=table, fk=fk))
-
- if fk_def:
- if table_exists(cr, "ir_values"):
- column_read, cast_write = _ir_values_value(cr, prefix="v")
- query = r"""
- UPDATE ir_values v
- SET value = {cast_write}
- FROM _upgrade_rrr r
- WHERE v.key = 'default'
- AND {column_read} = format(E'I%%s\n.', r.old)
- AND (v.model, v.name) IN %s
- """.format(column_read=column_read, cast_write=cast_write % r"format(E'I%%s\n.', r.new)")
- else:
- query = """
- UPDATE ir_default d
- SET json_value = r.new::varchar
- FROM _upgrade_rrr r, ir_model_fields f
- WHERE f.id = d.field_id
- AND d.json_value = r.old::varchar
- AND (f.model, f.name) IN %s
- """
-
- cr.execute(query, [tuple(fk_def)])
-
- cr.execute("SELECT id FROM ir_model WHERE model=%s", [model_dst])
- model_dest_id = cr.fetchone()[0]
-
- # indirect references
- for ir in indirect_references(cr, bound_only=True):
- if ir.table in ignores:
- continue
- if ir.company_dependent_comodel:
- if ir.company_dependent_comodel == model_src:
- if model_src != model_dst:
- cr.execute(
- "UPDATE ir_model_fields SET relation = %s WHERE model = %s AND name = %s",
- [
- model_dst,
- model_of_table(cr, ir.table),
- ir.res_id,
- ],
- )
- query = format_query(
- cr,
- """
- WITH _upg_cd AS (
- SELECT t.id,
- jsonb_object_agg(j.key, COALESCE(r.new, j.value::int)) as value
- FROM {table} t
- JOIN jsonb_each_text(t.{column}) j
- ON true
- LEFT JOIN _upgrade_rrr r
- ON r.old = j.value::integer
- WHERE {{parallel_filter}}
- GROUP BY t.id
- HAVING bool_or(r.new IS NOT NULL)
- )
- UPDATE {table} t
- SET {column} = u.value
- FROM _upg_cd u
- WHERE u.id = t.id
- """,
- table=ir.table,
- column=ir.res_id,
- )
- explode_execute(cr, query, table=ir.table, alias="t")
- # ensure all new ids exist
- cr.execute(
- format_query(
- cr,
- """
- Select t.id AS id,
- j.key AS c_id,
- j.value AS ref
- FROM {table} t
- JOIN JSONB_EACH_TEXT(t.{column}) j
- ON True
- WHERE j.value IS NOT NULL
- AND NOT EXISTS (
- SELECT 1 FROM {dest_table} WHERE id = j.value::int
- )
- """,
- table=ir.table,
- column=ir.res_id,
- dest_table=table_of_model(cr, model_dst),
- )
- )
- invalid_ref = cr.dictfetchall()
- if invalid_ref:
- raise RuntimeError(
- "Invalid company dependent values for {}.{} referencing model {}: {}".format(
- model_of_table(cr, ir.table), ir.res_id, model_dst, invalid_ref
- )
- )
- continue
- res_model_upd = []
- if ir.res_model:
- res_model_upd.append(format_query(cr, "{} = %(model_dst)s", ir.res_model))
- if ir.res_model_id:
- res_model_upd.append(format_query(cr, "{} = %(model_dest_id)s", ir.res_model_id))
- upd = SQLStr(", ".join(res_model_upd))
- res_model_whr = SQLStr(" AND ".join(res_model_upd))
- whr = ir.model_filter(placeholder="%(model_src)s", prefix="t.")
-
- if not id_update:
- jmap_expr = SQLStr("true") # no-op
- jmap_expr_upd = SQLStr("")
- else:
- jmap_expr = format_query(cr, "{} = _upgrade_rrr.new", ir.res_id)
- jmap_expr_upd = SQLStr(", " + jmap_expr)
-
- query = format_query(
- cr,
- """
- UPDATE {table} t
- SET {upd}
- {jmap_expr_upd}
- FROM _upgrade_rrr
- {{extra_join}}
- WHERE {whr}
- AND _upgrade_rrr.old = t.{res_id}
- """,
- table=ir.table,
- res_id=ir.res_id,
- whr=whr,
- jmap_expr_upd=jmap_expr_upd,
- upd=upd,
- )
-
- unique_indexes = []
- if ir.res_model:
- unique_indexes += _get_unique_indexes_with(cr, ir.table, ir.res_id, ir.res_model)
- if ir.res_model_id:
- unique_indexes += _get_unique_indexes_with(cr, ir.table, ir.res_id, ir.res_model_id)
- if unique_indexes:
- query = format_query(
- cr,
- query,
- extra_join=format_query(
- cr,
- """
- LEFT JOIN _upgrade_rrr AS _upgrade_rrr2
- ON _upgrade_rrr.new = _upgrade_rrr2.new
- AND _upgrade_rrr.old < _upgrade_rrr2.old
- LEFT JOIN {table} t2
- ON _upgrade_rrr2.old = t2.{res_id}
- """,
- table=ir.table,
- res_id=ir.res_id,
- ),
- )
- conditions = []
- for _, uniq_cols in unique_indexes:
- uniq_cols = set(uniq_cols) - {ir.res_id, ir.res_model, ir.res_model_id} # noqa: PLW2901
- conditions.append(
- format_query(
- cr,
- """
- -- there is no target already present within the same constraint
- NOT EXISTS(SELECT 1 FROM {table} WHERE {res_model_whr} AND {jmap_expr} AND {where_clause})
- -- there is no other entry with the same target within the same constraint
- AND NOT (t2 IS NOT NULL AND {where_clause2})
- """,
- table=ir.table,
- res_model_whr=res_model_whr,
- jmap_expr=jmap_expr,
- where_clause=SQLStr(" AND ".join(format_query(cr, "{0}=t.{0}", col) for col in uniq_cols))
- if uniq_cols
- else SQLStr("True"),
- where_clause2=SQLStr(" AND ".join(format_query(cr, "t2.{0}=t.{0}", col) for col in uniq_cols))
- if uniq_cols
- else SQLStr("True"),
- )
- )
- query = format_query(
- cr,
- """{prev_query} AND {cond};
-
- DELETE FROM {table} t USING _upgrade_rrr WHERE {whr} AND t.{res_id} = _upgrade_rrr.old;
- """,
- prev_query=query,
- cond=SQLStr("\nAND ".join(conditions)),
- table=ir.table,
- whr=whr,
- res_id=ir.res_id,
- )
- cr.execute(query, locals())
- else:
- fmt_query = cr.mogrify(format_query(cr, query, extra_join=SQLStr("")), locals()).decode()
- parallel_execute(cr, explode_query_range(cr, fmt_query, table=ir.table, alias="t"))
-
- # reference fields
- cr.execute("SELECT model, name FROM ir_model_fields WHERE ttype='reference'")
- for model, column in cr.fetchall():
- table = table_of_model(cr, model)
- if table not in ignores and column_updatable(cr, table, column):
- cr.execute(
- """
- WITH _ref AS (
- SELECT concat(%s, ',', old) as old, concat(%s, ',', new) as new
- FROM _upgrade_rrr
- )
- UPDATE "{table}" t
- SET "{column}" = r.new
- FROM _ref r
- WHERE t."{column}" = r.old
- """.format(table=table, column=column),
- [model_src, model_dst],
- )
-
- cr.execute("DROP TABLE _upgrade_rrr")
- if parent_field and model_dst == model_src:
- if column_exists(cr, model_src_table, "parent_path"):
- fk_target = target_of(cr, model_src_table, parent_field)
- if fk_target:
- if fk_target[0] != model_src_table:
- _logger.warning(
- "`%s` has a `parent_path` but `%s` is not a self-referencing FK", model_src_table, parent_field
- )
- else:
- update_parent_path(cr, model_src, parent_field)
- elif parent_field != "parent_id": # check non-default value
- _logger.error("`%s` in `%s` is not a self-referencing FK", parent_field, model_src_table)
- elif column_exists(cr, model_src_table, "parent_left"):
- _logger.warning("Possibly missing update of parent_left/right in `%s`", model_src_table)
-
-
-def replace_in_all_jsonb_values(cr, table, column, old, new, extra_filter=None):
- r"""
- Replace values in JSONB columns.
-
- This function replaces `old` by `new` in JSONB values. It is useful for replacing
- values in *all* translations of translated fields.
-
- :param str table: table name where the values are to be replaced
- :param str column: column name where the values are to be replaced
- :param str old: original value to replace, can be a simple term (str) or a Postgres
- regular expression wrapped by :class:`~odoo.upgrade.util.pg.PGRegexp`
- :param str new: new value to set, can be a simple term or a expression using
- `\` notation to refer to *captured groups* if `old` is a
- regexp expression
- :param str extra_filter: extra `WHERE` compatible clause to filter the values to
- update, must use the `t` alias for the table, it can also
- include `{parallel_filter}` to execute the query in parallel,
- see :func:`~odoo.upgrade.util.pg.explode_execute`
- """
- re_old = (
- old
- if isinstance(old, PGRegexp)
- else "{}{}{}".format(
- r"\y" if re.match(r"\w", old[0]) else "",
- re.escape(old),
- r"\y" if re.match(r"\w", old[-1]) else "",
- )
- )
- match = str(Json(re_old))[1:-1] # escapes re_old into json string
-
- if extra_filter is None:
- extra_filter = "true"
-
- query = cr.mogrify(
- """
- WITH upd AS (
- SELECT t.id,
- jsonb_object_agg(v.key, regexp_replace(v.value, %s, %s, 'g')) AS value
- FROM "{table}" t
- JOIN LATERAL jsonb_each_text(t."{column}") v
- ON true
- WHERE jsonb_path_match(t."{column}", 'exists($.* ? (@ like_regex {match}))')
- AND {extra_filter}
- GROUP BY t.id
- )
- UPDATE "{table}" t
- SET "{column}" = upd.value
- FROM upd
- WHERE upd.id = t.id
- """.format(**locals()),
- [re_old, new],
- ).decode()
-
- if "{parallel_filter}" in query:
- explode_execute(cr, query, table=table, alias="t")
- else:
- cr.execute(query)
-
-
-def ensure_mail_alias_mapping(cr, model, record_xmlid, alias_xmlid, alias_name):
- _validate_model(model)
-
- cr.execute("SELECT id FROM ir_model WHERE model = %s", [model])
- (model_id,) = cr.fetchone()
- alias_id = ensure_xmlid_match_record(
- cr,
- alias_xmlid,
- "mail.alias",
- {
- "alias_name": alias_name,
- "alias_parent_model_id": model_id,
- },
- )
-
- if alias_id:
- ensure_xmlid_match_record(
- cr,
- record_xmlid,
- model,
- {"alias_id": alias_id},
- )
-
-
-def remove_act_window_view_mode(cr, model, view_mode):
- default = "list,form" if version_gte("saas~17.5") else "tree,form"
- cr.execute(
- """
- WITH upd AS (
- UPDATE ir_act_window act
- SET view_mode = COALESCE(
- NULLIF(
- ARRAY_TO_STRING(ARRAY_REMOVE(STRING_TO_ARRAY(view_mode, ','), %s), ','),
- '' -- invalid value
- ),
- %s -- default value
- )
- WHERE act.res_model = %s
- AND %s = ANY(STRING_TO_ARRAY(act.view_mode, ','))
- RETURNING act.id
-
- )
- DELETE FROM ir_act_window_view av
- USING upd
- WHERE upd.id = av.act_window_id
- AND av.view_mode=%s
- """,
- [view_mode, default, model, view_mode, view_mode],
- )
-
-
-def _remove_redundant_tcalls(cr, match):
- """
- Remove t-calls of the removed view.
-
- This function removes the t-calls to `match`.
-
- :param str match: t-calls value to remove, typically it would be a view's xml_id or key
- """
- arch_col = (
- get_value_or_en_translation(cr, "ir_ui_view", "arch_db")
- if column_exists(cr, "ir_ui_view", "arch_db")
- else "arch"
- )
- cr.execute(
- format_query(
- cr,
- """
- SELECT iv.id,
- imd.module,
- imd.name
- FROM ir_ui_view iv
- LEFT JOIN ir_model_data imd
- ON iv.id = imd.res_id
- AND imd.model = 'ir.ui.view'
- WHERE {} ~ %s
- """,
- sql.SQL(arch_col),
- ),
- [r"""\yt-call=(["']){}\1""".format(re.escape(match))],
- )
- standard_modules = set(modules.get_modules()) - {"studio_customization"}
- for vid, module, name in cr.fetchall():
- with edit_view(cr, view_id=vid) as arch:
- for node in arch.findall(".//t[@t-call='{}']".format(match)):
- node.getparent().remove(node)
- if not module or module not in standard_modules:
- _logger.info(
- "The view %swith ID: %s has been updated, removed t-calls to deprecated %r",
- ("`{}.{}` ".format(module, name) if module else ""),
- vid,
- match,
- )
-
-
-def update_parent_path(cr, model, parent_field="parent_id"):
- """
- Trigger the update of parent paths in a model.
-
- :meta private: exclude from online docs
- """
- if not version_gte("saas~11.3"):
- _logger.error("parent_left and parent_right must be computed via the ORM")
- return
- table = table_of_model(cr, model)
- name_field = "name" if column_exists(cr, table, "name") else "id"
- break_recursive_loops(cr, model, parent_field, name_field)
- query = format_query(
- cr,
- """
- WITH RECURSIVE __parent_store_compute(id, parent_path) AS (
- SELECT row.id,
- concat(row.id, '/')
- FROM {table} row
- WHERE row.{parent_field} IS NULL
-
- UNION
-
- SELECT row.id,
- concat(comp.parent_path, row.id, '/')
- FROM {table} row
- JOIN __parent_store_compute comp
- ON row.{parent_field} = comp.id
- )
- UPDATE {table} row
- SET parent_path = comp.parent_path
- FROM __parent_store_compute comp
- WHERE row.id = comp.id
- """,
- table=table,
- parent_field=parent_field,
- )
- cr.execute(query)
diff --git a/src/util/release-note.xml b/src/util/release-note.xml
deleted file mode 100644
index 64725a55a..000000000
--- a/src/util/release-note.xml
+++ /dev/null
@@ -1,19 +0,0 @@
-
-
- Hello,
- Your database has successfully been upgraded to the latest version 🥳.
-
-
-
-
- Meet Odoo . now !
-
-
![]()
-
- Want to know more? Check out the full release note.
- Want to know more? Check out the full release note.
-
-
diff --git a/src/util/report-migration.xml b/src/util/report-migration.xml
deleted file mode 100644
index 5350eca2d..000000000
--- a/src/util/report-migration.xml
+++ /dev/null
@@ -1,58 +0,0 @@
-
-
- Congratulations, you have just upgraded to Odoo
- Here are changes that may impact day to day flows in this new version.
- Want to know more? Check out the full functional release note.
- Want to know more? Check out the full functional release note.
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- During the upgrade some fields have been removed. The records below have been automatically corrected.
-
-
-
-
diff --git a/src/util/report.py b/src/util/report.py
deleted file mode 100644
index 2dd965aba..000000000
--- a/src/util/report.py
+++ /dev/null
@@ -1,341 +0,0 @@
-# -*- coding: utf-8 -*-
-import logging
-import os
-import re
-import sys
-from textwrap import dedent
-
-import lxml
-from docutils.core import publish_string
-
-try:
- import markdown
-except ImportError:
- markdown = None
-
-from .helpers import _validate_model
-from .misc import parse_version
-
-# python3 shims
-try:
- basestring # noqa: B018
-except NameError:
- basestring = unicode = str
-
-try:
- from markupsafe import Markup, escape
-
- from odoo.tools.misc import html_escape
-
- if html_escape is not escape:
- Markup = None
-except ImportError:
- Markup = None
-
-try:
- try:
- from odoo.api import SUPERUSER_ID
- except ImportError:
- from odoo import SUPERUSER_ID
- from odoo import release
- from odoo.tools.mail import html_sanitize
-except ImportError:
- from openerp import SUPERUSER_ID, release
- from openerp.tools.mail import html_sanitize
-
-
-if sys.version_info > (3,):
- from odoo.tools import html_escape
-else:
- # In python2, `html_escape` always returns a byte-string with non-ascii characters replaced
- # by their html entities.
-
- import werkzeug.utils
-
- # Avoid DeprecationWarning while still remaining compatible with werkzeug pre-0.9
- if parse_version(getattr(werkzeug, "__version__", "0.0")) < parse_version("0.9.0"):
-
- def html_escape(text):
- return werkzeug.utils.escape(text, quote=True).encode("ascii", "xmlcharrefreplace")
-
- else:
-
- def html_escape(text):
- return werkzeug.utils.escape(text).encode("ascii", "xmlcharrefreplace")
-
-
-try:
- from odoo.addons.base.models.ir_module import MyWriter # > 11.0
-except ImportError:
- try:
- from odoo.addons.base.module.module import MyWriter
- except ImportError:
- from openerp.addons.base.module.module import MyWriter
-
-from .exceptions import MigrationError
-from .misc import has_enterprise, split_osenv, version_between, version_gte
-from .orm import env, get_admin_channel, guess_admin_id
-
-migration_reports = {}
-_logger = logging.getLogger(__name__)
-
-
-_ENV_AM = set(split_osenv("UPG_ANNOUNCE_MEDIA", default="discuss"))
-ANNOUNCE_MEDIA = _ENV_AM & {"", "discuss", "logger"}
-if _ENV_AM - ANNOUNCE_MEDIA:
- raise ValueError(
- "Invalid value for the environment variable `UPG_ANNOUNCE_MEDIA`: {!r}. "
- "Authorized values are a combination of 'discuss', 'logger', or an empty string.".format(
- os.getenv("UPG_ANNOUNCE_MEDIA")
- )
- )
-ANNOUNCE_MEDIA -= {""}
-
-
-ODOO_SHOWCASE_VIDEOS = {
- "19.0": "OZLP-SCHW7A",
- "saas~18.4": "fiPyJXzeNjQ",
- "saas~18.3": "oyev2DxC5yY",
- "saas~18.2": "bwn_HWuLuTA",
- "saas~18.1": "is9oLyIkQGk",
- "18.0": "gbE3azm_Io0",
- "saas~17.4": "8F4-uDwom8A",
- "saas~17.2": "ivjgo_2-wkE",
- "17.0": "qxb74CMR748",
- "16.0": "RVFZL3D9plg",
-}
-
-
-def add_to_migration_reports(message, category="Other", format="text"):
- assert format in {"text", "html", "md", "rst"}
- if format == "md":
- message = md2html(dedent(message))
- elif format == "rst":
- message = rst2html(message)
- raw = False
- if format != "text":
- if Markup:
- message = Markup(message)
- else:
- raw = True
- migration_reports.setdefault(category, []).append((message, raw))
- migration_reports_length = sum(len(msg) for reps in migration_reports.values() for msg, _ in reps) + sum(
- map(len, migration_reports)
- )
- if migration_reports_length > 1000000:
- _logger.warning("Upgrade report is growing suspiciously long: %s characters so far.", migration_reports_length)
-
-
-def announce_release_note(cr):
- filepath = os.path.join(os.path.dirname(__file__), "release-note.xml")
- with open(filepath, "rb") as fp:
- contents = fp.read()
- if not version_gte("15.0"):
- contents = contents.replace(b"t-out", b"t-esc")
- report = lxml.etree.fromstring(contents)
- e = env(cr)
- major_version, minor_version = re.findall(r"\d+", release.major_version)
- values = {
- "version": release.major_version,
- "major_version": major_version,
- "minor_version": minor_version,
- "odoo_showcase_video_id": ODOO_SHOWCASE_VIDEOS.get(release.major_version, ""),
- }
- _logger.info("Rendering release note for version %s", release.version)
- render = e["ir.qweb"].render if hasattr(e["ir.qweb"], "render") else e["ir.qweb"]._render
- message = render(report, values=values)
- _announce_to_db(cr, message, to_admin_only=False)
-
-
-def announce_migration_report(cr):
- filepath = os.path.join(os.path.dirname(__file__), "report-migration.xml")
- with open(filepath, "rb") as fp:
- contents = fp.read()
- if Markup:
- contents = contents.replace(b"t-raw", b"t-out")
- if not version_gte("15.0"):
- contents = contents.replace(b"t-out", b"t-esc")
- report = lxml.etree.fromstring(contents)
- e = env(cr)
- major_version, minor_version = re.findall(r"\d+", release.major_version)
- values = {
- "action_view_id": e.ref("base.action_ui_view").id,
- "version": release.major_version,
- "major_version": major_version,
- "minor_version": minor_version,
- "messages": migration_reports,
- "get_anchor_link_to_record": get_anchor_link_to_record,
- }
- _logger.info(migration_reports)
- render = e["ir.qweb"].render if hasattr(e["ir.qweb"], "render") else e["ir.qweb"]._render
- message = render(report, values=values)
- _announce_to_db(cr, message)
- # To avoid posting multiple time the same messages in case this method is called multiple times.
- migration_reports.clear()
-
-
-def _announce_to_db(cr, message, to_admin_only=True):
- """Send a rendered message to the database via mail channel."""
- if not isinstance(message, basestring):
- message = message.decode("utf-8")
- if message.strip():
- message = message.replace("{", "{{").replace("}", "}}")
- kw = {}
- # If possible, post the migration report message to administrators only.
- recipient = get_admin_channel(cr) if to_admin_only else None
- if recipient:
- kw["recipient"] = recipient
- announce(cr, release.major_version, message, format="html", header=None, footer=None, **kw)
-
-
-def rst2html(rst):
- overrides = {
- "embed_stylesheet": False,
- "doctitle_xform": False,
- "output_encoding": "unicode",
- "xml_declaration": False,
- }
- html = publish_string(source=dedent(rst), settings_overrides=overrides, writer=MyWriter())
- return html_sanitize(html, silent=False)
-
-
-def md2html(md):
- assert markdown
-
- mdversion = markdown.__version_info__ if hasattr(markdown, "__version_info__") else markdown.version_info
- extensions = [
- "markdown.extensions.nl2br",
- "markdown.extensions.sane_lists",
- ]
- if mdversion[0] < 3:
- extensions.append("markdown.extensions.smart_strong")
-
- return markdown.markdown(md, extensions=extensions)
-
-
-_DEFAULT_HEADER = """
-Odoo has been upgraded to version {version}.
-What's new in this upgrade?
-"""
-
-_DEFAULT_FOOTER = "Enjoy the new Odoo Online!
"
-
-_DEFAULT_RECIPIENT = "mail.%s_all_employees" % ["group", "channel"][version_gte("9.0")]
-
-
-def announce(
- cr,
- version,
- msg,
- format="rst",
- recipient=_DEFAULT_RECIPIENT,
- header=_DEFAULT_HEADER,
- footer=_DEFAULT_FOOTER,
- pluses_for_enterprise=None,
-):
- if not ANNOUNCE_MEDIA:
- return
- if pluses_for_enterprise is None:
- # default value depend on format and version
- major = version[0]
- pluses_for_enterprise = (major == "s" or int(major) >= 9) and format == "md"
-
- if pluses_for_enterprise:
- plus_re = r"^(\s*)\+ (.+)\n"
- replacement = r"\1- \2\n" if has_enterprise() else ""
- msg = re.sub(plus_re, replacement, msg, flags=re.M)
-
- if format == "rst":
- msg = rst2html(msg)
- elif format == "md":
- msg = md2html(msg)
-
- message = ((header or "") + msg + (footer or "")).format(version=version)
- if "logger" in ANNOUNCE_MEDIA:
- _logger.info(message)
-
- if "discuss" not in ANNOUNCE_MEDIA:
- return
-
- # do not notify early, in case the migration fails halfway through
- ctx = {"mail_notify_force_send": False, "mail_notify_author": True}
-
- uid = guess_admin_id(cr)
- try:
- registry = env(cr)
- user = registry["res.users"].browse([uid])[0].with_context(ctx)
-
- def ref(xid):
- return registry.ref(xid).with_context(ctx)
-
- except MigrationError:
- try:
- from openerp.modules.registry import RegistryManager # noqa: PLC0415
- except ImportError:
- from openerp.modules.registry import Registry as RegistryManager # noqa: PLC0415
- registry = RegistryManager.get(cr.dbname)
- user = registry["res.users"].browse(cr, SUPERUSER_ID, uid, context=ctx)
-
- def ref(xid):
- rmod, _, rxid = recipient.partition(".")
- return registry["ir.model.data"].get_object(cr, SUPERUSER_ID, rmod, rxid, context=ctx)
-
- # default recipient
- poster = user.message_post if hasattr(user, "message_post") else user.partner_id.message_post
-
- if recipient:
- try:
- if isinstance(recipient, str): # noqa: SIM108
- recipient = ref(recipient)
- else:
- recipient = recipient.with_context(**ctx)
- poster = recipient.message_post
- except (ValueError, AttributeError):
- # Cannot find record, post the message on the wall of the admin
- pass
-
- type_field = ["type", "message_type"][version_gte("9.0")]
- # From 12.0, system notificatications are sent by email,
- # and do not increment the upper right notification counter.
- # While comments, in a mail.channel, do.
- # We want the notification counter to appear for announcements, so we force the comment type from 12.0.
- type_value = ["notification", "comment"][version_gte("12.0")]
- subtype_key = ["subtype", "subtype_xmlid"][version_gte("saas~13.1")]
-
- kw = {type_field: type_value, subtype_key: "mail.mt_comment"}
-
- try:
- poster(body=message, partner_ids=[user.partner_id.id], **kw)
- except Exception:
- _logger.warning("Cannot announce message", exc_info=True)
- else:
- # Chat window with the report will be open post-upgrade for the admin user
- if version_between("9.0", "saas~18.1") and user.partner_id and recipient:
- channel_member_model = (
- "discuss.channel.member"
- if version_gte("saas~16.3")
- else "mail.channel.member"
- if version_gte("16.0")
- else "mail.channel.partner"
- )
- domain = [("partner_id", "=", user.partner_id.id), ("channel_id", "=", recipient.id)]
- try:
- registry[channel_member_model].search(domain)[:1].with_context(ctx).fold_state = "open"
- except Exception:
- _logger.warning("Cannot unfold chat window", exc_info=True)
-
-
-def get_anchor_link_to_record(model, id, name=None, action_id=None):
- _validate_model(model)
- if not name:
- name = "{}(id={})".format(model, id)
- if version_gte("saas~17.2"):
- part1 = "action-{}".format(action_id) if action_id else model
- url = "/odoo/{}/{}?debug=1".format(part1, id)
- else:
- url = "/web?debug=1#view_type=form&model={}&action={}&id={}".format(model, action_id or "", id)
-
- anchor_tag = '{}'.format(url, html_escape(name))
- if Markup:
- anchor_tag = Markup(anchor_tag)
- return anchor_tag
diff --git a/src/util/snippets.py b/src/util/snippets.py
deleted file mode 100644
index 280f46dc2..000000000
--- a/src/util/snippets.py
+++ /dev/null
@@ -1,350 +0,0 @@
-# -*- coding: utf-8 -*-
-import logging
-import multiprocessing
-import re
-import sys
-from concurrent.futures import ProcessPoolExecutor
-
-from lxml import etree, html
-from psycopg2 import sql
-from psycopg2.extensions import quote_ident
-from psycopg2.extras import Json
-
-from .const import NEARLYWARN
-from .helpers import table_of_model
-from .misc import log_progress, make_pickleable_callback
-from .pg import column_exists, column_type, get_max_workers, table_exists
-
-_logger = logging.getLogger(__name__)
-utf8_parser = html.HTMLParser(encoding="utf-8")
-
-
-class Snippet:
- def __init__(self, name, tag="*", klass="", selector=""):
- self.name = name
- self.tag = tag
- self.klass = klass or name
- self.selector = selector or f'//{tag}[hasclass("{self.klass}")]'
-
-
-def add_snippet_names(cr, table, column, snippets, select_query):
- """
- Execute the select_query then for each snippet contained in arch add the right data-snippet attribute on the right element.
-
- :param str table: The table we are working on
- :param str column: The column we are working on
- :param list snippets: list of all snippets to migrate
- :param str select_query: a query that when executed will return (id, list of snippets contained in the arch, arch)
- """
- _logger.info("Add snippet names on %s.%s", table, column)
- cr.execute(select_query)
-
- it = log_progress(cr.fetchall(), _logger, qualifier="rows", size=cr.rowcount, log_hundred_percent=True)
-
- def quote(ident):
- return quote_ident(ident, cr._cnx)
-
- for res_id, regex_matches, arch in it:
- regex_matches = [match[0] for match in regex_matches] # noqa: PLW2901
- arch = arch.replace("\r", "") # otherwise html parser below will transform \r ->
# noqa: PLW2901
- body = html.fromstring(arch, parser=utf8_parser)
- changed = False
- for snippet in snippets:
- if snippet.klass in regex_matches:
- body_snippets = body.xpath(snippet.selector)
- for body_snippet in body_snippets:
- body_snippet.attrib["data-snippet"] = snippet.name
- changed = True
- if changed:
- body = etree.tostring(body, encoding="unicode")
- cr.execute(f"UPDATE {quote(table)} SET {quote(column)} = %s WHERE id = %s", [body, res_id])
-
-
-def add_snippet_names_on_html_field(cr, table, column, snippets, regex):
- """Search for all the snippets in the fields mentioned (should be html fields) and add the corresponding data-snippet on them."""
- query = cr.mogrify(
- sql.SQL(
- """
- SELECT id, array((SELECT regexp_matches({column}, %(regex)s, 'g'))), {column}
- FROM {table}
- WHERE {column} ~ %(regex)s
- """
- ).format(column=sql.Identifier(column), table=sql.Identifier(table)),
- {"regex": regex},
- ).decode()
- where = cr.mogrify(sql.SQL("{column} ~ %s").format(column=sql.Identifier(column)), [regex]).decode()
- ids_ranges = determine_chunk_limit_ids(cr, table, [column], where)
- for id0, id1 in ids_ranges:
- add_snippet_names(cr, table, column, snippets, query + f" AND id BETWEEN {id0} AND {id1}")
-
-
-def get_regex_from_snippets_list(snippets):
- return "(%s)" % "|".join(snippet.klass for snippet in snippets)
-
-
-def get_html_fields(cr):
- # yield (table, column) of stored html fields (that needs snippets updates)
- for table, columns in html_fields(cr):
- for column in columns:
- yield table, quote_ident(column, cr._cnx)
-
-
-def html_fields(cr):
- cr.execute(
- """
- SELECT f.model, array_agg(f.name)
- FROM ir_model_fields f
- JOIN ir_model m ON m.id = f.model_id
- WHERE f.ttype = 'html'
- AND f.store = true
- AND m.transient = false
- AND f.model NOT LIKE 'ir.actions%'
- AND f.model != 'mail.message'
- GROUP BY f.model
- """
- )
- for model, columns in cr.fetchall():
- table = table_of_model(cr, model)
- if not table_exists(cr, table):
- # an SQL VIEW
- continue
- existing_columns = [column for column in columns if column_exists(cr, table, column)]
- if existing_columns:
- yield table, existing_columns
-
-
-def parse_style(attr):
- """
- Convert an HTML style attribute's text into a dict mapping property names to property values.
-
- :param str attr: value of an HTML style attribute
- :return: dict of CSS property values per property name
- """
- # Captures two groups:
- # - identifier: sequence of word character or hyphen that is followed by a colon
- # - value: sequence of:
- # - any non semicolon character or
- # - sequence of any non single quote character or escaped single quote
- # surrounded by single quotes or
- # - sequence of any non double quote character or escaped double quote
- # surrounded by double quotes
- regex = r"""
- ([\w\-]+)\s*:\s*((?:[^;\"']|'(?:[^']|(?:\\'))*'|\"(?:[^\"]|(?:\\\"))*\")+)
- """.strip()
- return dict(re.findall(regex, attr))
-
-
-def format_style(styles):
- """
- Convert a dict of CSS property names to property values into an HTML style attribute string.
-
- :param dict styles: CSS property value per property name
- :return: str HTML style attribute
- """
- style = "; ".join(["%s: %s" % entry for entry in styles.items()])
- if len(style) > 0 and style[-1] != ";":
- style += ";"
- return style
-
-
-def html_converter(transform_callback, selector=None):
- """
- Create an upgrade converter for a single HTML text content or for HTML elements that match a selector.
-
- :param func transform_callback: transforms an HTML tree and returns True if
- a change happened
- :param str selector: targets the elements to loop on
- :return: object HTMLConverter with callback
- """
- return HTMLConverter(make_pickleable_callback(transform_callback), selector)
-
-
-class BaseConverter:
- def __init__(self, callback, selector=None):
- self.callback = callback
- self.selector = selector
-
- def for_html(self):
- return HTMLConverter(self.callback, self.selector)
-
- def for_qweb(self):
- return QWebConverter(self.callback, self.selector)
-
- def has_changed(self, els):
- if self.selector:
- converted = [self.callback(el) for el in els.xpath(self.selector)]
- return any(converted)
- return self.callback(els)
-
- def __call__(self, content):
- # Remove `` header
- if not content:
- return (False, content)
- content = re.sub(r"^<\?xml .+\?>\s*", "", content.strip())
- # Wrap in node before parsing to preserve external comments and multi-root nodes,
- # except for when this looks like a full html doc, because in this case the wrap tag breaks the logic in
- # https://github.com/lxml/lxml/blob/2ac88908ffd6df380615c0af35f2134325e4bf30/src/lxml/html/html5parser.py#L184
- els = self._loads(content if content.strip()[:5].lower() == "{content}")
- has_changed = self.has_changed(els)
- new_content = re.sub(r"(^|$|^$)", "", self._dumps(els).strip()) if has_changed else content
- return (has_changed, new_content)
-
- def _loads(self, string):
- raise NotImplementedError
-
- def _dumps(self, node):
- raise NotImplementedError
-
-
-class HTMLConverter(BaseConverter):
- def for_html(self):
- return self
-
- def _loads(self, string):
- return html.fromstring(string, parser=utf8_parser)
-
- def _dumps(self, node):
- return html.tostring(node, encoding="unicode")
-
-
-class QWebConverter(BaseConverter):
- def for_qweb(self):
- return self
-
- def _loads(self, string):
- return html.fromstring(string, parser=html.XHTMLParser(encoding="utf-8"))
-
- def _dumps(self, node):
- return etree.tostring(node, encoding="unicode")
-
-
-class Convertor:
- def __init__(self, converters, callback):
- self.converters = converters
- self.callback = callback
-
- def __call__(self, row):
- converters = self.converters
- columns = self.converters.keys()
- converter_callback = self.callback
- res_id, *contents = row
- changes = {}
- for column, content in zip(columns, contents):
- if content and converters[column]:
- # jsonb column; convert all keys
- new_content = {}
- has_changed, new_content["en_US"] = converter_callback(content.pop("en_US"))
- if has_changed:
- for lang, value in content.items():
- _, new_content[lang] = converter_callback(value)
- new_content = Json(new_content)
- else:
- has_changed, new_content = converter_callback(content)
- changes[column] = new_content
- if has_changed:
- changes["id"] = res_id
- return changes
-
-
-def convert_html_columns(cr, table, columns, converter_callback, where_column="IS NOT NULL", extra_where="true"):
- r"""
- Convert HTML content for the given table column.
-
- :param cursor cr: database cursor
- :param str table: table name
- :param str column: column name
- :param func converter_callback: conversion function that converts the HTML
- text content and returns a tuple with a boolean that indicates whether a
- change happened and the new content must be saved
- :param str where_column: filtering such as
- - "like '%abc%xyz%'"
- - "~* '\yabc.*xyz\y'"
- :param str extra_where: extra filtering on the where clause
- """
- assert "id" not in columns
-
- converters = {column: "->>'en_US'" if column_type(cr, table, column) == "jsonb" else "" for column in columns}
- select = ", ".join(f'"{column}"' for column in columns)
- where = " OR ".join(f'"{column}"{converters[column]} {where_column}' for column in columns)
-
- base_select_query = f"""
- SELECT id, {select}
- FROM {table}
- WHERE ({where})
- AND ({extra_where})
- """
- split_queries = [
- (base_select_query + "\n AND id BETWEEN {} AND {}".format(*x))
- for x in determine_chunk_limit_ids(cr, table, columns, "({}) AND ({})".format(where, extra_where))
- ]
-
- update_sql = ", ".join(f'"{column}" = %({column})s' for column in columns)
- update_query = f"UPDATE {table} SET {update_sql} WHERE id = %(id)s"
-
- extrakwargs = {"mp_context": multiprocessing.get_context("fork")} if sys.version_info >= (3, 7) else {}
- with ProcessPoolExecutor(max_workers=get_max_workers(), **extrakwargs) as executor:
- convert = Convertor(converters, converter_callback)
- for query in log_progress(split_queries, logger=_logger, qualifier=f"{table} updates"):
- cr.execute(query)
- for data in executor.map(convert, cr.fetchall(), chunksize=1000):
- if "id" in data:
- cr.execute(update_query, data)
-
-
-def determine_chunk_limit_ids(cr, table, column_arr, where):
- bytes_per_chunk = 100 * 1024 * 1024
- columns = ", ".join(quote_ident(column, cr._cnx) for column in column_arr if column != "id")
- cr.execute(
- f"""
- WITH info AS (
- SELECT id,
- sum(pg_column_size(({columns}, id))) OVER (ORDER BY id) / {bytes_per_chunk} AS chunk
- FROM {table}
- WHERE {where}
- ) SELECT min(id), max(id) FROM info GROUP BY chunk
- """
- )
- return cr.fetchall()
-
-
-def convert_html_content(
- cr,
- converter_callback,
- where_column="IS NOT NULL",
- **kwargs,
-):
- r"""
- Convert HTML content.
-
- :param cursor cr: database cursor
- :param func converter_callback: conversion function that converts the HTML
- text content and returns a tuple with a boolean that indicates whether a
- change happened and the new content must be saved
- :param str where_column: filtering such as
- - "like '%abc%xyz%'"
- - "~* '\yabc.*xyz\y'"
- :param dict kwargs: extra keyword arguments to pass to :func:`convert_html_column`
- """
- if hasattr(converter_callback, "for_html"): # noqa: SIM108
- html_converter = converter_callback.for_html()
- else:
- # trust the given converter to handle HTML
- html_converter = converter_callback
-
- for table, columns in html_fields(cr):
- convert_html_columns(cr, table, columns, html_converter, where_column=where_column, **kwargs)
-
- if hasattr(converter_callback, "for_qweb"):
- qweb_converter = converter_callback.for_qweb()
- else:
- _logger.log(NEARLYWARN, "Cannot adapt converter callback %r for qweb; using it directly", converter_callback)
- qweb_converter = converter_callback
-
- convert_html_columns(
- cr,
- "ir_ui_view",
- ["arch_db"],
- qweb_converter,
- where_column=where_column,
- **dict(kwargs, extra_where="type = 'qweb'"),
- )
diff --git a/src/util/specific.py b/src/util/specific.py
deleted file mode 100644
index 4345e5145..000000000
--- a/src/util/specific.py
+++ /dev/null
@@ -1,244 +0,0 @@
-# -*- coding: utf-8 -*-
-import logging
-
-from .exceptions import UpgradeError
-from .helpers import _validate_table
-from .misc import _cached, version_gte
-from .models import rename_model
-from .modules import rename_module
-from .orm import env
-from .pg import column_exists, format_query, parallel_execute, rename_table, table_exists
-from .report import add_to_migration_reports
-
-try:
- from odoo.tools.translate import _get_translation_upgrade_queries
-except ImportError:
- if version_gte("16.0"):
- raise
-
-_logger = logging.getLogger(__name__)
-
-
-def dbuuid(cr):
- return _dbuuids(cr)[-1]
-
-
-@_cached
-def _dbuuids(cr):
- cr.execute(
- """
- SELECT REPLACE(value, 'upg-neuter-', '')
- FROM ir_config_parameter
- WHERE key IN ('database.uuid', 'origin.database.uuid')
- ORDER BY key
- """
- )
- return [uuid for (uuid,) in cr.fetchall()]
-
-
-def dispatch_by_dbuuid(cr, version, callbacks):
- """
- Allow to execute a migration script for a specific database only, based on its dbuuid.
-
- .. example::
- .. code-block:: python
- def db_yellowbird(cr, version):
- cr.execute("DELETE FROM ir_ui_view WHERE id=837")
-
- util.dispatch_by_dbuuid(cr, version, {
- "ef81c07aa90936a89f4e7878e2ebc634a24fcd66": db_yellowbird,
- })
-
- :param str version: Odoo version
- :param dict[str, function] callbacks: mapping dbuuids to the functions to run against matching dbs
-
- .. warning::
- - Only the first match of (icp["database.uuid"], icp["origin.database.uuid"]) in `callbacks` is executed.
-
- .. tip::
- - If looking to prevent a callback from running against a descendant db, one can use a noop `callback`:
- .. example::
- .. code-block:: python
- noop = lambda *args: None
- util.dispatch_by_dbuuid(cr, version, {
- "dbuuid": noop,
- "ancestor's dbuuid": db_yellowbird,
- })
- """
- for uuid in _dbuuids(cr):
- if uuid in callbacks:
- func = callbacks[uuid]
- _logger.info("calling dbuuid-specific function `%s`", func.__name__)
- func(cr, version)
- break
-
-
-def rename_custom_model(cr, model_name, new_model_name, custom_module=None, report_details=""):
- cr.execute("SELECT 1 FROM ir_model WHERE model = %s", [model_name])
- if not cr.rowcount:
- _logger.warning("Model %r not found: skip renaming", model_name)
- return
-
- rename_model(cr, model_name, new_model_name, rename_table=True)
- module_details = " from module '{}'".format(custom_module) if custom_module else ""
- add_to_migration_reports(
- category="Custom models",
- message="The custom model '{model_name}'{module_details} was renamed to '{new_model_name}'. {report_details}".format(
- **locals()
- ),
- )
-
-
-def rename_custom_module(cr, old_module_name, new_module_name, report_details="", author="%"):
- cr.execute("SELECT 1 FROM ir_module_module WHERE name = %s AND author ILIKE %s", [old_module_name, author])
- if not cr.rowcount:
- return
-
- rename_module(cr, old_module_name, new_module_name)
- _logger.warning("Custom module %r renamed to %r", old_module_name, new_module_name)
- add_to_migration_reports(
- category="Custom modules",
- message="The custom module '{old_module_name}' was renamed to '{new_module_name}'. {report_details}".format(
- **locals()
- ),
- )
-
-
-def rename_custom_table(
- cr,
- table_name,
- new_table_name,
- custom_module=None,
- report_details="",
-):
- if not table_exists(cr, table_name):
- _logger.warning("Table %r not found: skip renaming", table_name)
- return
-
- rename_table(cr, table_name, new_table_name, remove_constraints=False)
-
- module_details = " from module '{}'".format(custom_module) if custom_module else ""
- add_to_migration_reports(
- category="Custom tables/columns",
- message="The custom table '{table_name}'{module_details} was renamed to '{new_table_name}'. {report_details}".format(
- **locals()
- ),
- )
-
-
-def rename_custom_column(cr, table_name, col_name, new_col_name, custom_module=None, report_details=""):
- _validate_table(table_name)
- if not column_exists(cr, table_name, col_name):
- _logger.warning("Column %r not found on table %r: skip renaming", col_name, table_name)
- return
- cr.execute('ALTER TABLE "{}" RENAME COLUMN "{}" TO "{}"'.format(table_name, col_name, new_col_name))
- module_details = " from module '{}'".format(custom_module) if custom_module else ""
- add_to_migration_reports(
- category="Custom tables/columns",
- message="The custom column '{col_name}' of the table '{table_name}'{module_details} was renamed to '{new_col_name}'."
- " {report_details}".format(**locals()),
- )
-
-
-def reset_cowed_views(cr, xmlid, key=None):
- if "." not in xmlid:
- raise ValueError("Please use fully qualified name .")
-
- module, _, name = xmlid.partition(".")
- if not key:
- key = xmlid
- cr.execute(
- """
- UPDATE ir_ui_view u
- SET arch_prev = u.arch_db,
- arch_db = v.arch_db
- FROM ir_ui_view v
- JOIN ir_model_data m
- ON m.res_id = v.id AND m.model = 'ir.ui.view'
- WHERE u.key = %s
- AND m.module = %s
- AND m.name = %s
- AND u.website_id IS NOT NULL
- RETURNING u.id
- """,
- [key, module, name],
- )
- return set(sum(cr.fetchall(), ()))
-
-
-def translation2jsonb_all_missing(cr):
- """Convert all missing translated fields to jsonb.
-
- This helper function is designed to run only in an on-premise Odoo instance
- where any unofficial modules you use are installed, after going through the
- upgrade to version 16.0.
-
- It will find all unofficial fields and convert translations from the
- ``_ir_translation`` table into the new JSONB format.
-
- Modules must be available in the environment to be loaded.
-
- :return: Converted fields.
- """
- if not version_gte("16.0"):
- raise UpgradeError("JSONB translations are only available from Odoo 16")
- if not table_exists(cr, "_ir_translation"):
- _logger.getChild("translation2jsonb_all_missing").info("Missing `_ir_translation` table; skip.")
- return None
- _env = env(cr)
- fields = []
- cr.execute(
- """
- SELECT STRING_TO_ARRAY(name, ','),
- ARRAY_AGG(DISTINCT lang)
- FROM _ir_translation
- WHERE type IN ('model', 'model_terms')
- AND name LIKE '%,%'
- AND lang != 'en_US'
- GROUP BY name
- """
- )
- for (mname, fname), langs in cr.fetchall():
- try:
- model = _env[mname]
- field = model._fields[fname]
- except KeyError:
- continue
- if (
- not model._auto
- or model._abstract
- or model._transient
- or not field.store
- or not field.translate
- or field.manual
- ):
- continue
- # Check if the field is already converted
- cr.execute(
- format_query(cr, "SELECT 1 FROM {} WHERE {} ?| %s LIMIT 1", model._table, fname),
- [langs],
- )
- if not cr.rowcount:
- fields.append(field)
- return translation2jsonb(cr, *fields)
-
-
-def translation2jsonb(cr, *fields):
- """Convert selected translated fields to jsonb.
-
- Migrates translations for chosen fields, from the ``_ir_translation`` table
- into the new JSONB format.
-
- :param odoo.fields.Field fields: Fields to convert.
- :return: Converted fields.
- """
- if not version_gte("16.0"):
- raise UpgradeError("JSONB translations are only available from Odoo 16")
- all_cleanup_queries = []
- _logger.info("Migrating translations for fields %r", fields)
- for field in fields:
- migrate_queries, cleanup_queries = _get_translation_upgrade_queries(cr, field)
- all_cleanup_queries.extend(cleanup_queries)
- parallel_execute(cr, migrate_queries)
- parallel_execute(cr, all_cleanup_queries)
- return fields
diff --git a/src/util/spreadsheet/__init__.py b/src/util/spreadsheet/__init__.py
deleted file mode 100644
index 05e68b1a9..000000000
--- a/src/util/spreadsheet/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-from .misc import *
-from .tokenizer import *
diff --git a/src/util/spreadsheet/misc.py b/src/util/spreadsheet/misc.py
deleted file mode 100644
index 0228f1ae9..000000000
--- a/src/util/spreadsheet/misc.py
+++ /dev/null
@@ -1,74 +0,0 @@
-from .. import json, pg
-
-MEMORY_CAP = 2 * 10**8 # 200MB
-
-
-def iter_commands(cr, like_all=(), like_any=()):
- if not (bool(like_all) ^ bool(like_any)):
- raise ValueError("Please specify `like_all` or `like_any`, not both")
-
- with pg.named_cursor(cr, itersize=1) as ncr:
- ncr.execute(
- pg.format_query(
- cr,
- """
- WITH filtered AS (
- SELECT id,
- commands,
- LENGTH(commands) AS commands_length
- FROM spreadsheet_revision
- WHERE commands LIKE {condition} (%s::text[])
- ), smaller AS (
- SELECT id,
- commands,
- sum(commands_length) OVER (ORDER BY id) / %s AS num
- FROM filtered
- WHERE commands_length <= %s
- )
- SELECT array_agg(id ORDER BY id),
- array_agg(commands ORDER BY id),
- min(id) AS sort_key
- FROM smaller
- GROUP BY num
-
- UNION ALL
-
- SELECT ARRAY[id],
- ARRAY[commands],
- id AS sort_key
- FROM filtered
- WHERE commands_length > %s
- ORDER BY sort_key
- """,
- condition=pg.SQLStr("ALL" if like_all else "ANY"),
- ),
- [list(like_any or like_all), MEMORY_CAP, MEMORY_CAP, MEMORY_CAP],
- )
- for ids, commands, _ in ncr:
- for revision_id, data in zip(ids, commands):
- data_loaded = json.loads(data)
- if "commands" not in data_loaded:
- continue
- data_old = json.dumps(data_loaded, sort_keys=True)
-
- changed = yield data_loaded["commands"]
- if changed is None:
- changed = data_old != json.dumps(data_loaded, sort_keys=True)
-
- if changed:
- cr.execute(
- "UPDATE spreadsheet_revision SET commands=%s WHERE id=%s",
- [json.dumps(data_loaded), revision_id],
- )
-
-
-def process_commands(cr, callback, *args, **kwargs):
- gen = iter_commands(cr, *args, **kwargs)
- try:
- cmd = next(gen)
- while True:
- changed = callback(cmd)
- cmd = gen.send(changed)
-
- except StopIteration:
- pass
diff --git a/src/util/spreadsheet/tokenizer.py b/src/util/spreadsheet/tokenizer.py
deleted file mode 100644
index d3a02ed1f..000000000
--- a/src/util/spreadsheet/tokenizer.py
+++ /dev/null
@@ -1,248 +0,0 @@
-import re
-
-"""
-This entire file is a direct translation of the original JavaScript code found in https://github.com/odoo/o-spreadsheet/blob/master/src/formulas/tokenizer.ts.
-"""
-
-
-class CellErrorType:
- NotAvailable = "#N/A"
- InvalidReference = "#REF"
- BadExpression = "#BAD_EXPR"
- CircularDependency = "#CYCLE"
- UnknownFunction = "#NAME?"
- DivisionByZero = "#DIV/0!"
- GenericError = "#ERROR"
-
-
-DEFAULT_LOCALES = [
- {
- "name": "English (US)",
- "code": "en_US",
- "thousandsSeparator": ",",
- "decimalSeparator": ".",
- "dateFormat": "m/d/yyyy",
- "timeFormat": "hh:mm:ss a",
- "formulaArgSeparator": ",",
- }
-]
-DEFAULT_LOCALE = DEFAULT_LOCALES[0]
-
-NEWLINE = "\n"
-
-
-def get_formula_number_regex(decimal_separator):
- decimal_separator = re.escape(decimal_separator)
- return re.compile(r"^-?\d+(%s?\d*(e\d+)?)?|^-?%s\d+(?!\w|!)" % (decimal_separator, decimal_separator))
-
-
-def escape_regexp(string):
- return re.escape(string)
-
-
-full_row_xc = r"(\$?[A-Z]{1,3})?\$?[0-9]{1,7}\s*:\s*(\$?[A-Z]{1,3})?\$?[0-9]{1,7}\s*"
-full_col_xc = r"\$?[A-Z]{1,3}(\$?[0-9]{1,7})?\s*:\s*\$?[A-Z]{1,3}(\$?[0-9]{1,7})?\s*"
-
-cell_reference = re.compile(r"\$?([A-Z]{1,3})\$?([0-9]{1,7})", re.IGNORECASE)
-range_reference = re.compile(
- r"^\s*('.+'!|[^']+!)?(%s|%s|%s)$" % (cell_reference.pattern, full_row_xc, full_col_xc), re.IGNORECASE
-)
-
-white_space_special_characters = [
- "\t",
- "\f",
- "\v",
- chr(int("00a0", 16)),
- chr(int("1680", 16)),
- chr(int("2000", 16)),
- chr(int("200a", 16)),
- chr(int("2028", 16)),
- chr(int("2029", 16)),
- chr(int("202f", 16)),
- chr(int("205f", 16)),
- chr(int("3000", 16)),
- chr(int("feff", 16)),
-]
-white_space_regexp = re.compile("|".join(map(re.escape, white_space_special_characters)) + r"|(\r\n|\r|\n)")
-
-
-def replace_special_spaces(text):
- if not text:
- return ""
- if not white_space_regexp.search(text):
- return text
- return white_space_regexp.sub(lambda match: NEWLINE if match.group(1) else " ", text)
-
-
-POSTFIX_UNARY_OPERATORS = ["%"]
-OPERATORS = ["+", "-", "*", "/", ":", "=", "<>", ">=", ">", "<=", "<", "^", "&"] + POSTFIX_UNARY_OPERATORS
-
-
-def tokenize(string, locale=DEFAULT_LOCALE):
- string = replace_special_spaces(string)
- result = []
- if string:
- chars = TokenizingChars(string)
-
- while not chars.is_over():
- token = (
- tokenize_space(chars)
- or tokenize_args_separator(chars, locale)
- or tokenize_parenthesis(chars)
- or tokenize_operator(chars)
- or tokenize_string(chars)
- or tokenize_debugger(chars)
- or tokenize_invalid_range(chars)
- or tokenize_number(chars, locale)
- or tokenize_symbol(chars)
- )
-
- if not token:
- token = ("UNKNOWN", chars.shift())
-
- result.append(token)
-
- return result
-
-
-def tokenize_debugger(chars):
- if chars.current == "?":
- chars.shift()
- return "DEBUGGER", "?"
- return None
-
-
-parenthesis = {"(": ("LEFT_PAREN", "("), ")": ("RIGHT_PAREN", ")")}
-
-
-def tokenize_parenthesis(chars):
- value = chars.current
- if value in parenthesis:
- chars.shift()
- return parenthesis[value]
- return None
-
-
-def tokenize_args_separator(chars, locale):
- if chars.current == locale["formulaArgSeparator"]:
- value = chars.shift()
- return "ARG_SEPARATOR", value
- return None
-
-
-def tokenize_operator(chars):
- for op in OPERATORS:
- if chars.current_starts_with(op):
- chars.advance_by(len(op))
- return "OPERATOR", op
- return None
-
-
-FIRST_POSSIBLE_NUMBER_CHARS = set("0123456789")
-
-
-def tokenize_number(chars, locale):
- if chars.current not in FIRST_POSSIBLE_NUMBER_CHARS and chars.current != locale["decimalSeparator"]:
- return None
- match = re.match(get_formula_number_regex(locale["decimalSeparator"]), chars.remaining())
- if match:
- chars.advance_by(len(match.group(0)))
- return "NUMBER", match.group(0)
- return None
-
-
-def tokenize_string(chars):
- if chars.current == '"':
- start_char = chars.shift()
- letters = start_char
- while chars.current and (chars.current != start_char or letters[-1] == "\\"):
- letters += chars.shift()
- if chars.current == '"':
- letters += chars.shift()
- return "STRING", letters
- return None
-
-
-separator_regexp = re.compile(r"^[\w\.!\$]+")
-
-
-def tokenize_symbol(chars):
- result = ""
- if chars.current == "'":
- last_char = chars.shift()
- result += last_char
- while chars.current:
- last_char = chars.shift()
- result += last_char
- if last_char == "'":
- if chars.current and chars.current == "'":
- last_char = chars.shift()
- result += last_char
- else:
- break
- if last_char != "'":
- return "UNKNOWN", result
- match = separator_regexp.match(chars.remaining())
- if match:
- value = match.group(0)
- result += value
- chars.advance_by(len(value))
- if result:
- value = result
- is_reference = range_reference.match(value)
- if is_reference:
- return "REFERENCE", value
- return "SYMBOL", value
- return None
-
-
-def tokenize_space(chars):
- length = 0
- while chars.current == NEWLINE:
- length += 1
- chars.shift()
- if length:
- return "SPACE", NEWLINE * length
-
- while chars.current == " ":
- length += 1
- chars.shift()
-
- if length:
- return "SPACE", " " * length
- return None
-
-
-def tokenize_invalid_range(chars):
- if chars.current.startswith(CellErrorType.InvalidReference):
- chars.advance_by(len(CellErrorType.InvalidReference))
- return "INVALID_REFERENCE", CellErrorType.InvalidReference
- return None
-
-
-class TokenizingChars:
- def __init__(self, text):
- self.text = text
- self.current_index = 0
- self.current = text[0]
-
- def shift(self):
- current = self.current
- self.current_index += 1
- self.current = self.text[self.current_index] if self.current_index < len(self.text) else None
- return current
-
- def advance_by(self, length):
- self.current_index += length
- self.current = self.text[self.current_index] if self.current_index < len(self.text) else None
-
- def is_over(self):
- return self.current_index >= len(self.text)
-
- def remaining(self):
- return self.text[self.current_index :]
-
- def current_starts_with(self, string):
- if self.current != string[0]:
- return False
- return all(self.text[self.current_index + j] == string[j] for j in range(1, len(string)))
diff --git a/tools/compile23.py b/tools/compile23.py
deleted file mode 100755
index c136cb295..000000000
--- a/tools/compile23.py
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/env python3
-# ruff: noqa: T201
-import subprocess
-import sys
-from pathlib import PurePath
-from shutil import which
-
-py2_only_patterns = []
-py2_files = []
-
-py3_only_patterns = [
- "tools/*.py",
- # tests are only run from version 12. python2 compatibility is not needed.
- "src/testing.py",
- "src/util/jinja_to_qweb.py",
- "src/util/snippets.py",
- "src/util/convert_bootstrap.py",
- "src/*/tests/*.py",
- "src/*/17.0.*/*.py",
-]
-py3_files = []
-
-rc = 0
-
-for filename in sys.argv[1:]:
- p = PurePath(filename)
- if p.suffix != ".py":
- continue
-
- if not filename.islower():
- print(f"filename {filename!r} is not lowercase")
- rc = 1
-
- if any(p.match(pattern) for pattern in py2_only_patterns):
- py2_files.append(filename)
- elif any(p.match(pattern) for pattern in py3_only_patterns):
- py3_files.append(filename)
- else:
- # not an explicit match to a python version. Test against both versions.
- py2_files.append(filename)
- py3_files.append(filename)
-
-
-if py2_files:
- if which("python2"):
- s = subprocess.run(["python2", "-m", "compileall", "-f", "-q", *py2_files], check=False)
- if s.returncode:
- rc = 1
- else:
- lines = [
- "WARNING: `python2` hasn't been found in $PATH",
- "You must ensure the following files are compatible with python2:",
- *[f" - {f}" for f in py2_files],
- ]
- width = max(map(len, lines))
- message = "\n".join(f"@ {line: <{width}s} @" for line in lines)
- extra = "@" * (width + 4)
- print(f"{extra}\n{message}\n{extra}", file=sys.stderr)
-
-if py3_files:
- s = subprocess.run(["python3", "-m", "compileall", "-f", "-q", *py3_files], check=False)
- if s.returncode:
- rc = 1
-
-sys.exit(rc)
diff --git a/tools/fetch-release-notes-video-id.py b/tools/fetch-release-notes-video-id.py
deleted file mode 100755
index fb127f916..000000000
--- a/tools/fetch-release-notes-video-id.py
+++ /dev/null
@@ -1,96 +0,0 @@
-#!/usr/bin/env -S uv run --script --quiet
-
-# /// script
-# requires-python = ">=3.12"
-# dependencies = [
-# "httpx",
-# "libcst",
-# "lxml",
-# ]
-# ///
-
-import re
-import sys
-from pathlib import Path
-
-import httpx
-import libcst as cst
-from lxml import etree
-
-if len(sys.argv) != 2:
- sys.exit(f"Usage: {sys.argv[0]} VERSION")
-
-version = sys.argv[1]
-
-VERSION_RE = re.compile(r"^(?:saas[~-])?([0-9]+)(?:\.([0-9]+))?$")
-
-if (match := VERSION_RE.match(version)) is None:
- sys.exit(f"Invalid version: {version!r}")
-
-major, minor = match.groups(default="0")
-
-version_url = major if minor == "0" else f"{major}-{minor}"
-full_version = f"{major}.0" if minor == "0" else f"saas~{major}.{minor}"
-
-html = httpx.get(f"https://www.odoo.com/odoo-%7Bversion_url%7D-release-notes")
-if html.status_code != 200:
- sys.exit(f"Cannot fetch release notes page for version {version}")
-
-root = etree.fromstring(html.text, parser=etree.HTMLParser())
-iframe = root.xpath("//main//iframe[contains(@src, 'youtube.com') or contains(@src, 'youtube-nocookie.com')]")
-if not iframe:
- sys.exit(f"Cannot find youtube video in {html.url}")
-
-yt_link = httpx.URL(iframe[0].attrib["src"])
-video_id = yt_link.path.removeprefix("/embed/")
-
-
-report_py = Path(__file__).parent.parent / "src" / "util" / "report.py"
-
-source_tree = cst.parse_module(report_py.read_bytes())
-
-
-class Transformer(cst.CSTTransformer):
- def __init__(self):
- self.video_dict = None
- self.key_found = False
- super().__init__()
-
- def visit_Assign(self, node):
- match node:
- case cst.Assign(
- targets=[cst.AssignTarget(target=cst.Name(value="ODOO_SHOWCASE_VIDEOS"))],
- value=video_dict,
- ):
- self.video_dict = video_dict
- return True
- return False
-
- def visit_Dict(self, node):
- return node is self.video_dict
-
- def leave_DictElement(self, original_node, updated_node):
- if original_node.key.raw_value == full_version:
- self.key_found = True
- if original_node.value.raw_value != video_id:
- updated_node = updated_node.with_changes(value=cst.SimpleString(f'"{video_id}"'))
- return updated_node
-
- def leave_Dict(self, original_node, updated_node):
- if original_node is self.video_dict:
- if self.key_found:
- elements = updated_node.elements
- else:
- new_elem = updated_node.elements[0].with_changes(
- key=cst.SimpleString(f'"{full_version}"'), value=cst.SimpleString(f'"{video_id}"')
- )
- elements = [new_elem, *updated_node.elements]
-
- elements = sorted(elements, reverse=True, key=lambda e: VERSION_RE.match(e.key.raw_value).groups("0"))
- updated_node = updated_node.with_changes(elements=elements)
- return updated_node
-
-
-modified_tree = source_tree.visit(Transformer())
-
-report_py.write_text(modified_tree.code)
diff --git a/tools/generate-inherit.py b/tools/generate-inherit.py
deleted file mode 100755
index cd6f16f73..000000000
--- a/tools/generate-inherit.py
+++ /dev/null
@@ -1,498 +0,0 @@
-#!/usr/bin/env python3
-# ruff: noqa: ERA001
-
-import io
-import itertools
-import logging
-import subprocess
-import sys
-import tokenize
-from argparse import ArgumentParser, Namespace
-from ast import literal_eval
-from collections import defaultdict
-from dataclasses import dataclass, field
-from functools import total_ordering
-from pathlib import Path
-from typing import Dict, List, NamedTuple, Optional, Set, Tuple
-
-import black
-import tomli
-
-try:
- from black.nodes import Visitor
-except ImportError:
- # old black version
- from black import Visitor
-
-
-logging.basicConfig(
- level=logging.INFO, stream=sys.stderr, format="%(asctime)s %(levelname)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
-)
-if sys.stderr.isatty():
- logging.addLevelName(logging.INFO, "\033[1;32m\033[1;49mINFO\033[0m")
- logging.addLevelName(logging.CRITICAL, "\033[1;37m\033[1;41mCRITICAL\033[0m")
-
-logger = logging.getLogger(__name__)
-
-if int(black.__version__.split(".")[0]) >= 22:
- logger.critical("Too recent version of `black`. Please install version 21.12b0 in order to parse python2 code.")
- sys.exit(1)
-
-MODELS = ["osv", "osv_memory", "Model", "TransientModel", "AbstractModel"]
-MODELS += [".".join(x).lstrip(".") for x in itertools.product(["openerp", "odoo", ""], ["osv", "models"], MODELS)]
-
-
-class Repo(NamedTuple):
- name: str
-
- @property
- def remote(self):
- return f"git@github.com:odoo/{self.name}.git"
-
-
-REPOSITORIES = [
- Repo("odoo"),
- Repo("enterprise"),
- Repo("design-themes"),
-]
-
-
-@total_ordering
-@dataclass(eq=False)
-class Version:
- name: str
-
- @property
- def fqn(self):
- if "." in self.name:
- return self.name.replace("-", "~")
- major, minor = self.ints
- return f"{major}.saas~{minor}"
-
- def __repr__(self):
- # This is hacky.
- # It will only be used when outputting the generated file.
- return f"parse_version({self.fqn!r})"
-
- @property
- def ints(self):
- s = list(map(int, self.name.replace("saas-", "").split(".")))
- if len(s) == 1:
- # < 11.0
- major = {range(1, 6): 7, range(6, 7): 8, range(7, 14): 9, range(14, 19): 10}
- for m, n in major.items():
- if s[0] in m:
- return (n, s[0])
- raise ValueError(self.name)
- return tuple(s)
-
- def __eq__(self, other):
- return self.name == other.name
-
- def __lt__(self, other):
- return self.ints < other.ints
-
- def __hash__(self):
- return hash(self.name) # Only name is relevant
-
- @classmethod
- def parse(cls, name):
- name = name.replace("~", "-")
- if not (name.startswith("saas-") or name.endswith(".0")):
- name = name.split(".", 1)[-1]
- return cls(name)
-
- @property
- def python_target(self):
- bounds = [
- [(7, 0), black.mode.TargetVersion.PY27],
- [(10, 17), black.mode.TargetVersion.PY36],
- [(15, 0), black.mode.TargetVersion.PY37],
- [(16, 3), black.mode.TargetVersion.PY310],
- ]
- for bound, target in reversed(bounds):
- if bound <= self.ints:
- return target
- raise RuntimeError(f"Cannot determine python target for {self.name}")
-
-
-@dataclass(order=True)
-class Inherit:
- model: str
- born: Version # inclusive
- dead: Optional[Version] = None # non-inclusive
- via: Optional[str] = None # Many2one field to parent in case of `_inherits`
-
- def apply_on(self, version: Version) -> bool:
- if self.dead is None:
- return self.born <= version
- return self.born <= version < self.dead
-
-
-_LAST_MAJOR = 17
-_VERSIONS = {Version(f"{major}.0") for major in range(7, _LAST_MAJOR + 1)}
-_VERSIONS |= {Version(f"saas-{saas}") for saas in range(1, 19)}
-_VERSIONS |= {Version(f"saas-{major}.{minor}") for major in range(11, _LAST_MAJOR) for minor in range(1, 6)}
-
-VERSIONS = sorted(_VERSIONS)
-
-IGNORED_FILES = [
- # defines `_name = LITERAL % CONSTANT`
- # does not have _inherit(s)
- "odoo/addons/google_calendar/google_calendar.py",
- "odoo/addons/google_calendar/models/google_calendar.py",
- "odoo/addons/website_version/models/google_management.py",
- "enterprise/website_version/models/google_management.py",
-]
-
-# Sometimes, new modules are added during a version lifetime and not forward-ported to dead saas~* version.
-# Theses versions being dead and no upgrade to these versions being made, we can consider it contains some models
-# Without it, we would end with holes in inherit tree.
-VIRTUAL_INHERITS = {
- "account.avatax": [
- Inherit(model="account.move", born=Version("14.0"), dead=Version("15.0"), via=None),
- Inherit(model="sale.order", born=Version("14.0"), dead=Version("15.0"), via=None),
- Inherit(model="sale.subscription", born=Version("14.0"), dead=Version("15.0"), via=None),
- ],
- "account.avatax.unique.code": [
- Inherit(model="account.avatax", born=Version("14.0"), dead=Version("15.0"), via=None),
- Inherit(model="res.partner", born=Version("14.0"), dead=Version("15.0"), via=None),
- ],
- "account.edi.common": [
- Inherit("account.edi.xml.cii", born=Version("14.0"), dead=Version("15.0")),
- Inherit("account.edi.xml.ubl_20", born=Version("14.0"), dead=Version("15.0")),
- ],
- "account.edi.xml.ubl_20": [
- Inherit("account.edi.xml.ubl_21", born=Version("14.0"), dead=Version("15.0")),
- Inherit("account.edi.xml.ubl_efff", born=Version("14.0"), dead=Version("15.0")),
- ],
- "account.edi.xml.ubl_21": [
- Inherit("account.edi.xml.ubl_bis3", born=Version("14.0"), dead=Version("15.0")),
- ],
- "account.edi.xml.ubl_bis3": [
- Inherit("account.edi.xml.ubl_de", born=Version("14.0"), dead=Version("15.0")),
- ],
- "account.report": [
- Inherit("account.cash.flow.report", born=Version("saas-11.1"), dead=Version("saas-12.5")),
- Inherit("l10n.lu.report.partner.vat.intra", born=Version("saas-13.1"), dead=Version("saas-13.2")),
- ],
- "l10n_cl.edi.util": [
- Inherit("stock.picking", born=Version("14.0"), dead=Version("saas-14.2")),
- Inherit(model="l10n_cl.daily.sales.book", born=Version("14.0"), dead=Version("saas-14.3"), via=None),
- ],
- "l10n_es.sii.account.tax.mixin": [
- Inherit(model="account.tax", born=Version("14.0"), dead=Version("saas-14.4"), via=None),
- Inherit(model="account.tax.template", born=Version("14.0"), dead=Version("saas-14.4"), via=None),
- ],
- "l10n_mx.trial.report": [
- Inherit("l10n_mx.trial.closing.report", born=Version("saas-11.1"), dead=Version("saas-12.2")),
- ],
- "l10n_mx_edi.pac.sw.mixin": [
- Inherit("account.invoice", born=Version("saas-11.1"), dead=Version("saas-12.5")),
- Inherit("account.payment", born=Version("saas-11.1"), dead=Version("saas-12.2")),
- ],
- "mail.activity.mixin": [
- Inherit("l10n_lu.yearly.tax.report.manual", born=Version("13.0"), dead=Version("15.0")),
- Inherit("l10n_uk.vat.obligation", born=Version("saas-15"), dead=Version("12.0")),
- ],
- "mail.thread": [
- Inherit("account.online.link", born=Version("12.0"), dead=Version("14.0")),
- Inherit(model="l10n_cl.daily.sales.book", born=Version("14.0"), dead=Version("saas-14.3"), via=None),
- Inherit("l10n_lu.yearly.tax.report.manual", born=Version("13.0"), dead=Version("15.0")),
- Inherit("l10n_uk.vat.obligation", born=Version("saas-15"), dead=Version("12.0")),
- ],
- "microsoft.outlook.mixin": [
- Inherit(model="fetchmail.server", born=Version("12.0"), dead=Version("saas-15.3"), via=None),
- Inherit(model="ir.mail_server", born=Version("12.0"), dead=Version("saas-15.3"), via=None),
- ],
- "pos.order.line": [
- Inherit(model="pos.order_line_pro_forma_be", born=Version("14.0"), dead=Version("saas-16.1")),
- ],
- "studio.mixin": [
- Inherit(model="ir.default", born=Version("14.0"), dead=Version("saas-14.2")),
- ],
- "google.gmail.mixin": [
- Inherit(model="fetchmail.server", born=Version("12.0"), dead=Version("15.0"), via=None),
- Inherit(model="ir.mail_server", born=Version("12.0"), dead=Version("15.0"), via=None),
- ],
-}
-
-
-# from lib2to3.refactor.RefactoringTool class
-def _read_python_source(filename):
- """Do our best to decode a Python source file correctly."""
- try:
- f = open(filename, "rb") # noqa: SIM115
- except OSError:
- return None, None
- try:
- encoding = tokenize.detect_encoding(f.readline)[0]
- finally:
- f.close()
- with io.open(filename, "r", encoding=encoding, newline="") as f:
- return f.read(), encoding
-
-
-class BootstrapVisitor(Visitor):
- result: Dict[str, List[Version]] = {}
-
- def to_str(self, node):
- if isinstance(node, black.Node):
- return "".join(self.to_str(c) for c in node.children)
- return node.value
-
- def visit_dictsetmaker(self, node):
- eval_context = {
- "Inherit": Inherit,
- "parse_version": Version.parse,
- }
-
- self.result = eval(f"{{ {self.to_str(node)} }}", eval_context)
- return []
-
-
-@dataclass
-class OdooVisitor(Visitor):
- inh: Dict[str, Set[Tuple[str, str]]] = field(default_factory=lambda: defaultdict(set))
-
- def to_str(self, node):
- if isinstance(node, black.Node):
- return "".join(self.to_str(c) for c in node.children)
- return node.value
-
- def visit_classdef(self, node):
- classparent = None
-
- children = iter(node.children)
- child = next(children)
- while child.type != black.token.COLON:
- if child.type == black.token.LPAR:
- classparent = self.to_str(next(children))
- child = next(children)
-
- if classparent in MODELS:
- suite = next(children)
- name = None
- inh = []
- for child in suite.children:
- if child.type == black.syms.simple_stmt:
- expr_stmt = child.children[0]
- if expr_stmt.type != black.syms.expr_stmt:
- continue
- attr = self.to_str(expr_stmt.children[0])
- if attr == "_name":
- node = expr_stmt.children[2]
- if node.type == black.token.NAME and node.value == "_description":
- # `_description` being required, some devs uses the following syntax: https://git.io/JUfhO
- node = expr_stmt.children[4]
- name = literal_eval(self.to_str(node))
- elif attr == "_inherit":
- node = expr_stmt.children[2]
- if node.type == black.token.NAME and node.value == "_name":
- inh.append((name, None))
- else:
- val = literal_eval(self.to_str(node))
- if isinstance(val, str):
- val = [val]
- inh.extend((v, None) for v in val)
- elif attr == "_inherits":
- val = literal_eval(self.to_str(expr_stmt.children[2]))
- inh.extend(val.items())
- else: # noqa: PLR5501
- # handle Many2one with delegate=True attribute
- if (
- len(expr_stmt.children) == 3
- and expr_stmt.children[1].type == black.token.EQUAL
- and expr_stmt.children[2].type == black.syms.power
- ):
- pw = expr_stmt.children[2]
- if (self.to_str(pw.children[0]) + self.to_str(pw.children[1])) == "fields.Many2one":
- via = self.to_str(expr_stmt.children[0])
- arglist = pw.children[2].children[1]
- comodel = None
- delegate = False
- for arg in arglist.children:
- if arg.type == black.token.STRING and comodel is None:
- comodel = literal_eval(self.to_str(arg))
- elif arg.type == black.syms.argument:
- if (
- self.to_str(arg.children[0]) == "delegate"
- and self.to_str(arg.children[2]) == "True"
- ):
- delegate = True
- if (
- self.to_str(arg.children[0]) == "comodel_name"
- and arg.children[2].type == black.token.STRING
- ):
- comodel = literal_eval(self.to_str(arg.children[2]))
- if delegate and comodel:
- inh.append((comodel, via))
-
- if name:
- for i, via in inh:
- if i != name:
- self.inh[i].add((name, via))
-
- return []
-
-
-def init_repos(path: Path) -> None:
- path.mkdir(parents=True, exist_ok=True)
-
- for repo in REPOSITORIES:
- p = path / repo.name
- if not p.exists():
- subprocess.run(
- ["git", "clone", repo.remote, repo.name],
- cwd=str(path),
- check=True,
- )
- else:
- subprocess.run(["git", "fetch", "-q"], cwd=str(p), check=True)
-
-
-def checkout(wd: Path, repo: Repo, version: Version) -> bool:
- gitdir = str(wd / repo.name)
-
- hasref = subprocess.run(
- ["git", "show-ref", "-q", "--verify", f"refs/remotes/origin/{version.name}"], cwd=gitdir, check=False
- )
- if hasref.returncode != 0:
- return False # unknown branch
- subprocess.run(
- ["git", "checkout", "-q", "--force", "-B", version.name, f"origin/{version.name}"], cwd=gitdir, check=True
- )
- return True
-
-
-def bootstrap(from_file: Path):
- logger.info("📂 Bootstrapping from %s", from_file)
- visitor = BootstrapVisitor()
-
- code, _ = _read_python_source(from_file)
- node = black.lib2to3_parse(code)
-
- list(visitor.visit(node))
- return defaultdict(list, visitor.result)
-
-
-def main(options: Namespace):
- wd = options.working_dir
- logger.info("⚙️ Initialize repositories into %s", wd)
- init_repos(wd)
-
- result = bootstrap(options.bootstrap_file) if options.bootstrap_file else defaultdict(list)
-
- for version in VERSIONS:
- if not (options.from_branch <= version <= options.to_branch):
- logger.info("⏭ Skip version %s", version.name)
- continue
-
- visitor = OdooVisitor()
-
- for model, virtuals in VIRTUAL_INHERITS.items():
- for virtual in virtuals:
- if virtual.apply_on(version):
- visitor.inh[model].add((virtual.model, virtual.via))
-
- any_repo = False
- for repo in REPOSITORIES:
- if not checkout(wd, repo, version):
- continue
- any_repo = True
- logger.info("🔎 Process %s at version %s", repo.name, version.name)
- r = wd / repo.name
- for pyfile in r.glob("**/*.py"):
- fname = str(pyfile.relative_to(wd))
- if fname in IGNORED_FILES or "test" in fname:
- continue
- code, _ = _read_python_source(pyfile)
- node = black.lib2to3_parse(code, [version.python_target])
- try:
- list(visitor.visit(node))
- except Exception:
- logger.critical("💥 Cannot parse %s (%s %s)", pyfile, repo.name, version.name)
- raise
-
- if not any_repo:
- # branch not found in any repo, don't store any inherits, even virtual ones
- continue
-
- if not visitor.inh:
- continue
-
- for model, children in result.items():
- for child in children:
- if (child.model, child.via) not in visitor.inh[model] and not child.dead and child.born < version:
- child.dead = version
-
- for model, children in visitor.inh.items():
- for child, via in children:
- for inh in result[model]:
- if inh.model == child and inh.via == via and (not inh.dead or inh.dead >= version):
- break
- else:
- result[model].append(Inherit(model=child, born=version, via=via))
-
- result = {m: sorted(result[m]) for m in sorted(result)}
- me = Path(sys.argv[0])
- pyproject = Path(black.find_pyproject_toml((str(me.parent),)))
-
- output = f"""\
-# This file is auto-generated by `{me.resolve().relative_to(pyproject.parent)}`. Edits will be lost.
-
-from collections import namedtuple
-
-try:
- from odoo.tools.misc import frozendict
- from odoo.tools.parse_version import parse_version
-except ImportError:
- from openerp.tools.parse_version import parse_version
- try:
- from openerp.tools.misc import frozendict
- except ImportError:
- # frozendict only appears with new api in 8.0
- frozendict = dict
-
-Inherit = namedtuple("Inherit", "model born dead via") # NOTE: dead is non-inclusive
-
-inheritance_data = frozendict({result!r})
-"""
-
- with open(pyproject, "rb") as fp:
- line_length = tomli.load(fp)["tool"]["ruff"]["line-length"]
- mode = black.FileMode(target_versions={black.TargetVersion.PY27}, line_length=line_length)
- print(black.format_str(output, mode=mode), end="") # noqa: T201
-
-
-# def debug(options: Namespace):
-# from black.debug import DebugVisitor
-# assert options.bootstrap_file
-# code, _ = _read_python_source(options.bootstrap_file)
-# node = black.lib2to3_parse(code)
-
-# # DebugVisitor.show(node)
-# result = bootstrap(options.bootstrap_file)
-# print(f"inheritance_data = frozendict({result!r})")
-# # v = OdooVisitor()
-# # list(v.visit(node))
-# # print(v.inh)
-
-
-if __name__ == "__main__":
- parser = ArgumentParser(description="Regenerate `_inherit.py` from source files")
-
- parser.add_argument("--working-dir", "-w", dest="working_dir", type=Path, default="/tmp/inh")
- parser.add_argument("--bootstrap-file", "-b", dest="bootstrap_file", type=Path)
-
- parser.add_argument(
- "--from-branch", "-f", dest="from_branch", type=Version, choices=VERSIONS, default=VERSIONS[0], metavar="BRANCH"
- )
- parser.add_argument(
- "--to-branch", "-t", dest="to_branch", type=Version, choices=VERSIONS, default=VERSIONS[-1], metavar="BRANCH"
- )
-
- options = parser.parse_args()
-
- # debug(options)
- main(options)
diff --git a/tools/graph-upgrade-timing.py b/tools/graph-upgrade-timing.py
deleted file mode 100755
index 003989f44..000000000
--- a/tools/graph-upgrade-timing.py
+++ /dev/null
@@ -1,98 +0,0 @@
-#!/usr/bin/env -S uv run --script --quiet
-# ruff: noqa: RET503
-
-# /// script
-# requires-python = ">=3.11"
-# dependencies = [
-# "pygal[png]",
-# ]
-# ///
-
-import argparse
-import os
-import re
-import sys
-from datetime import datetime
-
-import pygal # also need cairosvg for png output
-
-DT_PAT = r"((\d{4}-\d{2}-\d{2} \d{2}:\d{2}):\d{2},\d{3}) "
-
-
-def process(options):
- pie = pygal.Pie()
- dt = None
- others = 0.0
- end_st = None
- end_mod = None
- for line in sys.stdin.readlines():
- if dt is None:
- match = re.match(DT_PAT, line)
- if match:
- dt = match.group(2)
-
- match = re.search(r"Module ([a-zA-Z0-9_]+) loaded in (\d+\.\d\d)s, \d+ queries", line)
- if match:
- time = float(match.group(2))
- if time > options.min_time:
- pie.add(match.group(1), time)
- else:
- others += time
- continue
-
- match = re.search(r"module ([a-zA-Z0-9_]+): Running migration \[\$", line)
- if match:
- mod = match.group(1)
- match = re.match(DT_PAT, line)
- if end_mod is None:
- end_mod = mod
- end_st = match.group(1)
- elif mod != end_mod:
- time = (datetime.fromisoformat(match.group(1)) - datetime.fromisoformat(end_st)).total_seconds()
- if time > options.min_time:
- pie.add(f"{end_mod} [end]", time)
- else:
- others += time
- end_mod = mod
- end_st = match.group(1)
-
- elif end_st:
- if " odoo.modules.loading:" in line or " odoo.addons.base.models.ir_model: Deleting " in line:
- # last `end-` script finished.
- match = re.match(DT_PAT, line)
- time = (datetime.fromisoformat(match.group(1)) - datetime.fromisoformat(end_st)).total_seconds()
- if time > options.min_time:
- pie.add(f"{end_mod} [end]", time)
- else:
- others += time
- end_st = None
-
- if options.min_time and others:
- pie.add("Other modules", others)
-
- title = f"{dt}"
- if options.min_time:
- title = f"{title} • Modules loaded in more than {options.min_time} seconds"
- pie.title = title
-
- if options.format == "png":
- return pie.render_to_png()
- elif options.format == "svg":
- return pie.render()
-
-
-def main():
- # cat migration-14.0-latest.log | python3 graph-upgrade-timing.py -m 15 > graph.svg
- parser = argparse.ArgumentParser()
- parser.add_argument("-f", "--format", type=str, choices=["svg", "png"], default="svg")
- parser.add_argument("-m", "--min-time", dest="min_time", type=float, default=0.0)
-
- options = parser.parse_args()
- with os.fdopen(sys.stdout.fileno(), "wb") as fp:
- fp.write(process(options))
-
- return 0
-
-
-if __name__ == "__main__":
- sys.exit(main())