path: root/scripts/api-review/cmakescan.py

#!/usr/bin/env python3
# Usage: see api-review-gen
# Copyright (C) 2023 The Qt Company Ltd.
# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
"""Script to scan a CMakeLists.txt file for a list of public API-defining headers.

A somewhat brutal recursive-descent parser for as much (and a random
bit more) of the file format as matters to the task in hand. Assumes
the source is well-enough formed. See its main()'s documentation for
details.
"""
import re
from pathlib import Path

class ParseError(Exception): pass
class StreamPosition(object):
    def __init__(self):
        self.line = self.column = 0
    def __repr__(self):
        return f'line: {self.line}, column: {self.column}'
    def consume(self, text):
        endl = text.rfind('\n')
        if endl < 0:
            self.column += len(text)
        else:
            self.column = len(text) - endl - 1
            self.line += text.count('\n')

        return text

class CMakeScanner(object):
    def __init__(self, module, root):
        """Set up a scanner for the named module.

        First argument, the module name, is used to select which
        function-calls are of interest. It is saved as self.module for
        future reference. Second argument, root, is the path to the
        directory in which the CMakeLists.txt file being parsed was
        found: this should be given relative to the git module's
        root. Paths of headers in the source-file lists are given
        relative to the CMakeLists.txt file, so this root path is
        prepended to each to make them relative to the git module's
        root."""
        self.module, self.root = module, root
    # Parsing is structured in various layers, as class methods.
    #
    # There's a stack of __*ing() methods reaching down from the top layer,
    # that forward the raw stream and its position tracker to __tokenize,
    # calling their respective layers' sub-layer processors, __*ed(), on the
    # stream of tokens they get back from the layer below, to produce the
    # stream they deliver to the layer above. The __*ed() call spots the starts
    # of the structures it's there to recognize and delegates to the plain
    # verb-named __* function, passing it a list of the opening token(s) of the
    # structure, to which it'll append the rest of the structure before
    # packaging the sequence of tokens as an object of a structure-appropriate
    # sub-type of TokenSeq, which it returns. Its collecting of tokens to
    # package in this way may well recurse into itself, if it spots the opening
    # of an inner instance of its structure type (or another recognized at the
    # same layer of the parser).  The structures we need to recognize are
    # simple enough that this suffices.

    class Token(object):
        def __init__(self, text):
            self.text = text
        def __bool__(self):
            return self.text
        def __eq__(self, text):
            return self.text == text

        def __str__(self):
            return f'{self.__class__.__name__}("{self.text}")'
        __repr__ = __str__

    # Sub-classes are used to package distinct types of Token.
    class Gap(Token): pass
    class Word(Token): pass
    class KeyWord(Word): pass
    class String(Token):
        @property
        def contents(self): # Only used for private filter regexes.
            # Should this unescape ?
            return self.text[1:-1]

    # Throw-away tool function, to tunnel into __tokenize() and delete later:
    # a pseudo-constructor for Word and KeyWord that choses which to use.
    def word(text, W=Word, K=KeyWord):
        if text in ('if', 'elseif', 'else', 'endif',
                    'foreach', 'endforeach',
                    'function', 'endfunction',
                    'macro', 'endmacro',
                    'set'):
            return K(text)
        return W(text)
    @staticmethod
    def __tokenize(stream, position, G=Gap, S=String, W=word):
        """Digests the stream into basic lexical tokens.

        First parameter, stream, must be an iterable over lines of text. These
        are tacitly ''.join()ed but only as fast as what's been scanned has
        been consumed enough to make it necessary to read the next. Second
        parameter, position, should be a StreamPosition object, whose consume()
        will be passed all characters as they are consumed by the tokenizer, so
        that it can keep track of the line and column number of the most
        recently digested character of the input. See __controlling() for how
        that information can propagate to error messages.

        Prunes out comments, combining any space before and after. Yields
        successive fragments of text as either (quoted) String, Word, KeyWord
        or Gap (spacing) objects or single characters.

        For these purposes, numbers are words. File paths and command-line
        parameters will be broken up at their punctuators; see __glued() for
        how those get stitched back together again, possibly with variable
        references within them expanded in the process. No attempt is made to
        parse quoted strings for variable references, even though CMake does
        support that; the parameter blocks we need don't seem to exercise that
        feature."""
        def alnumunder(ch):
            return ch.isalnum() or ch == '_'
        def gather(text, test):
            assert text and test(text[0])
            end = 1
            while end < len(text) and test(text[end]):
                end += 1
            return end

        text = space = ''
        for line in stream:
            assert not text or text.endswith('\n')
            text = text + line
            while text:
                tail = text.lstrip()
                gap = text[:-len(tail)] if tail else text
                if gap:
                    space += position.consume(gap)
                    text = tail
                    if not text:
                        break

                if text.startswith('"'):
                    try:
                        end = text.index('"', 1)
                        assert end > 0, "Expected ValueError on no match"
                        while text[end - 1] == '\\':
                            count = 1
                            while text[end - count - 1] == '\\':
                                count += 1
                            if count % 2 == 0:
                                break # This double-quote wasn't escaped
                            end = text.index('"', end + 1)
                    except ValueError:
                        break # out of the while loop; need to consume more.
                    yield S(position.consume(text[:end + 1]))
                    # We discard spacing around String tokens already.
                    text = text[end + 1:]
                    tail = text.lstrip()
                    position.consume(text[:-len(tail)])
                    text, space = tail, ''
                elif text.startswith('#'):
                    try:
                        end = text.index('\n')
                    except ValueError:
                        position.consume(text)
                        text = ''
                    else:
                        position.consume(text[:end])
                        text = text[end:]
                else:
                    if space:
                        # Spacing still matters around words and ${...} expressions.
                        yield G(space)
                        space = ''

                    if alnumunder(text[0]):
                        end = gather(text, alnumunder)
                        yield W(position.consume(text[:end]))
                        text = text[end:]
                    elif text.startswith(':'):
                        end = gather(text, lambda ch: ch == ':')
                        yield position.consume(text[:end])
                        text = text[end:]
                    else:
                        yield position.consume(text[0])
                        text = text[1:]
            # Empty or incomplete text; consume next line.
        # End stream.
        # If we have dangling space, ignore it; but not non-space.
        # We might have an unterminated quoted string.
        if text:
            raise ParseError("Dangling text at end of stream", text)
    del word

    class TokenSeq(object):
        """A sequence of objects and maybe some raw strings.

        At the lowest layer of parsing only Token objects appear, but higher
        layer constructs may include TokenSeq objects."""
        def __init__(self, *tokens):
            self.tokens = tokens
            assert self._well_formed(), tokens

        # For derived classes to over-ride:
        _joiner_ = ' '
        def _well_formed(self):
            return True

        # ... to configure constructor assertion and:
        @property
        def text(self):
            return self._joiner_.join(tok if isinstance(tok, str) else tok.text
                                      for tok in self.tokens)

        def __eq__(self, other):
            if not isinstance(other, str):
                other = other.text
            return self.text == other

        # In pratice, both callers pass Function as cls.
        def traverse(self, cls, test):
            """Iterates self's children and theirs.

            Yields those that are instances of cls for which test returns
            true. For children that are themselves TokenSeq, recurses into them
            rather than yielding them. Yields self, when relevant, after all
            relevant children of self."""
            for tok in self.tokens:
                try:
                    inner = tok.traverse
                except AttributeError:
                    if isinstance(tok, cls) and test(tok):
                        yield tok
                else:
                    yield from inner(cls, test)

            if isinstance(self, cls) and test(self):
                yield self

    # Evaluate layer
    #
    # Identifying ${...} and $<...> tokens for later substitution.
    #
    # This should also be applied to String contents, in principle.
    # However, no such tokens are seen in SOURCES blocks, for now at least.

    class Evaluate (TokenSeq):
        """Object representing a CMake variable reference.

        Used to package a ${...} or $<...> expression, for eventual
        evaluation."""
        _joiner_ = ''
        def _well_formed(self):
            return (len(self.tokens) >= 4 and self.tokens[0] == '$'
                    and (self.tokens[1], self.tokens[-1]) in (('{', '}'), ('<', '>')))
        @property
        def variable(self):
            return self.tokens[2]
        @property
        def default(self):
            if len(self.tokens) > 5 and self.tokens[3] == ':':
                return self.tokens[4]

    __vareval = {'{': '}', '<': '>'}
    @classmethod
    def __expand(cls, toks, tokenstream, E=Evaluate, S=Gap):
        """Parse a ${...} or $<...> token, after its opening.

        Caller has spotted the opening tokens and saved them in toks; this now
        finds the rest, potentially recursing into other tokens of the same
        form, and packages the result as an Evaluate object."""
        assert len(toks) > 1 and toks[0] == '$'
        closer = cls.__vareval[toks[1]]
        try: # We need one token of look-ahead
            last = next(tokenstream)
            while isinstance(last, S):
                last = next(tokenstream)
        except StopIteration:
            pass
        else:
            if last == closer:
                toks.append(last)
                return E(*toks)

            for tok in tokenstream:
                if isinstance(tok, S):
                    continue
                if tok == closer:
                    toks.append(last)
                    toks.append(tok)
                    return E(*toks)
                if last == '$' and isinstance(tok, str) and tok in cls.__vareval:
                    last = cls.__expand([last, tok], tokenstream)
                    continue
                toks.append(last)
                last = tok
            # Fell off end; include last in error message.
            toks.append(last)

        raise ParseError("Unclosed $-evaluator", closer, toks)

    @classmethod
    def __expanded(cls, tokenstream):
        """Identifies ${...} and $<...> constructs in the token stream.

        Forwards tokens not involved in those unchanged, packages those that
        make up these constructs into Evaluate objects."""
        try: # We need one token of look-ahead:
            last = next(tokenstream)
        except StopIteration:
            pass
        else:
            for tok in tokenstream:
                if not isinstance(last, str) or last != '$':
                    pass # Even if we do have < or {, it's just another token.
                elif isinstance(tok, str) and tok in cls.__vareval:
                    last = cls.__expand([last, tok], tokenstream)
                    continue

                yield last
                last = tok
            yield last

    # Glue layer
    #
    # Gluing together "words" and common separators with no spacing between
    # them, for example to build file-names out of path fragments. Spacing
    # ceases being significant after this step, so is discarded.

    class Glue(TokenSeq):
        _joiner_ = ''
        def _well_formed(self):
            return len(self.tokens) > 1
    @classmethod
    def __glued(cls, tokenstream, G=Glue, W=(Word, Evaluate), S=Gap):
        """Combine word fragments, discard spacing.

        As long as there is no spacing between the word fragments, we presume
        they are juxtaposed. Evaluations of ${...} and $<...> count as word
        fragments, as do alphanumeric and underscore sequences, along with the
        various punctuators common in file-names, command-line options and
        CMake package names. Each sequence of juxtaposed tokens to be
        recombined is packaged in a Glue object that replaces those
        tokens.

        Tokens not caught up in such sequences are simply forwarded as part of
        the resulting token stream, with the exception of Gap (spacing) tokens,
        as we have no further interest in these."""
        def glued(ts, Glue=G):
            return Glue(*ts) if len(ts) > 1 else ts[0]
        toks = [] # Potentially many tokens of look-ahead
        for tok in cls.__expanded(tokenstream):
            # The : is doubled, so we recognize '::' as a token.
            if tok in './-,::' if isinstance(tok, str) else isinstance(tok, W):
                toks.append(tok)
                continue
            if toks:
                yield glued(toks)
                toks.clear()
            if not isinstance(tok, S):
                yield tok
        if toks:
            yield glued(toks)

    @classmethod
    def __gluing(cls, stream, position):
        """Parses raw-level tokens to concatenate certain fragments.

        Recognizes ${...} and $<...> tokens and glues together, where there are
        no spacs between them, runs of these, plain words (including keywords)
        and punctuators that tend to show up in CMake identifiers, paths and
        command-line options. Once it has done this, spacing tokens are no
        longer relevant, so it discards them."""
        yield from cls.__glued(cls.__tokenize(stream, position))

    # Call layer
    #
    # Handling of functions, their parameter lists and the grouping of those
    # parameters by capitalized words - in two sub-layers.

    # Nest sub-layer
    #
    # Handling of (potentially nested) parenthesised sub-sequences of tokens:
    class Nest(TokenSeq):
        def _well_formed(self):
            return self.tokens[0] == '(' and self.tokens[-1] == ')'
        @classmethod
        def __rescan(cls, toks, joiner):
            toks = (toks[0],) + tuple(joiner(iter(toks[1:-1]))) + (toks[-1],)
            return cls(*toks)
        def rescan(self, joiner):
            return self.__rescan(self.tokens, joiner)

    @classmethod
    def __nest(cls, toks, tokenstream, N=Nest):
        """Parses a parenthesised sequence of tokens.

        Caller has found an open-parenthesis and stored it in toks, to which
        this function appends further tokens up to the matching close,
        replacing any inner parenthesised sequences in between by calling
        itself recursively, and packages the result in a Nest object."""
        for tok in tokenstream:
            if tok == ')':
                toks.append(tok)
                return N(*toks)

            if tok == '(':
                toks.append(cls.__nest([tok], tokenstream))
            else:
                toks.append(tok)
        raise ParseError("Unclosed parentheses", toks)

    @classmethod
    def __nested(cls, tokenstream):
        """Recognize parentheses and match opens up with closes.

        Propagates tokens not between parentheses unchanged, replaces each
        parenthesised sequence of tokens with a Nest object. Raises ParseError
        if it finds any unmatched parentheses."""
        for tok in tokenstream:
            if tok == ')':
                raise ParseError("Unopened parentheses", tok)

            if tok == '(':
                yield cls.__nest([tok], tokenstream)
            else:
                yield tok

    @classmethod
    def __nesting(cls, stream, position):
        """Parses glue-level tokens to match parentheses.

        Groups each matched pair of parentheses, plus everything between them,
        into a Nest; supports recursive nesting of inner parentheses, to
        arbitrary depth (albeit I've seen no examples of that in CMakeLists.txt
        files)."""
        yield from cls.__nested(cls.__gluing(stream, position))

    # Function / Param sub-layer
    #
    # Handling of function calls and grouping of their parameters into blocks
    @staticmethod
    def __is_param_name(tok, W=Word, K=KeyWord):
        """Recognizer for parameter names in parameter lists.

        Only upper-case Words that aren't KeyWords are considered; and some
        names are excluded because they appear in the values of other
        parameters."""
        return (isinstance(tok, W) and not isinstance(tok, K)
                and tok.text.isupper()
                # Various DEFINES are upper-case, starting with QT,
                # and POLICIES have a QTP prefix.
                and not ((tok.text.startswith('QT')
                          and tok.text != 'QT_LICENSE_ID')
                         or tok.text.startswith('TEST_')))

    class Param(TokenSeq):
        """A single parameter block.

        Starts with a word for which __is_param_name() is true, followed by
        assorted things for which it isn't. Far from a complete solution in
        general, but we only care about source-file groups and the private
        header filter groups, for which it suffices. Has properties:

          name -- the first token
          value -- all the remaining tokens
          is_source -- true for source lists
          is_private_filter -- is name PRIVATE_HEADER_FILTERS ?

        The recognized source lists are SOURCES and NO_*_SOURCES.

        Although headers matching {RHI,QPA}_HEADER_FILTERS are not subject to
        strict compatibility commitments, they are considered semi-public API,
        hence reviewed so that - at least - we're aware of and do discuss any
        incompatible changes to them. So these are not filtered out here."""
        @property
        def name(self):
            return self.tokens[0]
        @property
        def value(self):
            return self.tokens[1:]

        @property
        def is_source(self):
            if self.name == 'SOURCES':
                return True
            name = self.name if isinstance(self.name, str) else self.name.text
            return name.startswith('NO_') and name.endswith('_SOURCES')
        @property
        def is_private_filter(self):
            return self.name == 'PRIVATE_HEADER_FILTERS'

        def headers(self, lookup, *exclude):
            """Iterate the public headers named in this parameter block.

            First argument is a callable to which to pass each token, that will
            iterate over possible values for that token. See __evaluate().

            Optional further arguments are callables which take a string
            (presumed to be a file name) and return true if it should be
            excluded; for example, matchers for regexes named in a
            PRIVATE_HEADER_FILTERS parameter (e.g. ".*\\.qpb\\.h" for
            protobuf).

            Only names ending with .h are considered and those ending in _p.h
            or that have a 3rdparty path component are already filtered out, as
            are any starting with a ../ path component. The intent is to mirror
            what syncqt does to generate the public include/ directory."""
            if self.is_source:
                for tok in self.value:
                    for name in lookup(tok):
                        if not name.endswith('.h') or name.endswith('_p.h'):
                            continue
                        if name.startswith('3rdparty/') or '/3rdparty/' in name:
                            continue
                        if name.startswith('../'):
                            continue
                        if any(e(name) for e in exclude):
                            continue
                        yield name

    @classmethod
    def __split_params(cls, tokenstream, P=Param):
        """Break up a parameter list into blocks.

        Each token for which __is_param_name() is true starts a block, which is
        packaged as a Param object if it has any further tokens in it, This
        ensures that the SOURCES (and similar variables) are easy to pick out
        from parameter lists."""
        def grouped(ps, Param=P):
            return Param(*ps) if len(ps) > 1 else ps[0]
        isName, params = cls.__is_param_name, []
        for tok in tokenstream:
            if isName(tok):
                if params:
                    yield grouped(params)
                    params.clear()
                params.append(tok)
            elif params:
                params.append(tok)
            else:
                yield tok
        if params:
            yield grouped(params)

    class Function(TokenSeq):
        """A function call.

        This comprises a function name (which may be a keyword) and a Nest
        representing its parameter list; these can be accessed by properties
        .function and .parameters; the latter omits the parentheses round the
        parameter list."""
        def _well_formed(self):
            # tokens[0] is a Word or Glue, tokens[1] is a Nest.
            if len(self.tokens) == 2:
                ps = self.tokens[1].tokens
                return ps[0] == '(' and ps[-1] == ')'
            return False
        @property
        def function(self):
            return self.tokens[0].text
        @property
        def parameters(self):
            return self.tokens[1].tokens[1:-1]

        @staticmethod
        def __adds_module(func):
            return func.startswith('qt_internal_add_') and func.endswith('_module')
        def grows_module(self, module):
            """True if this function contributes to the contents of the named module.

            This basically tests whether module is self.parameters[0] and
            self.function is either qt_internal_extend_target(module ...) or
            qt_internal_add*_module(module ...)."""
            func = self.function
            if self.__adds_module(func) or func == 'qt_internal_extend_target':
                mod = self.parameters[0]
                return not isinstance(mod, str) and mod.text == module
            return False

    @classmethod
    def __called(cls, tokenstream, F=Function, N=Nest, W=(Word, Glue), K=KeyWord):
        """Recognize function calls and turn them into Function objects.

        Assumes there are no function calls within the parameter list of the
        function calls. Recognizes any word (including keyword) followed by a
        parenthesized token sequence, with or without space between the word
        (function name) and open-parenthesis (of the parameter list). If the
        function isn't a keyword, the parameter list gets rescanned to
        decompose it into blocks starting with upper-case names, such as
        SOURCES; see __split_params() and Nest.rescan()."""
        try: # We need one token of look-ahead.
            last = next(tokenstream)
        except StopIteration:
            pass
        else:
            for tok in tokenstream:
                if isinstance(tok, N) and isinstance(last, W):
                    if not isinstance(last, K):
                        tok = tok.rescan(cls.__split_params)
                    last = F(last, tok)
                    continue
                yield last
                last = tok
            yield last

    # Control structure layer
    #
    # Handling of if/elseif/else/endif and foreach/endforeach blocks.
    #
    # Assumption: no conditionals appear *inside* parameter lists.
    # Conditional inclusion in a SOURCES list is handled by
    # conditional calls to functions to extend the list.

    class Conditional(TokenSeq):
        def _well_formed(self):
            return (self.tokens[0].function == 'if'
                    and self.tokens[-1].function == 'endif')
    @classmethod
    def __conditional(cls, tokenstream, toks, seenelse=False,
                      C=Conditional, F=Function, K=KeyWord):
        """Recognize if() blocks and turn them into Conditional objects.

        This may, of course, involve recursively parsing inner conditionals and
        loops."""
        for tok in tokenstream:
            if isinstance(tok, F) and isinstance(tok.tokens[0], K):
                if tok.function == 'if':
                    toks.append(cls.__conditional(tokenstream, [tok]))
                    continue
                if tok.function == 'endif':
                    toks.append(tok)
                    return C(*toks)
                if tok.function == 'foreach':
                    toks.append(cls.__foreach(tokenstream, [tok]))
                    continue
                if seenelse and tok.function in ('elseif', 'else'):
                    raise ParseError("Not allowed after else()", tok, toks)
                if tok.function == 'elseif':
                    toks.append(tok)
                    return cls.__conditional(tokenstream, toks)
                if tok.function == 'else':
                    toks.append(tok)
                    return cls.__conditional(tokenstream, toks, True)
                if tok.function == 'endforeach':
                    raise ParseError("Unstarted loop", tok, toks)
            toks.append(tok)
        if toks:
            raise ParseError("Unterminated conditional", toks)

    class ForEach(TokenSeq):
        def _well_formed(self):
            return (self.tokens[0].function == 'foreach'
                    and self.tokens[-1].function == 'endforeach')
    @classmethod
    def __foreach(cls, tokenstream, toks, E=ForEach, F=Function, K=KeyWord):
        """Recognize foreach() blocks and turn them into ForEach objects.

        This may, of course, involve recursively parsing inner loops and
        conditional blocks."""
        for tok in tokenstream:
            if isinstance(tok, F) and isinstance(tok.tokens[0], K):
                if tok.function == 'foreach':
                    toks.append(cls.__foreach(tokenstream, [tok]))
                    continue
                if tok.function == 'endforeach':
                    toks.append(tok)
                    return E(*toks)
                if tok.function == 'if':
                    toks.append(cls.__conditional(tokenstream, [tok]))
                    continue
                if tok.function in ('elseif', 'else', 'endif'):
                    raise ParseError("Unstarted conditional fragment", tok, toks)
            toks.append(tok)

    @classmethod
    def __controlled(cls, tokenstream, F=Function, K=KeyWord):
        """Recognize control structures, recursively.

        Represents them as Conditional and ForEach objects. These may be
        arbitrarily nested. Raises ParseError on mismatched control
        structure."""
        for tok in tokenstream:
            if isinstance(tok, F) and isinstance(tok.tokens[0], K):
                if tok.function == 'if':
                    yield cls.__conditional(tokenstream, [tok])
                    continue
                if tok.function == 'foreach':
                    yield cls.__foreach(tokenstream, [tok])
                    continue
                if tok.function in ('elseif', 'else', 'endif'):
                    raise ParseError("Unstarted conditional fragment", tok)
                if tok.function == 'endforeach':
                    raise ParseError("Unstarted loop", tok)
            yield tok

    @classmethod
    def __controlling(cls, stream):
        """Parses nesting-level tokens to identify control constructs.

        Recognizes function calls and (as grouped special cases of those)
        if/else/elsif/endif chains and foreach/endforeach blocks,
        recursively. Turns a nesting-level token stream into the top level
        token stream. Collaborates with __tokenize() to keep track of position
        in the input stream and adds a description of it to the end of the
        .args of any Exception triggered during parsing."""
        position = StreamPosition()
        try:
            yield from cls.__controlled(cls.__called(cls.__nesting(stream, position)))
        except Exception as what:
            what.args += (repr(position),)
            raise

    # End of parsing (all done by class methods).

    # Actual evaluation of Evaluate nodes and actual gluing of Glue
    # nodes, including lookup of variables set:
    def __single_value(self, token):
        """Converts a variable name to the variable's single value.

        If the variable was not set, token.text is returned instead; if the
        variable was set to more than one word, ParseError is raised."""
        seq = iter(self.__evaluate(token))
        try:
            result = next(seq)
        except StopIteration:
            # Represent the expansion of an unset variable by its raw text
            return token.text
        for extra in seq:
            raise ParseError("Glued variable has multi-candidate value", token, result, extra)
        return result

    def __evaluate(self, tok, E=Evaluate, G=Glue, T=TokenSeq):
        """Convert a token to its final textual representation."""
        if isinstance(tok, E):
            # We're not interested in generated files, in the build tree:
            if not '_BUILD_' in tok.variable.text:
                yield from self.__lookup(tok.variable, tok.default)
        elif isinstance(tok, G):
            yield ''.join(self.__single_value(t) for t in tok.tokens)
        elif isinstance(tok, T):
            for kid in tok.tokens:
                yield from self.__evaluate(kid)
        else:
            yield tok if isinstance(tok, str) else tok.text

    def __lookup(self, variable, default=None, F=Function):
        """Yields every non-empty word of the expanded variable.

        First parameter, variable, is the variable name to look up. Optional
        second, default, is the value to use if it has never been set. If the
        variable has been set several times, for example in different branches
        of a conditional, all values to which it has been set are yielded.

        This may, of course, involve recursively expanding any other variables
        used within the value of the variable. Assumes such recursion never
        needs to expand the same variable within its own expansion."""
        for tok in self.tokens:
            for kid in tok.traverse(F, lambda k: k.function == 'set'):
                ps = kid.parameters
                if ps[0] == variable:
                    for tok in ps[1:]:
                        for val in self.__evaluate(tok):
                            if val:
                                yield val
                                default = None
        if default is not None:
            yield from self.__evaluate(default)

    # Traverse parameter blocks of interest:
    def __modparam(self, test, F=Function, P=Param):
        """Iterates parameter blocks that meet a given condition.

        Calls its single parameter, test, on each Param block of each Function,
        found during a traverse() of each token in self.tokens, that defines or
        extends our module as a target; yields those for which test() returns
        true."""
        for tok in self.tokens:
            for kid in tok.traverse(F, lambda k: k.grows_module(self.module)):
                for param in kid.parameters:
                    if isinstance(param, P) and test(param):
                        yield param

    @property
    def __private_filters(self, S=String):
        """Iterator over private header regular expressions.

        Specifically, those found in PRIVATE_HEADER_FILTERS blocks of function
        calls that define or extend our module."""
        for param in self.__modparam(lambda p: p.is_private_filter):
            for tok in param.value:
                if isinstance(tok, str):
                    yield tok
                elif isinstance(tok, S):
                    yield tok.contents
                else:
                    raise ParseError(
                        "Neither regex nor string as private header filter",
                        tok, param.value)

    # Public API:
    def ingest(self, stream):
        """Parse stream, save result in self.tokens

        After this, self.headers will be able to report the interesting part of
        what it read."""
        self.tokens = tuple(self.__controlling(stream))

    @property
    def headers(self):
        """Iterates the public headers of self.module

        Won't find any unless it's ingest()ed the relevant CMakeLists.txt
        stream first."""
        filters = tuple(re.compile(f).match for f in self.__private_filters)
        for param in self.__modparam(lambda p: p.is_source):
            for tok in param.headers(self.__evaluate, *filters):
                for word in self.__evaluate(tok):
                    assert isinstance(word, str)
                    yield str(self.root.joinpath(word)) + '\n'

def main(args, source, sink, grumble):
    """Driver for the CMakeScanner class.

    Pipe the content of the CMakeLists.txt file in on source (stdin). Name the
    module it supposedly defines on the command line (args) along with the path
    of the CMakeLists.txt file (relative to its module's root). The directory
    part of this filename will be prepended to each public header read from its
    contents. Delivers '\n'-joined names of the headers that define public API
    to sink (stdout.write). May complain about problems to grumble
    (stderr.write)."""
    me = 'cmakescan.py'
    if args and args[0].endswith(me):
        me, args = args[0], args[1:]
    if len(args) < 2 or Path(args[1]).name != 'CMakeLists.txt':
        grumble(f"{me}: Pass name of module and path of CMakeLists.txt on command line\n")
        return 1
    scan = CMakeScanner(args[0], Path(args[1]).parent)
    scan.ingest(source)
    sink(''.join(scan.headers))
    return 0

if __name__ == '__main__':
    import sys
    sys.exit(main(sys.argv, sys.stdin, sys.stdout.write, sys.stderr.write))