#!/usr/bin/env python3 # Usage: see api-review-gen # Copyright (C) 2023 The Qt Company Ltd. # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only """Script to scan a CMakeLists.txt file for a list of public API-defining headers. A somewhat brutal recursive-descent parser for as much (and a random bit more) of the file format as matters to the task in hand. Assumes the source is well-enough formed. See its main()'s documentation for details. """ import re from pathlib import Path class ParseError(Exception): pass class StreamPosition(object): def __init__(self): self.line = self.column = 0 def __repr__(self): return f'line: {self.line}, column: {self.column}' def consume(self, text): endl = text.rfind('\n') if endl < 0: self.column += len(text) else: self.column = len(text) - endl - 1 self.line += text.count('\n') return text class CMakeScanner(object): def __init__(self, module, root): """Set up a scanner for the named module. First argument, the module name, is used to select which function-calls are of interest. It is saved as self.module for future reference. Second argument, root, is the path to the directory in which the CMakeLists.txt file being parsed was found: this should be given relative to the git module's root. Paths of headers in the source-file lists are given relative to the CMakeLists.txt file, so this root path is prepended to each to make them relative to the git module's root.""" self.module, self.root = module, root # Parsing is structured in various layers, as class methods. # # There's a stack of __*ing() methods reaching down from the top layer, # that forward the raw stream and its position tracker to __tokenize, # calling their respective layers' sub-layer processors, __*ed(), on the # stream of tokens they get back from the layer below, to produce the # stream they deliver to the layer above. The __*ed() call spots the starts # of the structures it's there to recognize and delegates to the plain # verb-named __* function, passing it a list of the opening token(s) of the # structure, to which it'll append the rest of the structure before # packaging the sequence of tokens as an object of a structure-appropriate # sub-type of TokenSeq, which it returns. Its collecting of tokens to # package in this way may well recurse into itself, if it spots the opening # of an inner instance of its structure type (or another recognized at the # same layer of the parser). The structures we need to recognize are # simple enough that this suffices. class Token(object): def __init__(self, text): self.text = text def __bool__(self): return self.text def __eq__(self, text): return self.text == text def __str__(self): return f'{self.__class__.__name__}("{self.text}")' __repr__ = __str__ # Sub-classes are used to package distinct types of Token. class Gap(Token): pass class Word(Token): pass class KeyWord(Word): pass class String(Token): @property def contents(self): # Only used for private filter regexes. # Should this unescape ? return self.text[1:-1] # Throw-away tool function, to tunnel into __tokenize() and delete later: # a pseudo-constructor for Word and KeyWord that choses which to use. def word(text, W=Word, K=KeyWord): if text in ('if', 'elseif', 'else', 'endif', 'foreach', 'endforeach', 'function', 'endfunction', 'macro', 'endmacro', 'set'): return K(text) return W(text) @staticmethod def __tokenize(stream, position, G=Gap, S=String, W=word): """Digests the stream into basic lexical tokens. First parameter, stream, must be an iterable over lines of text. These are tacitly ''.join()ed but only as fast as what's been scanned has been consumed enough to make it necessary to read the next. Second parameter, position, should be a StreamPosition object, whose consume() will be passed all characters as they are consumed by the tokenizer, so that it can keep track of the line and column number of the most recently digested character of the input. See __controlling() for how that information can propagate to error messages. Prunes out comments, combining any space before and after. Yields successive fragments of text as either (quoted) String, Word, KeyWord or Gap (spacing) objects or single characters. For these purposes, numbers are words. File paths and command-line parameters will be broken up at their punctuators; see __glued() for how those get stitched back together again, possibly with variable references within them expanded in the process. No attempt is made to parse quoted strings for variable references, even though CMake does support that; the parameter blocks we need don't seem to exercise that feature.""" def alnumunder(ch): return ch.isalnum() or ch == '_' def gather(text, test): assert text and test(text[0]) end = 1 while end < len(text) and test(text[end]): end += 1 return end text = space = '' for line in stream: assert not text or text.endswith('\n') text = text + line while text: tail = text.lstrip() gap = text[:-len(tail)] if tail else text if gap: space += position.consume(gap) text = tail if not text: break if text.startswith('"'): try: end = text.index('"', 1) assert end > 0, "Expected ValueError on no match" while text[end - 1] == '\\': count = 1 while text[end - count - 1] == '\\': count += 1 if count % 2 == 0: break # This double-quote wasn't escaped end = text.index('"', end + 1) except ValueError: break # out of the while loop; need to consume more. yield S(position.consume(text[:end + 1])) # We discard spacing around String tokens already. text = text[end + 1:] tail = text.lstrip() position.consume(text[:-len(tail)]) text, space = tail, '' elif text.startswith('#'): try: end = text.index('\n') except ValueError: position.consume(text) text = '' else: position.consume(text[:end]) text = text[end:] else: if space: # Spacing still matters around words and ${...} expressions. yield G(space) space = '' if alnumunder(text[0]): end = gather(text, alnumunder) yield W(position.consume(text[:end])) text = text[end:] elif text.startswith(':'): end = gather(text, lambda ch: ch == ':') yield position.consume(text[:end]) text = text[end:] else: yield position.consume(text[0]) text = text[1:] # Empty or incomplete text; consume next line. # End stream. # If we have dangling space, ignore it; but not non-space. # We might have an unterminated quoted string. if text: raise ParseError("Dangling text at end of stream", text) del word class TokenSeq(object): """A sequence of objects and maybe some raw strings. At the lowest layer of parsing only Token objects appear, but higher layer constructs may include TokenSeq objects.""" def __init__(self, *tokens): self.tokens = tokens assert self._well_formed(), tokens # For derived classes to over-ride: _joiner_ = ' ' def _well_formed(self): return True # ... to configure constructor assertion and: @property def text(self): return self._joiner_.join(tok if isinstance(tok, str) else tok.text for tok in self.tokens) def __eq__(self, other): if not isinstance(other, str): other = other.text return self.text == other # In pratice, both callers pass Function as cls. def traverse(self, cls, test): """Iterates self's children and theirs. Yields those that are instances of cls for which test returns true. For children that are themselves TokenSeq, recurses into them rather than yielding them. Yields self, when relevant, after all relevant children of self.""" for tok in self.tokens: try: inner = tok.traverse except AttributeError: if isinstance(tok, cls) and test(tok): yield tok else: yield from inner(cls, test) if isinstance(self, cls) and test(self): yield self # Evaluate layer # # Identifying ${...} and $<...> tokens for later substitution. # # This should also be applied to String contents, in principle. # However, no such tokens are seen in SOURCES blocks, for now at least. class Evaluate (TokenSeq): """Object representing a CMake variable reference. Used to package a ${...} or $<...> expression, for eventual evaluation.""" _joiner_ = '' def _well_formed(self): return (len(self.tokens) >= 4 and self.tokens[0] == '$' and (self.tokens[1], self.tokens[-1]) in (('{', '}'), ('<', '>'))) @property def variable(self): return self.tokens[2] @property def default(self): if len(self.tokens) > 5 and self.tokens[3] == ':': return self.tokens[4] __vareval = {'{': '}', '<': '>'} @classmethod def __expand(cls, toks, tokenstream, E=Evaluate, S=Gap): """Parse a ${...} or $<...> token, after its opening. Caller has spotted the opening tokens and saved them in toks; this now finds the rest, potentially recursing into other tokens of the same form, and packages the result as an Evaluate object.""" assert len(toks) > 1 and toks[0] == '$' closer = cls.__vareval[toks[1]] try: # We need one token of look-ahead last = next(tokenstream) while isinstance(last, S): last = next(tokenstream) except StopIteration: pass else: if last == closer: toks.append(last) return E(*toks) for tok in tokenstream: if isinstance(tok, S): continue if tok == closer: toks.append(last) toks.append(tok) return E(*toks) if last == '$' and isinstance(tok, str) and tok in cls.__vareval: last = cls.__expand([last, tok], tokenstream) continue toks.append(last) last = tok # Fell off end; include last in error message. toks.append(last) raise ParseError("Unclosed $-evaluator", closer, toks) @classmethod def __expanded(cls, tokenstream): """Identifies ${...} and $<...> constructs in the token stream. Forwards tokens not involved in those unchanged, packages those that make up these constructs into Evaluate objects.""" try: # We need one token of look-ahead: last = next(tokenstream) except StopIteration: pass else: for tok in tokenstream: if not isinstance(last, str) or last != '$': pass # Even if we do have < or {, it's just another token. elif isinstance(tok, str) and tok in cls.__vareval: last = cls.__expand([last, tok], tokenstream) continue yield last last = tok yield last # Glue layer # # Gluing together "words" and common separators with no spacing between # them, for example to build file-names out of path fragments. Spacing # ceases being significant after this step, so is discarded. class Glue(TokenSeq): _joiner_ = '' def _well_formed(self): return len(self.tokens) > 1 @classmethod def __glued(cls, tokenstream, G=Glue, W=(Word, Evaluate), S=Gap): """Combine word fragments, discard spacing. As long as there is no spacing between the word fragments, we presume they are juxtaposed. Evaluations of ${...} and $<...> count as word fragments, as do alphanumeric and underscore sequences, along with the various punctuators common in file-names, command-line options and CMake package names. Each sequence of juxtaposed tokens to be recombined is packaged in a Glue object that replaces those tokens. Tokens not caught up in such sequences are simply forwarded as part of the resulting token stream, with the exception of Gap (spacing) tokens, as we have no further interest in these.""" def glued(ts, Glue=G): return Glue(*ts) if len(ts) > 1 else ts[0] toks = [] # Potentially many tokens of look-ahead for tok in cls.__expanded(tokenstream): # The : is doubled, so we recognize '::' as a token. if tok in './-,::' if isinstance(tok, str) else isinstance(tok, W): toks.append(tok) continue if toks: yield glued(toks) toks.clear() if not isinstance(tok, S): yield tok if toks: yield glued(toks) @classmethod def __gluing(cls, stream, position): """Parses raw-level tokens to concatenate certain fragments. Recognizes ${...} and $<...> tokens and glues together, where there are no spacs between them, runs of these, plain words (including keywords) and punctuators that tend to show up in CMake identifiers, paths and command-line options. Once it has done this, spacing tokens are no longer relevant, so it discards them.""" yield from cls.__glued(cls.__tokenize(stream, position)) # Call layer # # Handling of functions, their parameter lists and the grouping of those # parameters by capitalized words - in two sub-layers. # Nest sub-layer # # Handling of (potentially nested) parenthesised sub-sequences of tokens: class Nest(TokenSeq): def _well_formed(self): return self.tokens[0] == '(' and self.tokens[-1] == ')' @classmethod def __rescan(cls, toks, joiner): toks = (toks[0],) + tuple(joiner(iter(toks[1:-1]))) + (toks[-1],) return cls(*toks) def rescan(self, joiner): return self.__rescan(self.tokens, joiner) @classmethod def __nest(cls, toks, tokenstream, N=Nest): """Parses a parenthesised sequence of tokens. Caller has found an open-parenthesis and stored it in toks, to which this function appends further tokens up to the matching close, replacing any inner parenthesised sequences in between by calling itself recursively, and packages the result in a Nest object.""" for tok in tokenstream: if tok == ')': toks.append(tok) return N(*toks) if tok == '(': toks.append(cls.__nest([tok], tokenstream)) else: toks.append(tok) raise ParseError("Unclosed parentheses", toks) @classmethod def __nested(cls, tokenstream): """Recognize parentheses and match opens up with closes. Propagates tokens not between parentheses unchanged, replaces each parenthesised sequence of tokens with a Nest object. Raises ParseError if it finds any unmatched parentheses.""" for tok in tokenstream: if tok == ')': raise ParseError("Unopened parentheses", tok) if tok == '(': yield cls.__nest([tok], tokenstream) else: yield tok @classmethod def __nesting(cls, stream, position): """Parses glue-level tokens to match parentheses. Groups each matched pair of parentheses, plus everything between them, into a Nest; supports recursive nesting of inner parentheses, to arbitrary depth (albeit I've seen no examples of that in CMakeLists.txt files).""" yield from cls.__nested(cls.__gluing(stream, position)) # Function / Param sub-layer # # Handling of function calls and grouping of their parameters into blocks @staticmethod def __is_param_name(tok, W=Word, K=KeyWord): """Recognizer for parameter names in parameter lists. Only upper-case Words that aren't KeyWords are considered; and some names are excluded because they appear in the values of other parameters.""" return (isinstance(tok, W) and not isinstance(tok, K) and tok.text.isupper() # Various DEFINES are upper-case, starting with QT, # and POLICIES have a QTP prefix. and not ((tok.text.startswith('QT') and tok.text != 'QT_LICENSE_ID') or tok.text.startswith('TEST_'))) class Param(TokenSeq): """A single parameter block. Starts with a word for which __is_param_name() is true, followed by assorted things for which it isn't. Far from a complete solution in general, but we only care about source-file groups and the private header filter groups, for which it suffices. Has properties: name -- the first token value -- all the remaining tokens is_source -- true for source lists is_private_filter -- is name PRIVATE_HEADER_FILTERS ? The recognized source lists are SOURCES and NO_*_SOURCES. Although headers matching {RHI,QPA}_HEADER_FILTERS are not subject to strict compatibility commitments, they are considered semi-public API, hence reviewed so that - at least - we're aware of and do discuss any incompatible changes to them. So these are not filtered out here.""" @property def name(self): return self.tokens[0] @property def value(self): return self.tokens[1:] @property def is_source(self): if self.name == 'SOURCES': return True name = self.name if isinstance(self.name, str) else self.name.text return name.startswith('NO_') and name.endswith('_SOURCES') @property def is_private_filter(self): return self.name == 'PRIVATE_HEADER_FILTERS' def headers(self, lookup, *exclude): """Iterate the public headers named in this parameter block. First argument is a callable to which to pass each token, that will iterate over possible values for that token. See __evaluate(). Optional further arguments are callables which take a string (presumed to be a file name) and return true if it should be excluded; for example, matchers for regexes named in a PRIVATE_HEADER_FILTERS parameter (e.g. ".*\\.qpb\\.h" for protobuf). Only names ending with .h are considered and those ending in _p.h or that have a 3rdparty path component are already filtered out, as are any starting with a ../ path component. The intent is to mirror what syncqt does to generate the public include/ directory.""" if self.is_source: for tok in self.value: for name in lookup(tok): if not name.endswith('.h') or name.endswith('_p.h'): continue if name.startswith('3rdparty/') or '/3rdparty/' in name: continue if name.startswith('../'): continue if any(e(name) for e in exclude): continue yield name @classmethod def __split_params(cls, tokenstream, P=Param): """Break up a parameter list into blocks. Each token for which __is_param_name() is true starts a block, which is packaged as a Param object if it has any further tokens in it, This ensures that the SOURCES (and similar variables) are easy to pick out from parameter lists.""" def grouped(ps, Param=P): return Param(*ps) if len(ps) > 1 else ps[0] isName, params = cls.__is_param_name, [] for tok in tokenstream: if isName(tok): if params: yield grouped(params) params.clear() params.append(tok) elif params: params.append(tok) else: yield tok if params: yield grouped(params) class Function(TokenSeq): """A function call. This comprises a function name (which may be a keyword) and a Nest representing its parameter list; these can be accessed by properties .function and .parameters; the latter omits the parentheses round the parameter list.""" def _well_formed(self): # tokens[0] is a Word or Glue, tokens[1] is a Nest. if len(self.tokens) == 2: ps = self.tokens[1].tokens return ps[0] == '(' and ps[-1] == ')' return False @property def function(self): return self.tokens[0].text @property def parameters(self): return self.tokens[1].tokens[1:-1] @staticmethod def __adds_module(func): return func.startswith('qt_internal_add_') and func.endswith('_module') def grows_module(self, module): """True if this function contributes to the contents of the named module. This basically tests whether module is self.parameters[0] and self.function is either qt_internal_extend_target(module ...) or qt_internal_add*_module(module ...).""" func = self.function if self.__adds_module(func) or func == 'qt_internal_extend_target': mod = self.parameters[0] return not isinstance(mod, str) and mod.text == module return False @classmethod def __called(cls, tokenstream, F=Function, N=Nest, W=(Word, Glue), K=KeyWord): """Recognize function calls and turn them into Function objects. Assumes there are no function calls within the parameter list of the function calls. Recognizes any word (including keyword) followed by a parenthesized token sequence, with or without space between the word (function name) and open-parenthesis (of the parameter list). If the function isn't a keyword, the parameter list gets rescanned to decompose it into blocks starting with upper-case names, such as SOURCES; see __split_params() and Nest.rescan().""" try: # We need one token of look-ahead. last = next(tokenstream) except StopIteration: pass else: for tok in tokenstream: if isinstance(tok, N) and isinstance(last, W): if not isinstance(last, K): tok = tok.rescan(cls.__split_params) last = F(last, tok) continue yield last last = tok yield last # Control structure layer # # Handling of if/elseif/else/endif and foreach/endforeach blocks. # # Assumption: no conditionals appear *inside* parameter lists. # Conditional inclusion in a SOURCES list is handled by # conditional calls to functions to extend the list. class Conditional(TokenSeq): def _well_formed(self): return (self.tokens[0].function == 'if' and self.tokens[-1].function == 'endif') @classmethod def __conditional(cls, tokenstream, toks, seenelse=False, C=Conditional, F=Function, K=KeyWord): """Recognize if() blocks and turn them into Conditional objects. This may, of course, involve recursively parsing inner conditionals and loops.""" for tok in tokenstream: if isinstance(tok, F) and isinstance(tok.tokens[0], K): if tok.function == 'if': toks.append(cls.__conditional(tokenstream, [tok])) continue if tok.function == 'endif': toks.append(tok) return C(*toks) if tok.function == 'foreach': toks.append(cls.__foreach(tokenstream, [tok])) continue if seenelse and tok.function in ('elseif', 'else'): raise ParseError("Not allowed after else()", tok, toks) if tok.function == 'elseif': toks.append(tok) return cls.__conditional(tokenstream, toks) if tok.function == 'else': toks.append(tok) return cls.__conditional(tokenstream, toks, True) if tok.function == 'endforeach': raise ParseError("Unstarted loop", tok, toks) toks.append(tok) if toks: raise ParseError("Unterminated conditional", toks) class ForEach(TokenSeq): def _well_formed(self): return (self.tokens[0].function == 'foreach' and self.tokens[-1].function == 'endforeach') @classmethod def __foreach(cls, tokenstream, toks, E=ForEach, F=Function, K=KeyWord): """Recognize foreach() blocks and turn them into ForEach objects. This may, of course, involve recursively parsing inner loops and conditional blocks.""" for tok in tokenstream: if isinstance(tok, F) and isinstance(tok.tokens[0], K): if tok.function == 'foreach': toks.append(cls.__foreach(tokenstream, [tok])) continue if tok.function == 'endforeach': toks.append(tok) return E(*toks) if tok.function == 'if': toks.append(cls.__conditional(tokenstream, [tok])) continue if tok.function in ('elseif', 'else', 'endif'): raise ParseError("Unstarted conditional fragment", tok, toks) toks.append(tok) @classmethod def __controlled(cls, tokenstream, F=Function, K=KeyWord): """Recognize control structures, recursively. Represents them as Conditional and ForEach objects. These may be arbitrarily nested. Raises ParseError on mismatched control structure.""" for tok in tokenstream: if isinstance(tok, F) and isinstance(tok.tokens[0], K): if tok.function == 'if': yield cls.__conditional(tokenstream, [tok]) continue if tok.function == 'foreach': yield cls.__foreach(tokenstream, [tok]) continue if tok.function in ('elseif', 'else', 'endif'): raise ParseError("Unstarted conditional fragment", tok) if tok.function == 'endforeach': raise ParseError("Unstarted loop", tok) yield tok @classmethod def __controlling(cls, stream): """Parses nesting-level tokens to identify control constructs. Recognizes function calls and (as grouped special cases of those) if/else/elsif/endif chains and foreach/endforeach blocks, recursively. Turns a nesting-level token stream into the top level token stream. Collaborates with __tokenize() to keep track of position in the input stream and adds a description of it to the end of the .args of any Exception triggered during parsing.""" position = StreamPosition() try: yield from cls.__controlled(cls.__called(cls.__nesting(stream, position))) except Exception as what: what.args += (repr(position),) raise # End of parsing (all done by class methods). # Actual evaluation of Evaluate nodes and actual gluing of Glue # nodes, including lookup of variables set: def __single_value(self, token): """Converts a variable name to the variable's single value. If the variable was not set, token.text is returned instead; if the variable was set to more than one word, ParseError is raised.""" seq = iter(self.__evaluate(token)) try: result = next(seq) except StopIteration: # Represent the expansion of an unset variable by its raw text return token.text for extra in seq: raise ParseError("Glued variable has multi-candidate value", token, result, extra) return result def __evaluate(self, tok, E=Evaluate, G=Glue, T=TokenSeq): """Convert a token to its final textual representation.""" if isinstance(tok, E): # We're not interested in generated files, in the build tree: if not '_BUILD_' in tok.variable.text: yield from self.__lookup(tok.variable, tok.default) elif isinstance(tok, G): yield ''.join(self.__single_value(t) for t in tok.tokens) elif isinstance(tok, T): for kid in tok.tokens: yield from self.__evaluate(kid) else: yield tok if isinstance(tok, str) else tok.text def __lookup(self, variable, default=None, F=Function): """Yields every non-empty word of the expanded variable. First parameter, variable, is the variable name to look up. Optional second, default, is the value to use if it has never been set. If the variable has been set several times, for example in different branches of a conditional, all values to which it has been set are yielded. This may, of course, involve recursively expanding any other variables used within the value of the variable. Assumes such recursion never needs to expand the same variable within its own expansion.""" for tok in self.tokens: for kid in tok.traverse(F, lambda k: k.function == 'set'): ps = kid.parameters if ps[0] == variable: for tok in ps[1:]: for val in self.__evaluate(tok): if val: yield val default = None if default is not None: yield from self.__evaluate(default) # Traverse parameter blocks of interest: def __modparam(self, test, F=Function, P=Param): """Iterates parameter blocks that meet a given condition. Calls its single parameter, test, on each Param block of each Function, found during a traverse() of each token in self.tokens, that defines or extends our module as a target; yields those for which test() returns true.""" for tok in self.tokens: for kid in tok.traverse(F, lambda k: k.grows_module(self.module)): for param in kid.parameters: if isinstance(param, P) and test(param): yield param @property def __private_filters(self, S=String): """Iterator over private header regular expressions. Specifically, those found in PRIVATE_HEADER_FILTERS blocks of function calls that define or extend our module.""" for param in self.__modparam(lambda p: p.is_private_filter): for tok in param.value: if isinstance(tok, str): yield tok elif isinstance(tok, S): yield tok.contents else: raise ParseError( "Neither regex nor string as private header filter", tok, param.value) # Public API: def ingest(self, stream): """Parse stream, save result in self.tokens After this, self.headers will be able to report the interesting part of what it read.""" self.tokens = tuple(self.__controlling(stream)) @property def headers(self): """Iterates the public headers of self.module Won't find any unless it's ingest()ed the relevant CMakeLists.txt stream first.""" filters = tuple(re.compile(f).match for f in self.__private_filters) for param in self.__modparam(lambda p: p.is_source): for tok in param.headers(self.__evaluate, *filters): for word in self.__evaluate(tok): assert isinstance(word, str) yield str(self.root.joinpath(word)) + '\n' def main(args, source, sink, grumble): """Driver for the CMakeScanner class. Pipe the content of the CMakeLists.txt file in on source (stdin). Name the module it supposedly defines on the command line (args) along with the path of the CMakeLists.txt file (relative to its module's root). The directory part of this filename will be prepended to each public header read from its contents. Delivers '\n'-joined names of the headers that define public API to sink (stdout.write). May complain about problems to grumble (stderr.write).""" me = 'cmakescan.py' if args and args[0].endswith(me): me, args = args[0], args[1:] if len(args) < 2 or Path(args[1]).name != 'CMakeLists.txt': grumble(f"{me}: Pass name of module and path of CMakeLists.txt on command line\n") return 1 scan = CMakeScanner(args[0], Path(args[1]).parent) scan.ingest(source) sink(''.join(scan.headers)) return 0 if __name__ == '__main__': import sys sys.exit(main(sys.argv, sys.stdin, sys.stdout.write, sys.stderr.write))