path: root/scripts/api-review/api-review-gen

#!/bin/sh
# Copyright (C) 2020 The Qt Company Ltd.
# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
usage () {
    echo Usage: `basename $0` "[-h|-u] [-v] [-a] [-m mod] [-r name] [-t task] [--] prior soon";
}

help () {
    usage
    cat <<EOF

Prepare commits for pushing to Gerrit, one commit for each Qt module
within the git module, that express the interesting changes in API
between a prior release (tag or branch) and an imminent release
(branch). Certain boring changes are excluded; you can review them
after this has run with git diff -D; if necessary, you can amend
commits to remove any remaining boredom or include anything
mis-classified as boring, before pushing to Gerrit.

Run in the top-level directory of a work tree in a clean state; don't
expect it to be left in a clean state. In particular, any sub-modules
shall be checked out on their versions from the prior release.

Depends on the dulwich package for python; see your local package
manager or 'pip install dulwich' if managing python packages with pip.

Options:

 -a
 --amend   Amend existing commit rather than extending the review
           branch, if it exists already. This is done by saving fixup!
           commits for the new changes and rebasing with autosquash;
           you'll need to agree to the proposed rebase, so this can
           only be used interactively.

 -h
 --help    Print this help and exit.

 -m mod
 --module mod
           Limit to Qt modules (libraries) specified likewise with -m;
           by default all Qt modules are checked for API changes, but
           specifying this option limits the scan to only those so
           specified. Names are matched case-insensitively.

 -q
 --quiet   Don't mention anything but errors.

 --replace Replace existing branch, if present, re-using its last
           Change-Id, rather than updating it.  (Conflicts with
           --amend, which should be preferred unless these scripts
           have changed.  If both are used, the last wins.)  If
           --task-number has not been specified, any Task-number:
           footer in the earlier commit will also be preserved.

 -r name
 --release name
           Supply a name for the imminent release, if the branch given
           for it doesn't match (e.g. when preparing a review on dev,
           before the release branch is created).

 -t ID
 --task ID
 --task-number ID
           Include a Task-number: in the commit message, with the
           given id, ID, of the bug-tracker issue being used to keep
           track of the API change review.

 -u
 --usage   Print simple usage summary (first line above) and exit.

 -v
 --verbose Say what's happening as it happens.

 --        End option processing.

Arguments:

 prior     The commit (branch, tag or sha1) of a prior release.
 soon      The branch approaching release.

You should see the usual git output describing the commit created.
You should verify that git diff -D is all boring and the commit is all
interesting before pushing this commit to Gerrit.

Exit status is 1 on error, else 0.  Success may mean no change at all,
only boring changes or some interesting changes have been saved.  If
there are no changes, the repository is restored to its prior state
(including deleting the api-review-* branch, if created rather than
reused), except for any sub-modules.  The other two cases can be
distinguished by comparing the api-review-* branch with the prior
release using git diff --quiet; this succeeds if there is no
difference - i.e. if the API change was all boring so we made no
commit.

After a first pass of review, if prior or soon has changed (and been
fetched), it is possible to rebase the review branch onto prior and
re-run this script to update the review.  If you pass the --amend
option and there was a prior commit on the branch, the last such
commit shall be amended rather than a new commit added to the branch.
Otherwise, any change shall be recorded as a fresh commit - which you
can always squash onto an earlier commit later, after reviewing it, if
you prefer that work-flow so left out --amend.  Either way, you can
then push the squashed or amended commit to Gerrit for re-review, as
usual.
EOF
}
warn () { echo "$@" >&2; }
die () { warn "$@"; exit 1; }
second () {
    if [ $# -lt 2 ]
    then die "No argument supplied for $1"
    elif [ -z "$2" ]
    then die "Empty argument passed for $1 "
    fi
    echo "$2"
}

modules () {
    # Scan for CMakeLists.txt files defining non-Private, non-plugin Qt modules:
    # Known variants on qt_internal_add_*_module: protobuf, qml
    find src -type d -name .git -prune -o -type f -name CMakeLists.txt -print \
        | xargs grep '^[^#]*qt_internal_add\(_[a-z][a-z]*\)\?_module(' \
        | grep -wv 'qt_internal_add.*module([A-Za-z0-9]*Private' \
        | grep -wv 'qt_internal_add.*module([A-Za-z0-9]*plugin' \
        | sed -e 's/: *qt_internal_add.*_module(/ /' \
        | while read file module ignore
    do echo $module $file
    done
}

# Parse command-line:
MODULES=
RELNAME=
CHATTY=
AMEND=
TASK=
bad () { usage >&2; die "$@"; }
while [ $# -gt 0 ]
do  case "$1" in
        -u|--usage) usage; exit 0 ;;
        -h|--help) help; exit 0 ;;
        -a|--amend) AMEND=--amend; shift ;;
        -m|--module) M=`second "$@"`; MODULES="$MODULES $M"; shift 2 ;;
        --replace) AMEND=--replace; shift ;;
        -r|--release) RELNAME=`second "$@"`; shift 2 ;;
        -t|--task|--task-number) TASK=`second "$@"`; shift 2 ;;
        -v|--verbose) CHATTY=more; shift ;;
        -q|--quiet) CHATTY=less; shift ;;
        --) shift; break ;;
        -*) bad "Unrecognised option: $1" ;;
        *) break ;;
    esac
done

# Check basic expectations of context:
[ -d src ] || \
    die "I expect to be run in the top level directory of a module (see --help)."

GITMODULE=`/bin/pwd`
GITMODULE=`basename "$GITMODULE"`

THERE=`dirname $0`
[ -n "$THERE" -a -x "$THERE/resetboring.py" ] || \
    die "I don't know where resetboring.py is: please run me via an explicit path."
python3 -c 'from dulwich.repo import Repo; from dulwich.index import IndexEntry' || \
    die "I need dulwich installed (for resetboring.py; see --help)."
# dulwich 0.16.3 has been known to work; 0.9.4 is too old.

# Select revisions to compare:
[ $# -eq 2 ] || bad "Expected exactly two arguments, got $#: $@"
for arg
do git rev-parse "$arg" -- >/dev/null || bad "Failed to parse $arg as a git ref"
done
PRIOR="$1"
RELEASE="$2"
[ -n "$RELNAME" ] || RELNAME="$RELEASE"
RESTORE="`git branch | sed -n -e '/^\* (HEAD/ s/.* \([^ ]*\))$/\1/ p' -e '/^\*/ s/.* // p'`"

# Implement --verbose, --quiet:
mutter () { true; }
mention () { warn "$@"; }
# Option to pass to various git commands:
QUIET=
UNQUIET=-q
case "$CHATTY" in
    more) UNQUIET=
        mutter () { warn "$@"; }
        ;;
    less) QUIET=-q
        mention () { true; }
        ;;
    *) ;;
esac
run () { mutter "Running: $@"; eval "$@" || die "Failed ($?): $@"; }
logged () { mention "Running: $@"; eval "$@"; }
checkout () { run git checkout $QUIET "$@"; }
changeid () {
    git log --grep "$1: API comparison from $PRIOR to $RELNAME in $GITMODULE" "$PRIOR..$2" \
        | sed -ne '/Change-Id:/ s/.*: *//p'
}

cmakerecurse () {
    # usage: cmakerecurse $rev $mod $path/CMakeLists.txt
    # First identify headers overtly reported for this version in $3 for module $2:
    git show "$1:$3" 2>/dev/null | $THERE/cmakescan.py "$2" "$3" \
        || die "Failed to scan $f for $2 at $1"
    # Recurse into subdirectories, that might extend the module target:
    git show "$1:$3" 2>/dev/null \
        | sed -ne '/add_subdirectory/ s/add_subdirectory(// p' | tr ')' ' ' \
        | while read sub
    do cmakerecurse "$1" "$2" "`dirname "$3"`/$sub/`basename "$3"`"
    done
}

headers () {
    # usage: pipe CMake files one per line into | headers rev module
    # If a file did not exist at the given revision, it produces no output.
    while read f
    do cmakerecurse "$1" "$2" "$f"
    done
}

# We need to know both the new modules and the old:
PROC="$$-$GITMODULE"
NEWMODS="../.modules-$PROC"
checkout "$RELEASE"
modules >"$NEWMODS"

# Get API headers of $RELEASE checked out on a branch off $PRIOR:
BRANCH="api-review-$PRIOR-$RELNAME"
mutter "Checking for branch $BRANCH to check out"
case `git branch | grep -wF " $BRANCH" | grep "^[* ] $BRANCH"'$'` in
    '')
        checkout -b "$BRANCH" "$PRIOR"
        NEWBRANCH=yes
        if [ -n "$AMEND" ]
        then
            mention "Ignoring requested $AMEND: no prior $BRANCH"
            AMEND=
        fi
        ;;
    '* '*)
        case "$AMEND" in
            '--replace')
                mutter "On prior branch $BRANCH; shall be removed and recreated"
                ;;
            '--amend')
                mutter "Already on branch $BRANCH; preparing to amend it"
                ;;
            *)
                mutter "Already on branch $BRANCH; preparing to extend it"
                ;;
        esac
        ;;
    '  '*)
        case "$AMEND" in
            '--replace')
                mutter "Replacing existing branch $BRANCH (reusing its Change-Id)"
                ;;
            '--amend')
                mutter "Reusing existing branch $BRANCH; preparing to amend it"
                checkout "$BRANCH"
                ;;
            *)
                mutter "Reusing existing branch $BRANCH; preparing to extend it"
                checkout "$BRANCH"
                ;;
        esac
        ;;
esac

# Implement --replace and --amend:
if [ -n "$AMEND" ]
then
    # Suppress --amend or --replace unless we have a prior commit on $BRANCH:
    if git diff --quiet "$BRANCH" "$PRIOR"
    then
        mention "Suppressing requested $AMEND: no prior commit on $BRANCH"
        AMEND=
    else
        # Read last commit's Task-number, if --task hasn't specified it:
        [ -n "$TASK" ] || TASK=`git show --summary $BRANCH | sed -ne '/Task-number:/ s/.*: *//p'`
    fi
fi

OLDBRANCH=
if [ "$AMEND" = '--replace' ]
then
    OLDBRANCH="$BRANCH-old-$PROC"
    checkout "$RELEASE"
    # Rename old branch, recreate fresh:
    run git branch -m "$BRANCH" "$OLDBRANCH"
    checkout -b "$BRANCH" "$PRIOR"
fi

# Even when we do have a prior commit, the headers it reports as
# deleted are not actually deleted as part of that commit; so their
# deletion below shall ensure they're reported in the commit message,
# whether AMENDing or not.  We could filter these when not AMENDing,
# but (doing so would be fiddly and) any restored would then be
# described as deleted in the first commit's message, without
# mentioning that they're restored in the second (albeit any change in
# them shall show up in the diff).

# Make sure any sub-submodules are in their right states:
git submodule update --checkout

# Find a good place to prepare our commit messages
if [ -f .git ]
then GITDIR=`cut -d ' ' -f 2 <.git`
else GITDIR=.git
fi

# Suppress clang-format pre-commit hook and inane post-commit hook:
if [ -e $GITDIR/hooks/pre-commit ]
then
    PRECOMMIT=$GITDIR/hooks/isolated-pre-commit
    mv $GITDIR/hooks/pre-commit $PRECOMMIT

    if [ -e $GITDIR/hooks/post-commit ]
    then
        POSTCOMMIT=$GITDIR/hooks/isolated-post-commit
        mv $GITDIR/hooks/post-commit $POSTCOMMIT

        # But put it back when we're done:
        rehook () {
            mv $PRECOMMIT $GITDIR/hooks/pre-commit;
            mv $POSTCOMMIT $GITDIR/hooks/post-commit;
        }
        trap rehook EXIT
    else
        # But put it back when we're done:
        reclang () {
            mv $PRECOMMIT $GITDIR/hooks/pre-commit;
        }
        trap reclang EXIT
    fi
elif [ -e $GITDIR/hooks/post-commit ]
then
    POSTCOMMIT=$GITDIR/hooks/isolated-post-commit
    mv $GITDIR/hooks/post-commit $POSTCOMMIT

    # But put it back when we're done:
    resane () {
        mv $POSTCOMMIT $GITDIR/hooks/post-commit;
    }
    trap resane EXIT
fi

# consume file ...
# Removes the first-named file, if present, after streaming its contents.
# Follows that with what cat produces for any further arguments.
consume () {
    if [ -f "$1" ]
    then
        cat "$@"
        rm "$1"
    elif [ $# -gt 1 ]
    then
        shift
        cat "$@"
    fi
}

# Note: the loop is in a sub-shell, so can't change script variables.
# It can, however, write to local temp files:
COMMITTED="../.committed-$PROC"
BORED="../.bored-$PROC"
FIXED="../.fixed-$PROC"

# We need to include deleted and new Qt modules;
# and moved ones may involve more than one
modules | consume "$NEWMODS" - | $THERE/modunite | grep . | while read QTMODULE CMakeLists
# For each module, create a commit:
do
    # If --module was specified, limit to modules named by it:
    [ -z "$MODULES" ] || (echo " $MODULES" | grep -iwq "$QTMODULE") || continue
    mutter "Checking for API changes in $QTMODULE ($CMakeLists)"

    mutter "Purging obsolete headers for $QTMODULE"
    # To ensure renames get detected and handled correctly:
    echo $CMakeLists | tr ' ' '\n' | headers "$PRIOR" $QTMODULE | \
        while read h # Update former API headers, remove them if removed:
        do git checkout $UNQUIET "$RELEASE" -- "$h" || git rm $UNQUIET -f -- "$h"
        done 2>&1 | grep -wv "error: pathspec '.*' did not match any"
    mutter "Checking out $RELNAME's API headers for $QTMODULE"
    echo $CMakeLists | tr ' ' '\n' | headers "$RELEASE" $QTMODULE \
        | tr '\n' '\0' | xargs -0r git checkout "$RELEASE" -- 2>&1 \
        | grep -wv "error: pathspec '.*' did not match any"

    if git diff --quiet --cached
    then
        mutter "No changes to $QTMODULE API (not even boring ones)"
        continue
    fi

    mutter "Reverting the boring changes to $QTMODULE"
    run "$THERE/resetboring.py" --disclaim | while read f
    do git checkout $QUIET "$PRIOR" -- "$f" || logged rm -f "$f"
    done 2>&1 | grep -wv "error: pathspec '.*' did not match any"

    if git diff --quiet --cached
    then
        echo "$QTMODULE" >> "$BORED"
        continue
    fi

    # We can only amend (via fixup) if the existing branch has a
    # commit for this Qt module: otherwise, we just create a fresh
    # commit for it.
    FIXUP=`[ "$AMEND" != '--amend' ] || changeid "$QTMODULE" "$BRANCH"`

    # Compose commit message:
    if [ -n "$FIXUP" ]
    then
        echo "$QTMODULE" >> "$FIXED"
        echo "fixup! $QTMODULE: API comparison from $PRIOR to $RELNAME in $GITMODULE"
    else
        if [ -n "$OLDBRANCH" ]
        then
            CHANGEID=`changeid "$QTMODULE" "$OLDBRANCH"`
            # This Qt module might not have shown up before:
            if [ -z "$CHANGEID" ]
            then warn "No prior Change-Id from $BRANCH for $QTMODULE"
            else # But insist on well-formed Change-Id, if present:
                expr "$CHANGEID" : "^I[0-9a-f]\{40\}$" >/dev/null || \
                    die "Bad prior Change-Id ($CHANGEID) from $BRANCH"
            fi
        else CHANGEID=
        fi

        echo "$QTMODULE: API comparison from $PRIOR to $RELNAME in $GITMODULE"
        echo
        git status | grep 'deleted:' | tr '\t' ' '
        git diff | wc | python3 -c 'import sys; \
row = sys.stdin.readline().split(); \
print(); \
print("Excluded {} lines ({} words, {} bytes) of boring changes.".format(*row) \
      if any(int(x) != 0 for x in row) else "Found nothing boring to ignore.")'
        cat <<EOF

NOTE: Please review the appropriate documentation and make sure it is
correct, too!

Note to reviewers: if you know of other changes to API, not covered by
this review, fetch this from Gerrit - see its checkout instructions;
you might want to add -b $BRANCH to those - and you
can add files to the result.  This can be relevant for behavior
changes in the .cpp or, for changes to documented behavior, in .qdoc
files.

Just git checkout $RELEASE each relevant file; you can then git reset
HEAD each and git add -p to select the relevant changes or use git
reset -p HEAD to unstage what git checkout has staged.  Try not to
include extraneous (non-API) changes.  Once you've added what matters,
git commit --amend and push back to gerrit's refs/for/$RELEASE in the
usual way.

Between staging changes from $RELEASE and committing, you can filter
some boring changes by running qtqa/scripts/api-review/resetboring.py
in the top-level directory of your module; this selectively de-stages
things commonly deemed boring in review.  (You'll need the python
dulwich package installed, for this.)  You can git add -p after that,
of course, to restore any changes you think aren't boring.

Remember that the parent of this commit is $PRIOR, not $RELEASE, despite
the review being sent to the latter in Gerrit.
EOF
        [ -z "$CHANGEID$TASK" ] || echo
        [ -z "$TASK" ] || echo "Task-number: $TASK"
        [ -z "$CHANGEID" ] || echo "Change-Id: $CHANGEID"
    fi > "$GITDIR/COMMIT_EDITMSG"
    # The git status in that holds a lock that precludes the git commit;
    # so we can't just pipe the message and use -F - to deliver it.

    mention "Committing change for $QTMODULE"
    run git commit $QUIET -F "$GITDIR/COMMIT_EDITMSG"

    echo "$QTMODULE" >> "$COMMITTED"
done
[ -z "$OLDBRANCH" ] || run git branch -D "$OLDBRANCH"

FIXES="`consume "$FIXED"`"
if [ -n "$FIXES" ]
then
    warn "Doing a rebase to make new changes fix up old ones for each Qt module"
    run git rebase -i --autosquash --autostash "$PRIOR"
fi

COMMITS="`consume "$COMMITTED"`"
BORES="`consume "$BORED"`"

if [ -n "$COMMITS" ]
then
   mention "I recommend you review that what git diff -D reports (now) *is* boring."
   mention "(If any of it isn't, you can git add -p it and git commit --amend, during a rebase.)"
   mention "Then you can: git push gerrit $BRANCH:refs/for/$RELEASE%topic=api-change-review-$RELNAME"
   [ -n "$TASK" ] || warn "Warning: no Task-number: footer specified."
elif [ -n "$BORES" ]
then mention "All the change here looks boring: check with git diff -D"
elif [ -n "$NEWBRANCH" ]
then
    mutter "Clearing away unused branch and restoring $RESTORE"
    git checkout $QUIET "$RESTORE"
    git branch -D "$BRANCH"
    mention "No changes to API"
else
    mention "No new changes to API"
fi