path: root/tools/webhook_installer/installer.py

# Copyright (C) 2023 The Qt Company Ltd.
# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only

"""Use a passed repo list, or query gerrit for a list of repos, then iterate the list and
# query for the webhooks.config file on refs/meta/config for each repo.
# If the file exists and has a section titled `remote "qt-cherry-pick-bot"`,
# Then update the url of the section to point to the correct url for staging or production.
# If the file does not exist, create it using the template below.
# If the file exists but does not have a section titled `remote "qt-cherry-pick-bot"`,
# Then append the template below to the end of the file.
# Push the changes to gerrit. Publish the change and submit it."""

import sys
import json
import logging
import re
import urllib.request
import urllib.parse
import urllib.error
import base64
import argparse
from os import environ as envs
from urllib.parse import unquote, quote


# Parse arguments
parser = argparse.ArgumentParser(description="Add or update webhooks.config for qt-cherry-pick-bot")
parser.add_argument("--simulate", action="store_true", help="Perform a dry run but make no changes")
parser.add_argument("repos", nargs="*", help="List of repos to update. Defaults to all repos with"
                    " existing webhooks.config", default=[])
parser.add_argument("--add", action="store_true", help="Add webhooks.config to repos that don't"
                    " have it. Ignored if --repos is not specified")
parser.add_argument("--production", action="store_true", help="Update webhooks.config for"
                    " production, defaults to staging if not set")
parser.add_argument("--update-rootconfig", action="store_true", help="Update webhooks.config for the"
                    " 'All-Projects' repository with specific jira-closer-bot and "
                    "gerrit-webhooks-forwarder settings. Other repo arguments are ignored.")
parser.add_argument("--updateConfig", action="store_true", help="Update the entire"
                    " qt-cherry-pick-bot section if it differs from the template")
args = parser.parse_args()


gerrit_url = envs.get("GERRIT_URL", "https://codereview.qt-project.org")
CHERRY_PICK_PROD_URL = "https://qt-cherry-pick-bot.herokuapp.com/gerrit-events"
CHERRY_PICK_STAGE_URL = "https://qt-cherry-pick-bot-staging.herokuapp.com/gerrit-events"
# Gerrit user must have the following permissions:
# - Create change
# - Push
# - Submit
# - Approver
gerrit_username = envs.get("GERRIT_USERNAME", "")
gerrit_password = envs.get("GERRIT_PASSWORD", "")
gerrit_repos_with_webhooks = []
gerrit_repos_with_webhooks_to_update = []
gerrit_repos_with_webhooks_to_update_failed = []
gerrit_repos_with_webhooks_to_update_succeeded = []
gerrit_repos_without_webhooks = []
gerrit_repos_with_webhooks_not_updated = []


# Configuration for "All-Projects" repository when --update-rootconfig is used
ROOT_CONFIG_ALL_PROJECTS_PROD = """[remote "jira-closer-bot"]
  url = https://qt-cherry-pick-bot.herokuapp.com/jiracloser
  maxTries = 10
  event = change-merged

[remote "gerrit-webhooks-forwarder"]
  url = http://10.212.0.77:8080
  maxTries = 10"""

ROOT_CONFIG_ALL_PROJECTS_STAGING = """[remote "jira-closer-bot"]
  url = https://qt-cherry-pick-bot-staging.herokuapp.com/jiracloser
  maxTries = 10
  event = change-merged

# [remote "gerrit-webhooks-forwarder"]
#   url = http://10.212.0.77:8080
#   maxTries = 10"""


CONFIG_SECTION_HEADER = '[remote "qt-cherry-pick-bot"]'
CONFIG_SECTION_HEADER_ESC = r'\[remote "qt-cherry-pick-bot"\]' # Escaped for regex

# Determine target URL and template based on args
TARGET_URL = CHERRY_PICK_PROD_URL if args.production else CHERRY_PICK_STAGE_URL
WEBHOOK_TEMPLATE = f"""{CONFIG_SECTION_HEADER}
  url = {TARGET_URL}
  maxTries = 10
  event = patchset-created
  event = change-merged
  event = change-abandoned
  event = change-staged
  event = change-unstaged
  event = comment-added
  event = change-deferred
  event = change-restored
  event = change-integration-pass
  event = change-integration-fail
"""

gerrit_authstring = base64.b64encode(f"{gerrit_username}:{gerrit_password}"
                                     .encode("utf-8")).decode("utf-8")
gerrit_authstring = f"Basic {gerrit_authstring}"


# Logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)


# Functions
def get_gerrit_repos():
    """ Get a list of all repos from gerrit."""
    url = f"{gerrit_url}/a/projects/"
    request = urllib.request.Request(url)
    request.add_header("Authorization", gerrit_authstring)
    try:
        with urllib.request.urlopen(request) as response:
            if response.status == 200:
                content = json.loads(response.read().decode("utf-8").replace(")]}'", ""))
                return [content[repo]["id"] for repo in content.keys()
                        if content[repo]["state"] == "ACTIVE"]
            else:
                logger.error("Failed to get list of repos from gerrit: %s %s",
                             response.status, response.reason)
                sys.exit(1)
    except urllib.error.URLError as err:
        logger.error("Failed to connect to gerrit: %s", err.reason)
        sys.exit(1)


def get_gerrit_repo_webhooks(repo):
    """ Get the webhooks.config file from the refs/meta/config branch
    It is returned as a base64 encoded string. Decode before returning."""
    url = f"{gerrit_url}/a/projects/{repo}"
    print("trying", url)
    request = urllib.request.Request(url)
    request.add_header("Authorization", gerrit_authstring)

    try:
        with urllib.request.urlopen(request) as response:
            if response.status == 200:
                logger.info("Repo %s exists", urllib.parse.unquote(repo))
    except urllib.error.URLError as err:
        if err.status == 404:
            logger.error("Repo %s does not exist", urllib.parse.unquote(repo))
            sys.exit(1)
        logger.error("Failed to connect to gerrit: %s %s", err.status, err.reason)
        sys.exit(1)

    url = f"{gerrit_url}/a/projects/{repo}/branches/refs%2Fmeta%2Fconfig" \
          "/files/webhooks.config/content"
    request = urllib.request.Request(url)
    request.add_header("Authorization", gerrit_authstring)

    try:
        with urllib.request.urlopen(request) as response:
            if response.status == 200:
                logger.info("Got webhooks.config file from repo %s", urllib.parse.unquote(repo))
                return base64.b64decode(response.read()).decode("utf-8")
            else:
                logger.info("No webhooks.config file in repo %s: %s %s",
                            urllib.parse.unquote(repo), response.status, response.reason)
                gerrit_repos_without_webhooks.append(repo)
                return None
    except urllib.error.URLError as err:
        if err.status == 404:
            logger.info("No webhooks.config file in repo %s", urllib.parse.unquote(repo))
            gerrit_repos_without_webhooks.append(repo)
            return None
        logger.error("Failed to connect to gerrit: %s %s", err.status, err.reason)
        sys.exit(1)


def update_gerrit_repo_webhooks(repo, webhooks_config):
    """ Create a new change request with the webhooks.config file as the only file to be changed.
    The change request will be created with the following commit message:
    "Update webhook url for qt-cherry-pick-bot to [staging|production]"
    The new/modified webhooks.config file will be pushed to the change request.
    The change edit will be published.
    The change request will be self-reviewed and submitted.
    """

    # Create a new change request
    change_input = {"project": unquote(repo),
                    "branch": "refs/meta/config",
                    "subject": "Update webhook url for qt-cherry-pick-bot to "
                               f"{'production' if args.production else 'staging'}",
                    "is_work_in_progress": False
                    }
    if args.simulate:
        logger.info("Simulating change request creation for repo %s", unquote(repo))
        logger.info("Change request input: %s", change_input)
        change_request_id = "abcd1234"
    else:
        url = f"{gerrit_url}/a/changes/"
        data = json.dumps(change_input).encode("utf-8")
        request = urllib.request.Request(url, data=data, method="POST")
        request.add_header("Authorization", gerrit_authstring)
        request.add_header("Content-Type", "application/json")
        try:
            with urllib.request.urlopen(request, timeout=60) as response:
                if response.status == 201:
                    change_request = json.loads(response.read().decode("utf-8").replace(")]}'", ""))
                    change_request_id = change_request["id"]
                    logger.info("Created change request %s for repo %s",
                                change_request_id, urllib.parse.unquote(repo))
                else:
                    logger.error("Failed to create change request for repo %s",
                                 urllib.parse.unquote(repo))
                    logger.error("%s: %s", response.status, response.read().decode("utf-8"))
                    gerrit_repos_with_webhooks_to_update_failed.append(repo)
                    return None
        except urllib.error.URLError as err:
            logger.error("Failed to connect to gerrit: %s", err.reason)
            sys.exit(1)

    # Upload the webhooks.config file to the change request
    if args.simulate:
        logger.info("Simulating upload of webhooks.config to change request %s for repo %s",
                change_request_id, unquote(repo))
        logger.info("Webhooks.config: %s", webhooks_config)
    else:
        url = f"{gerrit_url}/a/changes/{change_request_id}/edit/webhooks.config"
        data = webhooks_config.encode("utf-8")
        request = urllib.request.Request(url, data=data, method="PUT")
        request.add_header("Authorization", gerrit_authstring)
        request.add_header("Content-Type", "application/octet-stream")
        try:
            with urllib.request.urlopen(request) as response:
                if response.status == 204:
                    logger.info("Uploaded webhooks.config to change request %s for repo %s",
                                change_request_id, urllib.parse.unquote(repo))
                elif response.status == 409:
                    logger.info("Webhooks.config is already up-to-date for %s",
                                urllib.parse.unquote(repo))
                    gerrit_repos_with_webhooks_not_updated.append(repo)
                    return None
                else:
                    logger.error("Failed to upload webhooks.config to change request"
                                 " %s for repo %s", change_request_id, urllib.parse.unquote(repo))
                    logger.error("%s: %s", response.status, response.read().decode("utf-8"))
                    gerrit_repos_with_webhooks_to_update_failed.append(repo)
                    return None
        except urllib.error.URLError as err:
            logger.error("Failed to connect to gerrit: %s", err.reason)
            sys.exit(1)

    # Publish the change request
    if args.simulate:
        logger.info("Simulating publishing change request " + change_request_id
                    + " for repo " + unquote(repo))
    else:
        url = f"{gerrit_url}/a/changes/{change_request_id}/edit:publish"
        request = urllib.request.Request(url, method="POST")
        request.add_header("Authorization", gerrit_authstring)
        try:
            with urllib.request.urlopen(request) as response:
                if response.status == 204:
                    logger.info("Published change request %s for repo %s",
                                change_request_id, urllib.parse.unquote(repo))
                else:
                    logger.error("Failed to publish change request %s for repo %s",
                                 change_request_id, urllib.parse.unquote(repo))
                    logger.error("%s: %s", response.status, response.read().decode("utf-8"))
                    gerrit_repos_with_webhooks_to_update_failed.append(repo)
                    return None
        except urllib.error.URLError as err:
            logger.error("Failed to connect to gerrit: %s", err.reason)
            sys.exit(1)

    # Self-review the change request
    if args.simulate:
        logger.info("Simulating self-reviewing change request " + change_request_id
                    + " for repo " + unquote(repo))
    else:
        url = f"{gerrit_url}/a/changes/{change_request_id}/revisions/current/review"
        data = json.dumps({"labels": {"Code-Review": 2}}).encode("utf-8")
        request = urllib.request.Request(url, data=data, method="POST")
        request.add_header("Authorization", gerrit_authstring)
        request.add_header("Content-Type", "application/json")
        try:
            with urllib.request.urlopen(request) as response:
                if response.status == 200:
                    logger.info("Self-reviewed change request %s for repo %s",
                                change_request_id, urllib.parse.unquote(repo))
                else:
                    logger.error("Failed to self-review change request %s for repo %s",
                                 change_request_id, urllib.parse.unquote(repo))
                    logger.error("%s: %s", response.status, response.read().decode("utf-8"))
                    gerrit_repos_with_webhooks_to_update_failed.append(repo)
                    return None
        except urllib.error.URLError as err:
            logger.error("Failed to connect to gerrit: %s", err.reason)
            sys.exit(1)

    # Submit the change request
    if args.simulate:
        logger.info("Simulating submitting change request " + change_request_id
                    + " for repo " + unquote(repo))
        gerrit_repos_with_webhooks_to_update_succeeded.append(repo)
    else:
        url = f"{gerrit_url}/a/changes/{change_request_id}/submit"
        data = None
        request = urllib.request.Request(url, data=data, method="POST")
        request.add_header("Authorization", gerrit_authstring)
        try:
            with urllib.request.urlopen(request) as response:
                if response.status == 200:
                    logger.info("Submitted change request %s for repo %s",
                                change_request_id, urllib.parse.unquote(repo))
                    gerrit_repos_with_webhooks_to_update_succeeded.append(repo)
                else:
                    logger.error("Failed to submit change request %s for repo %s",
                                 change_request_id, urllib.parse.unquote(repo))
                    logger.error("%s: %s", response.status, response.read().decode("utf-8"))
                    gerrit_repos_with_webhooks_to_update_failed.append(repo)
                    return None
        except urllib.error.URLError as err:
            logger.error("Failed to connect to gerrit: %s", err.reason)
            sys.exit(1)

    return True


def main():
    if args.update_rootconfig:
        repo_name = "All-Projects"
        encoded_repo_name = quote(repo_name, safe='')
        config_content_root: str
        env_type: str

        if args.production:
            config_content_root = ROOT_CONFIG_ALL_PROJECTS_PROD
            env_type = "production"
        else:
            config_content_root = ROOT_CONFIG_ALL_PROJECTS_STAGING
            env_type = "staging"

        logger.info(f"Processing --update-rootconfig for '{repo_name}' with {env_type} settings.")
        update_gerrit_repo_webhooks(encoded_repo_name, config_content_root)
    else:
        if len(args.repos) > 0:
            gerrit_repos = [quote(repo, safe='') for repo in args.repos]
        else:
            gerrit_repos = get_gerrit_repos()

        # Regex to find the section header and capture everything until the next section or EOF
        # DOTALL allows . to match newline, non-greedy .*? ensures we stop at the first \n[ or EOF
        section_re = re.compile(rf"({CONFIG_SECTION_HEADER_ESC}(?:.|\n)*?(?=\n\[|$))", re.DOTALL)
        # Regex to find the URL line within the section
        url_re = re.compile(r"^\s*url\s*=\s*(.*)", re.MULTILINE)

        for repo in gerrit_repos:
            needs_update = False
            new_webhooks_config = None
            webhooks_config = get_gerrit_repo_webhooks(repo)

            if webhooks_config:
                section_match = section_re.search(webhooks_config)
                if section_match:
                    existing_section = section_match.group(1).strip()
                    if args.updateConfig:
                        # Compare entire section with template
                        if existing_section != WEBHOOK_TEMPLATE.strip():
                            logger.info("Config section differs from template for repo %s", unquote(repo))
                            new_webhooks_config = section_re.sub(WEBHOOK_TEMPLATE, webhooks_config, count=1)
                            needs_update = True
                        else:
                            logger.info("Config section already matches template for repo %s", unquote(repo))
                            gerrit_repos_with_webhooks_not_updated.append(repo)
                    else:
                        # Only update the URL line within the existing section if --updateConfig is not set
                        url_match = url_re.search(existing_section)
                        if url_match:
                            existing_url = url_match.group(1).strip()
                            if existing_url != TARGET_URL:
                                logger.info("Webhook URL needs update for repo %s", unquote(repo))
                                # Replace only the URL line within the original config
                                new_url_line = f"  url = {TARGET_URL}"
                                # We need to be careful to replace the URL only within the matched section
                                # One way is to replace within existing_section and then replace existing_section in webhooks_config
                                updated_section = url_re.sub(new_url_line, existing_section, count=1)
                                new_webhooks_config = webhooks_config.replace(existing_section, updated_section, 1)
                                needs_update = True
                            else:
                                logger.info("Webhook URL already up-to-date for repo %s", unquote(repo))
                                gerrit_repos_with_webhooks_not_updated.append(repo)
                        else:
                            # This case implies the section exists but has no URL, which is unlikely for this template
                            # but if it happens, and --updateConfig is false, we might want to add the URL or log an error.
                            # For now, let's assume if --updateConfig is false, we only modify existing URLs.
                            # If --updateConfig was true, the whole section would be replaced.
                            logger.error("Could not find URL line in existing section for repo %s, and --updateConfig is false.", unquote(repo))
                            gerrit_repos_with_webhooks_to_update_failed.append(repo)
                elif args.add:
                    # Section header not found, but file exists and --add is specified
                    logger.info("Adding webhook section to existing config for repo %s", unquote(repo))
                    # Append template, ensuring a newline separates it from existing content if webhooks_config is not empty
                    separator = "\n" if webhooks_config.strip() else ""
                    new_webhooks_config = webhooks_config + separator + WEBHOOK_TEMPLATE
                    needs_update = True
                else:
                    # Section header not found, file exists, --add not specified
                    logger.info("No %s section in config for %s and --add not specified.",
                                CONFIG_SECTION_HEADER, unquote(repo))
                    gerrit_repos_with_webhooks_not_updated.append(repo) # Treat as not updated
            elif args.add:
                # webhooks.config file does not exist and --add is specified
                logger.info("Creating webhooks.config for repo %s", unquote(repo))
                new_webhooks_config = WEBHOOK_TEMPLATE
                needs_update = True
            else:
                # webhooks.config file does not exist, --add not specified
                logger.info("No webhooks.config for %s and --add not specified.", unquote(repo))
                gerrit_repos_without_webhooks.append(repo) # Keep track of repos without webhooks

            if needs_update and new_webhooks_config is not None:
                gerrit_repos_with_webhooks.append(repo) # Mark as attempted
                if update_gerrit_repo_webhooks(repo, new_webhooks_config):
                    logger.info("Successfully updated/added webhook config for repo %s", unquote(repo))
                else:
                    # update_gerrit_repo_webhooks logs errors and adds to failed list
                    logger.error("Failed to update/add webhook config for repo %s", unquote(repo))
            elif needs_update and new_webhooks_config is None:
                 # Should not happen if logic is correct, but good to catch
                 logger.error("Internal error: Update flagged but no new config generated for %s", unquote(repo))
                 gerrit_repos_with_webhooks_to_update_failed.append(repo)

    logger.info("Summary:")
    logger.info("Succeeded: %s", str([unquote(repo) for repo in gerrit_repos_with_webhooks_to_update_succeeded]))
    logger.info("Repos with webhooks that failed to update: %s",
                str([unquote(repo) for repo in gerrit_repos_with_webhooks_to_update_failed]))
    logger.info("Repos with webhooks that were not updated: %s",
                str([unquote(repo) for repo in gerrit_repos_with_webhooks_not_updated]))
    logger.info("Repos without webhooks: %s",
                str([unquote(repo) for repo in gerrit_repos_without_webhooks]))
    logger.info("Repos with webhooks that were not updated: %s",
                str([unquote(repo) for repo in gerrit_repos_with_webhooks_not_updated]))


if __name__ == "__main__":
    main()