start-local-buildbot-server

#!/usr/bin/env python3
#
# Copyright (C) 2017, 2021 Igalia S.L.
# Copyright (C) 2020 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
import sys
import signal
import os
import argparse
import subprocess
import tempfile
import shutil
import socket
import json
import traceback
import multiprocessing
from time import sleep

buildbot_server_tac = """
import os
from twisted.application import service
from buildbot.master import BuildMaster

basedir = os.path.dirname(os.path.realpath(__file__))
configfile = r'%(config_file)s'

application = service.Application('buildmaster')
BuildMaster(basedir, configfile).setServiceParent(application)
"""

buildbot_worker_tac = """
import os
from buildbot_worker.bot import Worker
from twisted.application import service

basedir = os.path.dirname(os.path.realpath(__file__))
application = service.Application('buildbot-worker')

buildmaster_host = 'localhost'
port = %(server_pb_port)s
workername = '%(worker_name)s'
passwd = 'password'
keepalive = 600

s = Worker(buildmaster_host, port, workername, passwd, basedir, keepalive)
s.setServiceParent(application)
"""


def check_tcp_port_open(address, port):
    s = socket.socket()
    try:
        s.connect((address, port))
        return True
    except ConnectionRefusedError:
        return False


def create_tempdir(tmpdir=None):
    if tmpdir is not None:
        if not os.path.isdir(tmpdir):
            raise ValueError('{} is not a directory'.format(tmpdir))
        return tempfile.mkdtemp(prefix=os.path.join(os.path.abspath(tmpdir), 'tmp'))
    return tempfile.mkdtemp()


def print_if_error_stdout_stderr(cmd, retcode, stdout=None, stderr=None, extramsg=None):
    if retcode != 0:
        if type(cmd) is list:
            cmd = ' '.join(cmd)
        print('WARNING: "{cmd}" returned {retcode} status code'.format(cmd=cmd, retcode=retcode))
        if stdout is not None:
            print('STDOUT:\n' + stdout.decode('utf-8'))
        if stderr is not None:
            print('STDERR:\n' + stderr.decode('utf-8'))
        if extramsg is not None:
            print(extramsg)


def cmd_exists(cmd):
    return any(os.access(os.path.join(path, cmd), os.X_OK)
               for path in os.environ['PATH'].split(os.pathsep))


class BuildbotTestRunner(object):

    def __init__(self, dir_to_copy, subdir_with_configuration):
        self._dir_to_copy = os.path.abspath(os.path.realpath(dir_to_copy))
        self._subdir_with_configuration = subdir_with_configuration
        self._configdir = os.path.join(self._dir_to_copy, self._subdir_with_configuration)
        if not os.path.isdir(self._configdir):
            raise RuntimeError('The configdir {} is not a directory'.format(self._configdir))
        if not os.path.isfile(os.path.join(self._configdir, 'config.json')):
            raise RuntimeError('The configdir {} does not contain a config.json file'.format(self._configdir))

        number_config_files = 0
        for file in os.listdir(self._configdir):
            if file.endswith('.cfg'):
                self._server_config_file_name = file
                number_config_files += 1
        if number_config_files == 0:
            raise RuntimeError('The configdir {} does not contain a .cfg file'.format(self._configdir))
        if number_config_files != 1:
            raise RuntimeError('The configdir {} has more than one .cfg file'.format(self._configdir))

        self._server_http_port, self._server_pb_port = self._get_config_tcp_ports()

    def _get_config_tcp_ports(self):

        def get_numeric_from_port_line(line):
            port_string = str(line)
            if 'tcp:' in port_string:
                port_string = port_string.split('tcp:', 1)[1]
                if ':' in port_string:
                    port_string = port_string.split(':', 1)[0]
            if port_string.isnumeric():
                return int((port_string))
            return None

        pb_port = None
        http_port = None
        with open(os.path.join(self._configdir, self._server_config_file_name)) as f:
            for line in f:
                if '=' in line:
                    if 'protocols' in line and 'pb' in line and 'port' in line:
                        pb = eval(line.split('=', 1)[1])
                        pb_port = get_numeric_from_port_line(pb['pb']['port'])
                    if 'www' in line and 'port' in line:
                        http = eval(line.split('=', 1)[1])
                        http_port = get_numeric_from_port_line(http['port'])
        if pb_port is None:
            print("Unable to detect pb port from config. Using default")
            pb_port = 17000
        if http_port is None:
            print("Unable to detect http port from config. Using default")
            http_port = 8010
        return http_port, pb_port

    def start(self, basetempdir=None, no_clean=False, number_workers=1, use_system_version=False):
        try:
            self._base_workdir_temp = os.path.abspath(os.path.realpath(create_tempdir(basetempdir)))
            if self._base_workdir_temp.startswith(self._configdir):
                raise ValueError('The temporal working directory {} cant be located inside configdir {}'.format(self._base_workdir_temp, self._configdir))
            if not use_system_version:
                self._setup_virtualenv()
            if not (cmd_exists('twistd') and cmd_exists('buildbot')):
                raise RuntimeError('Buildbot is not installed.')
            self._setup_server_workdir()
            server_runner = multiprocessing.Process(target=self._start_server)
            server_runner.start()
            self._wait_for_server_ready()
            if number_workers == 0:
                print(' - To manually attach a build worker use this info:\n'
                      + '     TCP port for the worker-to-server connection: {}\n'.format(self._server_pb_port)
                      + '     worker-id: the one defined at {}\n'.format(os.path.join(self._server_wordir, 'passwords.json'))
                      + '     password:  password\n')
            elif number_workers == 1:
                worker = 'local-worker'
                worker_runner = multiprocessing.Process(target=self._start_worker, args=(worker,))
                worker_runner.start()
                print(' - Worker started!.\n'
                      + '     Check the log for at {}/local-worker/worker.log\n'.format(self._base_workdir_temp)
                      + '     tail -f {}/local-worker/worker.log\n'.format(self._base_workdir_temp))
                worker_runner.join()
            else:
                worker_runners = []
                for worker in self._get_list_workers():
                    worker_runner = multiprocessing.Process(target=self._start_worker, args=(worker,))
                    worker_runner.start()
                    worker_runners.append(worker_runner)
                print(' - Workers started!.\n'
                      + '     Check the log for each one at {}/WORKERNAMEID/worker.log\n'.format(self._base_workdir_temp)
                      + '     tail -f {}/*/worker.log\n'.format(self._base_workdir_temp))
                for worker_runner in worker_runners:
                    worker_runner.join()
            server_runner.join()
        except KeyboardInterrupt:
            pass  # no print the exception
        except:
            traceback.print_exc()
        finally:
            try:
                # The children may exit between the check and the kill call.
                # Ignore any exception raised here.
                for c in multiprocessing.active_children():
                    # Send the signal to the whole process group.
                    # Otherwise some twistd sub-childs can remain alive.
                    os.killpg(os.getpgid(c.pid), signal.SIGKILL)
            except:
                pass
            if not no_clean:
                self._clean()
            sys.exit(0)

    def _wait_for_server_ready(self):
        server_ready_check_counter = 0
        while True:
            if os.path.isfile(self._server_ready_fd):
                return
            if server_ready_check_counter > 60:
                print('ERROR: buildbot server has not started after waiting 60 seconds for it.')

                if os.path.isfile(self._server_log):
                    print('The server.log file contains the following:')
                    with open(self._server_log, 'r') as f:
                        print(f.read())
                else:
                    print('There is no server.log on the directory. That means a general failure starting buildbot.')

                raise RuntimeError('buildbot server has not started after waiting 60 seconds for it.')
            sleep(1)
            server_ready_check_counter += 1

    def _create_mock_worker_passwords_dict(self):
        with open(os.path.join(self._server_wordir, 'config.json'), 'r') as config_json:
            config_dict = json.load(config_json)
        result = dict([(worker['name'], 'password') for worker in config_dict['workers']])
        return result

    def _setup_server_workdir(self):
        self._server_wordir = os.path.join(self._base_workdir_temp, os.path.basename(self._dir_to_copy))
        assert(not os.path.exists(self._server_wordir))
        print('Copying files from {} to {} ...'.format(self._dir_to_copy, self._server_wordir))
        shutil.copytree(self._dir_to_copy, self._server_wordir)
        self._server_wordir = os.path.join(self._server_wordir, self._subdir_with_configuration)
        assert(os.path.isdir(self._server_wordir))
        self._server_log = os.path.join(self._server_wordir, 'server.log')
        self._server_ready_fd = os.path.join(self._server_wordir, '.server-is-ready')
        print('Generating buildbot files at {} ...'.format(self._server_wordir))
        with open(os.path.join(self._server_wordir, 'buildbot.tac'), 'w') as f:
            f.write(buildbot_server_tac % {'config_file': self._server_config_file_name})
        with open(os.path.join(self._server_wordir, 'passwords.json'), 'w') as passwords_file:
            passwords_file.write(json.dumps(self._create_mock_worker_passwords_dict(), indent=4, sort_keys=True))

    def _setup_virtualenv(self):
        if cmd_exists('virtualenv'):
            print('Setting up virtualenv at {} ... '.format(self._base_workdir_temp))
            virtualenv_cmd = ['virtualenv', '-p', 'python3', 'venv']
            virtualenv_process = subprocess.Popen(virtualenv_cmd, cwd=self._base_workdir_temp,
                                                  stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            stdout, stderr = virtualenv_process.communicate()
            print_if_error_stdout_stderr(virtualenv_cmd, virtualenv_process.returncode, stdout, stderr)
            virtualenv_bindir = os.path.join(self._base_workdir_temp, 'venv', 'bin')
            virtualenv_pip = os.path.join(virtualenv_bindir, 'pip')
            if not os.access(virtualenv_pip, os.X_OK):
                print('Something went wrong setting up virtualenv'
                      'Trying to continue using the system version of buildbot')
                return
            print('Setting up buildbot dependencies on the virtualenv ... ')
            # The idea is to install the very same version of buildbot and its
            # dependencies than the ones used for running https://build.webkit.org/about
            buildbot_version = '2.10.5'
            pip_cmd = [virtualenv_pip, 'install',
                       'buildbot=={}'.format(buildbot_version),
                       'buildbot-console-view=={}'.format(buildbot_version),
                       'buildbot-grid-view=={}'.format(buildbot_version),
                       'buildbot-waterfall-view=={}'.format(buildbot_version),
                       'buildbot-worker=={}'.format(buildbot_version),
                       'buildbot-www=={}'.format(buildbot_version),
                       'lz4==1.1.0',
                       'mock==4',
                       'rapidfuzz==2.11.1',
                       'requests==2.21.0',
                       'twisted==21.2.0']
            pip_process = subprocess.Popen(pip_cmd, cwd=self._base_workdir_temp,
                                           stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            stdout, stderr = pip_process.communicate()
            print_if_error_stdout_stderr(pip_cmd, pip_process.returncode, stdout, stderr)
            os.environ['PATH'] = virtualenv_bindir + ':' + os.environ['PATH']
            return
        print('WARNING: virtualenv not installed. '
              'Trying to continue using the system version of buildbot')

    def _upgrade_db_needed(self):
        with open(self._server_log) as f:
            for l in f:
                if 'upgrade the database' in l:
                    return True
        return False

    def _start_server(self):
        # This is started via multiprocessing. We set a new process group here
        # to be able to reliably kill this subprocess and all of its child on clean.
        os.setsid()
        dbupgraded = False
        retry = True
        if check_tcp_port_open('localhost', self._server_pb_port):
            print('ERROR: There is some process already listening in port {}'.format(self._server_pb_port))
            return 1
        while retry:
            retry = False
            print('Starting the buildbot server process ...')
            twistd_cmd = ['twistd', '-l', self._server_log, '-noy', 'buildbot.tac']
            twistd_server_process = subprocess.Popen(twistd_cmd, cwd=self._server_wordir,
                                                     stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            while twistd_server_process.poll() is None:
                if check_tcp_port_open('localhost', self._server_pb_port):
                    print('Test buildbot server ready!\n'
                          + 'Press CTRL-C to stop\n\n'
                          + ' - See buildbot server log:\n'
                          + '     tail -f {}\n\n'.format(self._server_log)
                          + ' - Open a browser to:\n'
                          + '     http://localhost:{}\n'.format(self._server_http_port))
                    with open(self._server_ready_fd, 'w') as f:
                        f.write('ready')
                    twistd_server_process.wait()
                    return 0
                sleep(1)
            stdout, stderr = twistd_server_process.communicate()
            if twistd_server_process.returncode == 0 and self._upgrade_db_needed() and not dbupgraded:
                retry = True
                dbupgraded = True
                print('Upgrading the database ...')
                upgrade_cmd = ['buildbot', 'upgrade-master', self._server_wordir]
                upgrade_process = subprocess.Popen(upgrade_cmd, cwd=self._server_wordir,
                                                   stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                stdout, stderr = upgrade_process.communicate()
                print_if_error_stdout_stderr(upgrade_cmd, upgrade_process.returncode, stdout, stderr)
            else:
                print_if_error_stdout_stderr(twistd_cmd, twistd_server_process.returncode, stdout, stderr,
                                             'Check the log at {}'.format(self._server_log))
        return twistd_server_process.returncode

    def _get_list_workers(self):
        password_list = os.path.join(self._server_wordir, 'passwords.json')
        with open(password_list) as f:
            passwords = json.load(f)
        list_workers = []
        for worker in passwords:
            list_workers.append(str(worker))
        return list_workers

    def _start_worker(self, worker):
        # This is started via multiprocessing. We set a new process group here
        # to be able to reliably kill this subprocess and all of its child on clean.
        os.setsid()
        worker_workdir = os.path.join(self._base_workdir_temp, worker)
        os.mkdir(worker_workdir)
        with open(os.path.join(worker_workdir, 'buildbot.tac'), 'w') as f:
            f.write(buildbot_worker_tac % {'worker_name': worker, 'server_pb_port': self._server_pb_port})
        twistd_cmd = ['twistd', '-l', 'worker.log', '-noy', 'buildbot.tac']
        twistd_worker_process = subprocess.Popen(twistd_cmd, cwd=worker_workdir,
                                                 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        stdout, stderr = twistd_worker_process.communicate()
        print_if_error_stdout_stderr(twistd_cmd, twistd_worker_process.returncode, stdout, stderr,
                                     'Check the log at {}'.format(os.path.join(worker_workdir, 'worker.log')))
        return twistd_worker_process.returncode

    def _clean(self):
        if os.path.isdir(self._base_workdir_temp):
            print('\n\nCleaning {} ... \n'.format(self._base_workdir_temp))
            # shutil.rmtree can fail if we hold an open file descriptor on temp_dir
            # (which is very likely when cleaning) or if temp_dir is a NFS mount.
            # Use rm instead that always works.
            rm = subprocess.Popen(['rm', '-fr', self._base_workdir_temp])
            rm.wait()


if __name__ == '__main__':
    if sys.version_info < (3, 9):  # noqa: UP036
        print('ERROR: Minimum supported Python version for this code is Python 3.9')
        sys.exit(1)

    parser = argparse.ArgumentParser()
    configuration = parser.add_mutually_exclusive_group(required=True)
    configuration.add_argument('--ews', action='https://github.com/store_const', const='ews', dest='configuration',
                               help='Simulate the EWS buildbot server (ews-build.webkit.org)')
    configuration.add_argument('--post-commit', action='https://github.com/store_const', const='post_commit', dest='configuration',
                               help='Simulate the post-commit buildbot server (build.webkit.org)')
    configuration.add_argument('--config-dir', default=None, dest='configdir', type=str,
                               help='Specify the directory that contains the buildbot config files')
    parser.add_argument('--base-temp-dir', help='Path where the temporal working directory will be created. '
                        'Note: To trigger test builds with the test workers you need enough free space on that path.',
                        dest='basetempdir', default=None, type=str)
    parser.add_argument('--no-clean', help='Do not clean the temporal working dir on exit.',
                        dest='no_clean', action='https://github.com/store_true')
    workers = parser.add_mutually_exclusive_group(required=False)
    workers.add_argument('--no-workers', help='Do not start any workers.',
                        dest='number_workers', action='https://github.com/store_const', const=0)
    workers.add_argument('--all-workers', help='Instead of starting only one worker that round-robins between all the queues, '
                                               'start multiple parallel workers as defined on the server config.',
                        dest='number_workers', action='https://github.com/store_const', const=float('inf'))
    parser.add_argument('--use-system-version', help='Instead of setting up a virtualenv with the buildbot version '
                        'used by build.webkit.org, use the buildbot version installed on this system.',
                        dest='use_system_version', action='https://github.com/store_true')
    args = parser.parse_args()

    if args.configuration == "ews":
        dir_to_copy = os.path.dirname(__file__)
        subdir_with_configuration = "ews-build"
    elif args.configuration == "post_commit":
        dir_to_copy = os.path.dirname(__file__)
        subdir_with_configuration = "build-webkit-org"
    else:
        dir_to_copy = args.configdir
        subdir_with_configuration = ""

    if args.number_workers is None:
        args.number_workers = 1

    buildbot_test_runner = BuildbotTestRunner(dir_to_copy, subdir_with_configuration)
    buildbot_test_runner.start(args.basetempdir, args.no_clean, args.number_workers, args.use_system_version)