generate-bundle

#!/usr/bin/env python3
#
# Copyright (C) 2018, 2020 Igalia S.L.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
#    list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import argparse
import base64
import datetime
import hashlib
import json
import logging
import os
from pathlib import PurePath
import shutil
import subprocess
import sys
import tarfile
import tempfile
import zipfile

top_level_directory = os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..'))
sys.path.insert(0, os.path.join(top_level_directory, 'Tools', 'flatpak'))
sys.path.insert(0, os.path.join(top_level_directory, 'Tools', 'jhbuild'))
sys.path.insert(0, os.path.join(top_level_directory, 'Tools', 'Scripts', 'webkitpy'))
import jhbuildutils
import flatpakutils
from binary_bundling.ldd import SharedObjectResolver
from binary_bundling.bundle import BinaryBundler

INSTALL_DEPS_SCRIPT_TEMPLATE = """\
#!/bin/bash
set -eu -o pipefail

REQUIREDPACKAGES="%(packages_needed)s"

if ! which apt-get >/dev/null; then
    echo "This script only supports apt-get based distributions like Debian or Ubuntu."
    exit 1
fi

# Calling dpkg-query is slow, so call it only once and cache the results
TMPCHECKPACKAGES="$(mktemp)"
dpkg-query --show --showformat='${binary:Package} ${db:Status-Status}\\n' > "${TMPCHECKPACKAGES}"
TOINSTALL=""
for PACKAGE in ${REQUIREDPACKAGES}; do
    if ! grep -qxF "${PACKAGE} installed" "${TMPCHECKPACKAGES}"; then
        TOINSTALL="${TOINSTALL} ${PACKAGE}"
    fi
done
rm -f "${TMPCHECKPACKAGES}"

if [[ -z "${TOINSTALL}" ]]; then
    echo "All required dependencies are already installed"
else
    echo "Need to install the following extra packages: ${TOINSTALL}"
    [[ ${#} -gt 0 ]] && [[ "${1}" == "--printonly" ]] && exit 0
    SUDO=""
    [[ ${UID} -ne 0 ]] && SUDO="sudo"
    AUTOINSTALL=""
    if [[ ${#} -gt 0 ]] && [[ "${1}" == "--autoinstall" ]]; then
        AUTOINSTALL="-y"
        export DEBIAN_FRONTEND="noninteractive"
        [[ ${UID} -ne 0 ]] && SUDO="sudo --preserve-env=DEBIAN_FRONTEND"
        ${SUDO} apt-get update
    fi
    set -x
    ${SUDO} apt-get install --no-install-recommends ${AUTOINSTALL} ${TOINSTALL}
fi
"""

FONT_CONF_FILE = """\
<?xml version="1.0"?>
<!DOCTYPE fontconfig SYSTEM "fonts.dtd">
<fontconfig>
    <its:rules xmlns:its="http://www.w3.org/2005/11/its" version="1.0">
        <its:translateRule translate="no" selector="/fontconfig/*[not(self::description)]"/>
    </its:rules>

    <description>Default configuration file</description>

<!-- Font directory list, relative to XDG_DATA_HOME -->
    <dir prefix="xdg">fonts/fonts</dir>

<!-- Load system customization files, relative to previous dir -->
    <include ignore_missing="yes">conf.d</include>

<!-- Font cache directory list, relative to XDG_CACHE_HOME -->
    <cachedir prefix="xdg">fontconfig</cachedir>

<!-- Include also the system font config if is on the standard dir -->
    <include ignore_missing="yes">/etc/fonts/fonts.conf</include>

<!-- Accept deprecated 'mono' alias, replacing it with 'monospace' -->
    <match target="pattern">
        <test qual="any" name="family">
            <string>mono</string>
        </test>
        <edit name="family" mode="assign" binding="same">
            <string>monospace</string>
        </edit>
    </match>

<!-- Accept alternate 'sans serif' spelling, replacing it with 'sans-serif' -->
    <match target="pattern">
        <test qual="any" name="family">
            <string>sans serif</string>
        </test>
        <edit name="family" mode="assign" binding="same">
            <string>sans-serif</string>
        </edit>
    </match>

<!-- Accept deprecated 'sans' alias, replacing it with 'sans-serif' -->
    <match target="pattern">
        <test qual="any" name="family">
            <string>sans</string>
        </test>
        <edit name="family" mode="assign" binding="same">
            <string>sans-serif</string>
        </edit>
    </match>
</fontconfig>
"""

_log = logging.getLogger(__name__)
LOG_MESSAGE = 25

class Archiver(object):

    def __enter__(self):
        return self

    def __exit__(self, type, v, tb):
        return self._archive.close()

class TarArchiver(Archiver):

    def __init__(self, path):
        self._archive = tarfile.open(path, 'w:xz')

    def add_file(self, system_path, zip_path):
        return self._archive.add(system_path, zip_path)

class ZipArchiver(Archiver):

    def __init__(self, path):
        self._archive = zipfile.ZipFile(path, 'w', compression=zipfile.ZIP_DEFLATED)

    def add_file(self, system_path, zip_path):
        if os.path.islink(system_path):
            symlink_zip_info = zipfile.ZipInfo(zip_path)
            symlink_zip_info.create_system = 3 # Unix (for symlink support)
            symlink_zip_info.external_attr = 0xA1ED0000 # Zip softlink magic number
            return self._archive.writestr(symlink_zip_info, os.readlink(system_path))
        return self._archive.write(system_path, zip_path)

class BundleCreator(object):

    def __init__(self, configuration, platform, bundle_type, syslibs, ldd, should_strip_objects, compression_type, destination = None, revision = None, builder_name = None):
        self._configuration = configuration
        self._platform = platform.lower()
        self._revision = revision
        self._bundle_binaries = ['jsc', 'MiniBrowser'] if bundle_type == 'all' else [ bundle_type ]
        self._bundle_type = bundle_type
        self._buildername = builder_name
        self._syslibs = syslibs
        self._shared_object_resolver = SharedObjectResolver(ldd)
        self._should_strip_objects = should_strip_objects
        self._compression_type = compression_type
        self._tmpdir = None
        self._wrapper_scripts = []
        self._port_binary_preffix = 'WebKit' if self._platform == 'gtk' else 'WPE'
        wk_build_path = os.environ['WEBKIT_OUTPUTDIR'] if 'WEBKIT_OUTPUTDIR' in os.environ else \
                        os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'WebKitBuild'))
        self._buildpath = os.path.join(wk_build_path, self._configuration.capitalize())
        self._dlopenwrap_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'webkitpy', 'binary_bundling', 'dlopenwrap'))
        if not os.path.isdir(self._dlopenwrap_dir):
            raise RuntimeError("Can't find the dlopenwrap directory at: %s" % self._dlopenwrap_dir)

        default_bundle_name = bundle_type  + '_' + self._platform + '_' + self._configuration + '.' + self._compression_type
        if destination and  os.path.isdir(destination):
            self._bundle_file_path = os.path.join(destination, default_bundle_name)
        else:
            self._bundle_file_path = os.path.join(wk_build_path, default_bundle_name)

    def _create_tempdir(self, basedir = None):
        if basedir is not None:
            if not os.path.isdir(basedir):
                raise ValueError('%s is not a directory' % basedir)
            return tempfile.mkdtemp(prefix=os.path.join(os.path.abspath(basedir), 'tmp'))
        return tempfile.mkdtemp()

    def _run_cmd_and_get_output(self, command):
        _log.debug("EXEC %s" % command)
        command_process = subprocess.Popen(command,
                                           stdout=subprocess.PIPE,
                                           stderr=subprocess.STDOUT,
                                           encoding='utf-8')
        stdout, stderr = command_process.communicate()
        return command_process.returncode, stdout, stderr

    def _get_osprettyname(self):
        with open('/etc/os-release', 'r') as osrelease_handle:
            for line in osrelease_handle.readlines():
                if line.startswith('PRETTY_NAME='):
                    return line.split('=')[1].strip().strip('"')
        return None

    def _generate_readme(self):
        _log.info('Generate README.txt file')
        readme_file = os.path.join(self._tmpdir, 'README.txt')
        with open(readme_file, 'w') as readme_handle:
            readme_handle.write('Bundle details:\n')
            readme_handle.write('  - WebKit Platform: %s\n' % self._platform.upper())
            readme_handle.write('  - Configuration: %s\n' % self._configuration.capitalize())
            if self._revision:
                readme_handle.write('  - WebKit Revision: %s\n' % self._revision)
            readme_handle.write('  - Bundle type: %s\n' % self._bundle_type)
            if self._buildername:
                readme_handle.write('  - Builder name: %s\n' % self._buildername)
            readme_handle.write('  - Builder date: %s\n' % datetime.datetime.now().isoformat())
            readme_handle.write('  - Builder OS: %s\n' % self._get_osprettyname())
            if self._syslibs == 'generate-install-script':
                readme_handle.write('\nRequired dependencies:\n')
                readme_handle.write('  - This bundle depends on several system libraries that are assumed to be installed.\n')
                readme_handle.write('  - To ensure all the required libraries are installed execute the script: install-dependencies.sh\n')
                readme_handle.write('  - You can pass the flag "--autoinstall" to this script to automatically install the dependencies if needed.\n')
            readme_handle.write('\nRun instructions:\n')
            scripts = 'script' if len(self._wrapper_scripts) == 1 else 'scripts'
            readme_handle.write('  - Execute the wrapper %s in this directory:\n' % scripts)
            for wrapper_script in self._wrapper_scripts:
                readme_handle.write('    * %s\n' %wrapper_script)
        return True

    def _generate_wrapper_script(self, interpreter, binary_to_wrap):
        variables = dict()
        mydir = self._bundler.VAR_MYDIR
        lib_dir = os.path.join(self._bundler.destination_dir(), 'lib')
        syslib_dir = os.path.join(self._bundler.destination_dir(), 'sys/lib')
        bin_dir = os.path.join(self._bundler.destination_dir(), 'bin')
        share_dir = os.path.join(self._bundler.destination_dir(), 'sys/share')

        if os.path.isdir(os.path.join(syslib_dir, 'gio')):
            gio_var = 'GIO_MODULE_DIR' if self._syslibs == 'bundle-all' else 'GIO_EXTRA_MODULES'
            variables[gio_var] = '${%s}/sys/lib/gio' % mydir
        if os.path.isdir(os.path.join(syslib_dir, 'gst')):
            gst_var = 'GST_PLUGIN_SYSTEM_PATH_1_0' if self._syslibs == 'bundle-all' else 'GST_PLUGIN_PATH_1_0'
            variables[gst_var] = '${%s}/sys/lib/gst' % mydir
            variables['GST_REGISTRY_1_0'] = '${%s}/sys/lib/gst/gstreamer-1.0.registry' % mydir
            if os.path.isfile(os.path.join(bin_dir, 'gst-plugin-scanner')):
                variables['GST_PLUGIN_SCANNER'] = '${%s}/bin/gst-plugin-scanner' % mydir
            # pipewire spa plugins may be needed by gstreamer-pipewire
            if os.path.isdir(os.path.join(syslib_dir, 'pipewire')):
                variables['SPA_PLUGIN_DIR'] = '${%s}/sys/lib/pipewire' % mydir
        if os.path.isdir(os.path.join(syslib_dir, 'dri')):
            variables['LIBGL_DRIVERS_PATH'] = '${%s}/sys/lib/dri' % mydir
        if os.path.isdir(os.path.join(syslib_dir, 'glvnd/egl_vendor.d')):
            variables['__EGL_VENDOR_LIBRARY_DIRS'] = '${%s}/sys/lib/glvnd/egl_vendor.d' % mydir
        if os.path.isdir(os.path.join(share_dir)):
            assert(self._syslibs == 'bundle-all')
            variables['XDG_DATA_HOME'] = '${%s}/sys/share' % mydir
            variables['XDG_DATA_DIRS'] = '${%s}/sys/share' % mydir
        if os.path.isdir(os.path.join(share_dir, 'fonts')):
            variables['FONTCONFIG_PATH'] = '${%s}/sys/share/fonts/config' % mydir
        if os.path.isdir(os.path.join(syslib_dir, 'gtk')):
            variables['GTK_PATH'] = '${%s}/sys/lib/gtk' % mydir
        if binary_to_wrap != 'jsc':
            variables['WEBKIT_EXEC_PATH'] = '${%s}/bin' % mydir
            variables['WEBKIT_INJECTED_BUNDLE_PATH'] = '${%s}/lib' % mydir
        self._bundler.generate_wrapper_script(interpreter, binary_to_wrap, variables)
        self._wrapper_scripts.append(binary_to_wrap)

    def _generate_install_deps_script(self, system_packages_needed):
        if not system_packages_needed:
            return
        if 'MiniBrowser' in self._bundle_binaries:
            # Add some extra packages that are needed but the script can't automatically detect
            for extra_needed_pkg in ['ca-certificates', 'shared-mime-info']:
                system_packages_needed.add(extra_needed_pkg)
            # And remove some packages that may be detected due to indirect deps (gstreamer/gio) but are really not needed
            for not_needed_pkg in ['dconf-gsettings-backend', 'gvfs', 'pitivi', 'gstreamer1.0-convolver-pulseeffects', 'gstreamer1.0-x',
                                   'gstreamer1.0-adapter-pulseeffects', 'gstreamer1.0-autogain-pulseeffects', 'gstreamer1.0-alsa',
                                   'gstreamer1.0-clutter-3.0', 'gstreamer1.0-crystalizer-pulseeffects', 'gstreamer1.0-gtk3', 'gstreamer1.0-nice']:
                if not_needed_pkg in system_packages_needed:
                    system_packages_needed.remove(not_needed_pkg)
                # Sometimes the package is identified with an arch suffix, but not always
                not_needed_pkg_arch = not_needed_pkg + ':amd64'
                if not_needed_pkg_arch in system_packages_needed:
                    system_packages_needed.remove(not_needed_pkg_arch)
        installdeps_file = os.path.join(self._tmpdir, 'install-dependencies.sh')
        with open(installdeps_file, 'w') as installdeps_handle:
            installdeps_handle.write(INSTALL_DEPS_SCRIPT_TEMPLATE % {'packages_needed' : ' '.join(system_packages_needed)} )
        os.chmod(installdeps_file, 0o755)

    def _remove_tempdir(self):
        if not self._tmpdir:
            return
        if os.path.isdir(self._tmpdir):
            shutil.rmtree(self._tmpdir)

    def _touch_reset_mtime(self):
        _log.info('Update timestamps (touch)')
        now = datetime.datetime.now().timestamp()
        for root, dirs, files in os.walk(self._tmpdir):
            for name in files:
                os.utime(os.path.join(root, name), times=(now, now))
            for name in dirs:
                os.utime(os.path.join(root, name), times=(now, now))

    def create(self):
        self._tmpdir = self._create_tempdir(self._buildpath)
        self._bundler = BinaryBundler(self._tmpdir, self._buildpath)

        if os.path.isfile(self._bundle_file_path):
            _log.info('Removing previous bundle %s' % self._bundle_file_path)
            os.remove(self._bundle_file_path)

        for bundle_binary in self._bundle_binaries:
            self._create_bundle(bundle_binary)
        self._generate_readme()

        # Potentially avoids errors related to ZIP not able to store mtimes older than 1980
        self._touch_reset_mtime()

        if self._compression_type == 'zip':
            archiver = ZipArchiver(self._bundle_file_path)
        elif self._compression_type == 'tar.xz':
            archiver = TarArchiver(self._bundle_file_path)
        else:
            raise NotImplementedError('Support for compression type %s not implemented' % self._compression_type)
        self._create_archive(archiver)
        self._remove_tempdir()
        if not os.path.isfile(self._bundle_file_path):
            raise RuntimeError('Unable to create the file %s' % self._bundle_file_path)
        _log.log(LOG_MESSAGE, 'Bundle file created at: %s' % self._bundle_file_path)
        return self._bundle_file_path

    def _get_webkit_binaries(self):
        webkit_binaries = []
        bin_dir = os.path.join(self._buildpath, 'bin')
        for entry in os.listdir(bin_dir):
            if entry.startswith(self._port_binary_preffix) and (entry.endswith('Process') or entry.endswith('Driver')):
                binary = os.path.join(bin_dir, entry)
                if os.path.isfile(binary) and os.access(binary, os.X_OK):
                    webkit_binaries.append(binary)
        if len(webkit_binaries) < 2:
            raise RuntimeError('Could not find required WebKit Process binaries. Check if you are passing the right platform value.')
        return webkit_binaries

    def _get_webkit_lib(self, name):
        libkeyname = name.lower()
        lib_dir = os.path.join(self._buildpath, 'lib')
        bundle_lib = None
        for entry in os.listdir(lib_dir):
            if entry.endswith('.so') and libkeyname in entry.lower() and 'test' not in entry.lower():
                assert(bundle_lib == None)
                bundle_lib = os.path.join(lib_dir, entry)
                break
        assert(bundle_lib)
        return bundle_lib

    def _create_archive(self, archiver):
        _log.info('Create archive')
        with archiver:
            for dirname, subdirs, files in os.walk(self._tmpdir):
                for filename in files:
                    system_file_path = os.path.join(dirname, filename)
                    zip_file_path = system_file_path.replace(self._tmpdir, '', 1).lstrip('/')
                    archiver.add_file(system_file_path, zip_file_path)


    def _get_system_package_name(self, object):
        if not shutil.which('dpkg'):
            raise RuntimeError('Adding system dependencies only supported for dpkg-based distros. Try passing --syslibs=bundle-all')
        retcode, stdout, stderr = self._run_cmd_and_get_output(['dpkg', '-S', object])
        if retcode != 0:
            # Give a second-try with the realpath of the object.
            # This fixes issue on Ubuntu-20.04 that has a /lib symlink to /usr/lib
            # and objects point to /lib, but dpkg only recognizes the files on /usr/lib
            object_realpath = os.path.realpath(object)
            if object_realpath != object:
                retcode, stdout, stderr = self._run_cmd_and_get_output(['dpkg', '-S', object_realpath])
        if retcode != 0:
            # Package not found
            return None
        package = stdout.split(' ')[0].rstrip(':')
        _log.info('Add dependency on system package [%s]: %s' %(package, object))
        return package

    def _get_pkg_config_var(self, module, var_name, assert_value=True):
        retcode, stdout, stderr = self._run_cmd_and_get_output(['pkg-config', '--variable=%s' % var_name, module])
        if retcode != 0:
            raise RuntimeError('The pkg-config command returned status %d' % retcode)
        var_value = stdout.strip()
        if assert_value:
            if var_name.endswith('dir') and not os.path.isdir(var_value):
                raise RuntimeError('The pkg-config entry for %s is not a directory: %s' % (var_name, var_value))
            if len(var_value) == 0:
                raise RuntimeError('The pkg-config entry for %s has zero lenght' % var_value)
        return var_value

    def _list_files_directory(self, directory, list_inside_subdirs=False, filter_suffix=None):
        found_files = []
        for entry in os.listdir(directory):
            entry_path = os.path.join(directory, entry)
            if os.path.isdir(entry_path):
                if list_inside_subdirs:
                    found_files.extend(self._list_files_directory(entry_path, list_inside_subdirs, filter_suffix))
            elif os.path.isfile(entry_path):
                if filter_suffix is None or entry_path.endswith(filter_suffix):
                    found_files.append(entry_path)
        if len(found_files) == 0:
            raise RunTimeError('Can not find any matching file at directory %s' % directory)
        return found_files

    def _get_gio_modules(self):
        gio_module_dir = self._get_pkg_config_var('gio-2.0', 'giomoduledir')
        return self._list_files_directory(gio_module_dir, filter_suffix='.so')

    def _get_gtk_modules(self, subdir):
        gtk_lib_dir = self._get_pkg_config_var('gtk+-3.0', 'libdir')
        gtk_binary_version = self._get_pkg_config_var('gtk+-3.0', 'gtk_binary_version')
        gtk_module_dir = os.path.join(gtk_lib_dir, 'gtk-3.0', gtk_binary_version, subdir)
        if not os.path.isdir(gtk_module_dir):
            raise RuntimeError('Can not find the the gtk module dir %s' % gtk_module_dir)
        return self._list_files_directory(gtk_module_dir, filter_suffix='.so')

    def _get_gstreamer_modules(self):
        gstreamer_plugins_dir = self._get_pkg_config_var('gstreamer-1.0', 'pluginsdir')
        return self._list_files_directory(gstreamer_plugins_dir, filter_suffix='.so')

    def _get_gstreamer_plugin_scanner(self):
        def plugin_scanner_found(plugin_scanner_path):
            return (os.path.isfile(plugin_scanner_path) and os.access(plugin_scanner_path, os.X_OK))

        gstreamer_plugin_scanner_dir = gio_module_dir = self._get_pkg_config_var('gstreamer-1.0', 'pluginscannerdir', assert_value=False)
        gstreamer_plugin_scanner = os.path.join(gstreamer_plugin_scanner_dir, 'gst-plugin-scanner')
        if not plugin_scanner_found(gstreamer_plugin_scanner):
            # pluginscannerdir pkg-config variable is available only on recent versions of gst
            # give it a second chance with some known directories on debian/ubuntu and fedora/centos
            lib_dir = self._get_pkg_config_var('gstreamer-1.0', 'libdir')
            plugins_dir = self._get_pkg_config_var('gstreamer-1.0', 'pluginsdir')
            for gstreamer_plugin_scanner_dir in [ os.path.join(lib_dir, 'gstreamer1.0/gstreamer-1.0'), plugins_dir, '/usr/libexec/gstreamer-1.0']:
                gstreamer_plugin_scanner = os.path.join(gstreamer_plugin_scanner_dir, 'gst-plugin-scanner')
                if plugin_scanner_found(gstreamer_plugin_scanner):
                    break
        if not plugin_scanner_found(gstreamer_plugin_scanner):
            raise RuntimeError('Could not find gst-plugin-scanner binary at path "%s".' % gstreamer_plugin_scanner)
        return gstreamer_plugin_scanner

    def _get_libc_libraries(self):
        libc_library_names = [ 'libanl', 'libBrokenLocale', 'libc', 'libcrypt', 'libdl', 'libgcc_s', 'libm',
                               'libmvec', 'libnsl', 'libpthread', 'libresolv', 'librt', 'libthread_db', 'libutil']
        libc_libraries = []
        lib_dir = self._get_pkg_config_var('dri', 'libdir')
        for entry in os.listdir(lib_dir):
            if any(entry.startswith(l+'.so.') for l in libc_library_names) or (entry.startswith('libnss_') and '.so.' in entry):
                libc_libraries.append(os.path.join(lib_dir, entry))
        for libc_library_prefix in libc_library_names:
            if not any(os.path.basename(l).startswith(libc_library_prefix+'.so.') for l in libc_libraries):
                raise RuntimeError('Unable to find libc library %s' % libc_library_prefix)
        if not any(os.path.basename(l).startswith('libnss_') for l in libc_libraries):
            raise RuntimeError('Unable to find libc libnss_ libraries')
        return libc_libraries

    def _get_mesa_libraries(self):
        mesa_library_names = ['libglapi', 'libxcb-glx', 'libGLX', 'libEGL', 'libGL', 'libGLESv2', 'libEGL_mesa', 'libGLX_mesa']
        mesa_libraries = []
        lib_dir_dri = self._get_pkg_config_var('dri', 'libdir')
        # Some versions of the flatpak SDK ships the Mesa libraries into a non-standard path.
        candidate_lib_dirs = [ lib_dir_dri, os.path.join(lib_dir_dri, 'GL/default/lib') ]
        for lib_dir in candidate_lib_dirs:
            if os.path.isdir(lib_dir):
                for entry in os.listdir(lib_dir):
                    if any(entry.startswith(l+'.so.') for l in mesa_library_names):
                        mesa_libraries.append(os.path.join(lib_dir, entry))
        for mesa_library_prefix in mesa_library_names:
            if not any(os.path.basename(l).startswith(mesa_library_prefix+'.so.') for l in mesa_libraries):
                raise RuntimeError('Unable to find mesa library %s' % mesa_library_prefix)
        return mesa_libraries

    def _get_mesa_dri_drivers(self):
        dridriver_dir = self._get_pkg_config_var('dri', 'dridriverdir')
        return self._list_files_directory(dridriver_dir, filter_suffix='.so')

    def _get_gdk_pixbuf_loaders(self):
        loaders_dir = self._get_pkg_config_var('gdk-pixbuf-2.0', 'gdk_pixbuf_moduledir')
        return self._list_files_directory(loaders_dir, filter_suffix='.so')

    def _get_gdk_query_loaders_binary(self):
        gdk_pixbuf_query_loader_value = self._get_pkg_config_var('gdk-pixbuf-2.0', 'gdk_pixbuf_query_loaders')
        gdk_pixbuf_libdir = self._get_pkg_config_var('gdk-pixbuf-2.0', 'libdir')
        # Some distros (like Debian) don't ship the binary on the declared path on pkg-config, but inside $libdir/gdk-pixbuf-2.0
        for gdk_pixbuf_binary  in [ gdk_pixbuf_query_loader_value, os.path.join(gdk_pixbuf_libdir, 'gdk-pixbuf-2.0/gdk-pixbuf-query-loaders') ]:
            if os.path.isfile(gdk_pixbuf_binary) and os.access(gdk_pixbuf_binary, os.X_OK):
                return gdk_pixbuf_binary
        raise RuntimeError('Unable to find a valid path to gdk-pixbuf-query-loaders binary')

    def _get_gtk_query_immodules_binary(self):
        gtk_libdir = self._get_pkg_config_var('gtk+-3.0', 'libdir')
        for gtk_dir in [ '/usr/bin', gtk_libdir, os.path.join(gtk_libdir, 'libgtk-3-0'), os.path.join(gtk_libdir, 'gtk-3.0') ]:
            gtk_immodules_binary = os.path.join(gtk_dir, 'gtk-query-immodules-3.0')
            if os.path.isfile(gtk_immodules_binary) and os.access(gtk_immodules_binary, os.X_OK):
                return gtk_immodules_binary
        raise RuntimeError('Unable to find a valid path to gdk-pixbuf-query-loaders binary')

    def _get_pipewire_spa_basedir_and_plugins(self):
        spa_plugin_dir = self._get_pkg_config_var('libspa-0.2', 'plugindir', assert_value=False)
        # Currently the flatpak SDK doesn't declare plugindir (not sure why).. fallback to build it from libdir
        if not os.path.isdir(spa_plugin_dir):
            spa_plugin_dir = self._get_pkg_config_var('libspa-0.2', 'libdir')
            spa_plugin_dir = os.path.join(spa_plugin_dir, 'spa-0.2')
        pipewire_spa_plugins = self._list_files_directory(spa_plugin_dir, list_inside_subdirs=True, filter_suffix='.so')
        return spa_plugin_dir, pipewire_spa_plugins

    def _add_object_or_get_sysdep(self, object, object_type):
        provided_by_system_package = None
        if self._syslibs == 'bundle-all':
            self._bundler.copy_and_maybe_strip_patchelf(object, type=object_type, strip=self._should_strip_objects, patchelf_removerpath=True, patchelf_nodefaultlib=True)
        else:
            provided_by_system_package = self._get_system_package_name(object)
            if not provided_by_system_package:
                self._bundler.copy_and_maybe_strip_patchelf(object, type=object_type, strip=self._should_strip_objects, patchelf_removerpath=True, patchelf_nodefaultlib=False)
        return provided_by_system_package

    def _ensure_wpe_backend_symlink(self):
        # WPE/WPERenderer dlopens this library without a version suffix,
        # so we need to ensure there is a proper symlink
        bundle_lib_dir = os.path.join(self._tmpdir, 'lib')
        wpe_backend_soname = 'libWPEBackend-fdo-1.0.so'
        previous_dir = os.getcwd()
        for entry in os.listdir(bundle_lib_dir):
            if entry.startswith(wpe_backend_soname + '.'):
                os.chdir(bundle_lib_dir)
                if not os.path.exists(wpe_backend_soname):
                    os.symlink(entry, wpe_backend_soname)
                os.chdir(previous_dir)
                break
        if not os.path.isfile(os.path.join(bundle_lib_dir, wpe_backend_soname)):
            raise RuntimeError('Unable to create libWPEBackend-fdo-1.0.so symlink')

    def _dlopenwrap_make(self, action=None):
        dlopenwrap_path = os.path.join(self._dlopenwrap_dir, 'dlopenwrap.so')
        previous_cwd = os.getcwd()
        os.chdir(self._dlopenwrap_dir)
        if action == 'build':
            retcode, stdout, stderr = self._run_cmd_and_get_output(['make'])
            if (retcode != 0) or (not os.path.isfile(dlopenwrap_path)):
                raise RuntimeError('Error building dlopenwrap.so: The make command returned %d status with this output: %s %s' % (retcode, stdout, stderr))
        elif action == 'clean':
            retcode, stdout, stderr = self._run_cmd_and_get_output(['make', 'clean'])
            if (retcode != 0) or os.path.isfile(dlopenwrap_path):
                raise RuntimeError('Error cleaning dlopenwrap.so: The make command returned %d status with this output: %s %s' % (retcode, stdout, stderr))
        else:
            raise NotImplementedError('Unknown action %s' % action)
        os.chdir(previous_cwd)
        return dlopenwrap_path

    def _discover_egl_mesa_config_file(self):
        egl_libdir = self._get_pkg_config_var('egl', 'libdir')
        # Not known way of getting the directory dynamically, assume it is there.
        icd_egl_mesa_config_candidates = ['/usr/share/glvnd/egl_vendor.d/50_mesa.json',
                                           os.path.join(egl_libdir, 'GL/glvnd/egl_vendor.d/50_mesa.json'),
                                           os.path.join(egl_libdir, 'glvnd/egl_vendor.d/50_mesa.json')]
        for icd_egl_mesa_config_candidate in icd_egl_mesa_config_candidates:
            if os.path.isfile(icd_egl_mesa_config_candidate):
                return icd_egl_mesa_config_candidate
        raise RuntimeError('Could not find mesa EGL ICD config')

    def _get_mesa_egl_icd_driver_path(self, icd_config):
        with open(icd_config, 'r') as f:
            json_data = json.load(f)
        egl_icd_library_name = json_data['ICD']['library_path']
        if os.path.isabs(egl_icd_library_name):
            raise NotImplementedError('Path rewritting needs to be implemented to handle ICD config files with abspath')
        # Some versions of the flatpak SDK ships the EGL ICD library into a non-standard path.
        egl_libdir = self._get_pkg_config_var('egl', 'libdir')
        egl_icd_library_found = False
        egl_icd_library_paths = [os.path.join(egl_libdir, egl_icd_library_name), os.path.join(egl_libdir, 'GL/default/lib', egl_icd_library_name)]
        for egl_icd_library_path in egl_icd_library_paths:
            if os.path.isfile(egl_icd_library_path):
                egl_icd_library_found = True
                break
        if not egl_icd_library_found:
            raise RuntimeError('Unable to find the EGL ICD library %s' % egl_icd_library_path)
        return egl_icd_library_path

    def _get_total_size_dir(self, directory):
        size = 0
        for root, dirs, files in os.walk(directory):
            for file in files:
                file_path = os.path.join(root, file)
                if os.path.isfile(file_path):
                    size += os.path.getsize(file_path)
        return size

    def _generate_font_conf(self, data_directory):
        font_conf_path = os.path.join(data_directory, 'fonts.conf')
        with open(font_conf_path, 'w') as font_conf_handle:
            font_conf_handle.write(FONT_CONF_FILE)

    def _generate_tls_cafile(self, data_directory):
        cafile_path = os.path.join(data_directory, 'bundle-ca-certificates.pem')
        retcode, stdout, stderr = self._run_cmd_and_get_output(['trust', 'extract', '--format=pem-bundle', cafile_path])
        # trust fails inside flatpak due to https://gitlab.com/freedesktop-sdk/freedesktop-sdk/-/issues/904
        # fall-back to trying to manually copy the bundle
        if (retcode != 0) or (not os.path.isfile(cafile_path)) or (os.path.getsize(cafile_path) < 100):
            _log.warning('Failed to get system CA bundle with the trust command. Trying to fall-back to manual copy.')
            if os.path.isfile(cafile_path):
                os.unlink(cafile_path)
            for candidate_cafile in ['/etc/ssl/certs/ca-certificates.crt', '/etc/ssl/cert.pem', '/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem']:
                if os.path.isfile(candidate_cafile):
                    shutil.copy(candidate_cafile, cafile_path)
                    break
        if (not os.path.isfile(cafile_path)) or (os.path.getsize(cafile_path) < 100):
            raise RuntimeError('Failed to generate or copy the system TLS CAFile with the certificates.')

    def _create_bundle(self, bundle_binary):
        main_binary_path = os.path.join(self._buildpath, 'bin', bundle_binary)
        if not os.path.isfile(main_binary_path) or not os.access(main_binary_path, os.X_OK):
            raise ValueError('Cannot find binary for %s at %s' % (bundle_binary, main_binary_path) )

        copied_interpreter = None
        needs_to_create_wpe_backend_symlink = False
        gio_modules = []
        gtk_print_modules = []
        gtk_im_modules = []
        gstreamer_modules = []
        mesa_dri_drivers = []
        pipewire_spa_plugins = []
        gdk_pixbuf_loaders = []
        egl_icd_mesa_config_file = None
        libraries_checked = set()
        system_packages_needed = set()
        binaries_to_copy = set()
        objects_to_copy = [ main_binary_path ]
        # We only want to separate the libraries in lib and sys/lib for MiniBrowser and 'all' bundles
        self._bundler.set_use_sys_lib_directory('MiniBrowser' in self._bundle_binaries)
        if bundle_binary == 'MiniBrowser':
            # bunddle WebKit libraries
            objects_to_copy.extend(self._get_webkit_binaries())
            objects_to_copy.append(self._get_webkit_lib('InjectedBundle'))
            if self._platform == 'wpe':
                objects_to_copy.append(self._get_webkit_lib('InspectorResources'))
            # Bundle extra system related libraries
            gio_modules = self._get_gio_modules()
            objects_to_copy.extend(gio_modules)
            gstreamer_modules = self._get_gstreamer_modules()
            objects_to_copy.extend(gstreamer_modules)
            if self._syslibs == 'bundle-all':
                # need to ship a copy of gst pluging scanner
                objects_to_copy.append(self._get_gstreamer_plugin_scanner())
                # pipewire spa plugins
                pipewire_spa_plugin_basedir, pipewire_spa_plugins = self._get_pipewire_spa_basedir_and_plugins()
                objects_to_copy.extend(pipewire_spa_plugins)
                # graphics drivers
                mesa_dri_drivers = self._get_mesa_dri_drivers()
                objects_to_copy.extend(mesa_dri_drivers)
                objects_to_copy.extend(self._get_mesa_libraries())
                egl_icd_mesa_config_file = self._discover_egl_mesa_config_file()
                objects_to_copy.append(self._get_mesa_egl_icd_driver_path(egl_icd_mesa_config_file))
                # system extra libraries
                objects_to_copy.extend(self._get_libc_libraries())
                # gtk modules
                if self._platform == 'gtk':
                    gtk_print_modules = self._get_gtk_modules('printbackends')
                    objects_to_copy.extend(gtk_print_modules)
                    gtk_im_modules = self._get_gtk_modules('immodules')
                    objects_to_copy.extend(gtk_im_modules)
                    gdk_pixbuf_loaders = self._get_gdk_pixbuf_loaders()
                    objects_to_copy.extend(gdk_pixbuf_loaders)
                    objects_to_copy.append(self._get_gdk_query_loaders_binary())
                    objects_to_copy.append(self._get_gtk_query_immodules_binary())
                # our dlopenwrap lib
                objects_to_copy.append(self._dlopenwrap_make('build'))

        for object in objects_to_copy:
            system_package = None
            if object in gio_modules:
                system_package = self._add_object_or_get_sysdep(object, 'lib/gio')
            elif object in gstreamer_modules:
                system_package = self._add_object_or_get_sysdep(object, 'lib/gst')
            elif object in gtk_print_modules:
                system_package = self._add_object_or_get_sysdep(object, 'lib/gtk/printbackends')
            elif object in gtk_print_modules:
                system_package = self._add_object_or_get_sysdep(object, 'lib/gtk/printbackends')
            elif object in gtk_im_modules:
                system_package = self._add_object_or_get_sysdep(object, 'lib/gtk/immodules')
            elif object in mesa_dri_drivers:
                system_package = self._add_object_or_get_sysdep(object, 'lib/dri')
            elif object in gdk_pixbuf_loaders:
                system_package = self._add_object_or_get_sysdep(object, 'lib/gdk-pixbuf/loaders')
            elif object in pipewire_spa_plugins:
                # This is a level-2 indirect dependency (gstreamer -> gst-pipewire -> pipewire-spa-plugin)
                # For the case of system_packages we rely on the distro dependencies betwen packages
                # But the bundle-all case we need a special case because this dep is not automatically detected (dlopen of plugins)
                if self._syslibs == 'bundle-all':
                    pipewire_spa_plugin_dest_subdir = object.replace(pipewire_spa_plugin_basedir, '', 1)
                    pipewire_spa_plugin_dest_subdir = os.path.dirname(pipewire_spa_plugin_dest_subdir.lstrip('/'))
                    pipewire_spa_plugin_dest_subdir = os.path.join('lib/pipewire', pipewire_spa_plugin_dest_subdir)
                    self._bundler.copy_and_maybe_strip_patchelf(object, type=pipewire_spa_plugin_dest_subdir, strip=self._should_strip_objects, patchelf_removerpath=True, patchelf_nodefaultlib=True)
            elif '.so' in object:
                if 'libwpebackend' in object.lower():
                    needs_to_create_wpe_backend_symlink = True
                self._bundler.copy_and_maybe_strip_patchelf(object, type='lib', strip=self._should_strip_objects, patchelf_removerpath=True, patchelf_nodefaultlib=self._syslibs=='bundle-all')
            else:
                binaries_to_copy.add(object)
            # There is no need to examine the libraries linked with objects coming from a system package,
            # because system packages already declare dependencies between them.
            # However, if we are running with self._syslibs == 'bundle-all' then system_package will be None,
            # and everything will be examined and bundled as we don't account for system packages in that case.
            if system_package:
                system_packages_needed.add(system_package)
            else:
                libraries, interpreter = self._shared_object_resolver.get_libs_and_interpreter(object)
                if interpreter is None:
                    raise RuntimeError('Could not determine interpreter for binary %s' % object)
                if copied_interpreter is None:
                    if self._syslibs == 'bundle-all':
                        self._bundler.copy_and_maybe_strip_patchelf(interpreter, type='interpreter', strip=self._should_strip_objects, patchelf_removerpath=self._syslibs=='bundle-all', patchelf_nodefaultlib=self._syslibs=='bundle-all')
                    copied_interpreter = interpreter
                elif copied_interpreter != interpreter:
                    raise RuntimeError('Detected binaries with different interpreters: %s != %s' %(copied_interpreter, interpreter))
                for library in libraries:
                    if library in libraries_checked:
                        _log.debug('Skip already checked [lib]: %s' % library)
                        continue
                    libraries_checked.add(library)
                    system_package = self._add_object_or_get_sysdep(library, 'lib')
                    if system_package:
                        system_packages_needed.add(system_package)
                    elif 'libwpebackend' in library.lower():
                        needs_to_create_wpe_backend_symlink = True
        if needs_to_create_wpe_backend_symlink:
            self._ensure_wpe_backend_symlink()
        for binary in binaries_to_copy:
            if self._syslibs == 'bundle-all':
                self._bundler.copy_and_maybe_strip_patchelf(binary, type='bin', strip=self._should_strip_objects, patchelf_removerpath=True, patchelf_nodefaultlib=True, patchelf_setinterpreter_relativepath=interpreter)
            else:
                self._bundler.copy_and_maybe_strip_patchelf(binary, type='bin', strip=self._should_strip_objects, patchelf_removerpath=True, patchelf_nodefaultlib=False)

        # Now copy data files to share dir (only needed when bunding all for MiniBrowser).
        # We assume that the system uses standard paths at /usr/share and /etc for this resources
        # Every path should be checked and if some one is not found, then it will raise an error.
        if bundle_binary == 'MiniBrowser' and self._syslibs == 'bundle-all':
            target_share_dir = os.path.join(self._tmpdir, 'sys/share')
            # Copy glvnd EGL json files
            _log.info('Copy EGL config files')
            egl_icd_configdir_dest = os.path.join(self._tmpdir, 'sys/lib/glvnd/egl_vendor.d')
            if not os.path.isdir(egl_icd_configdir_dest):
                os.makedirs(egl_icd_configdir_dest)
                shutil.copy(egl_icd_mesa_config_file, egl_icd_configdir_dest)
            # Copy glib-2 compiled schemas
            _log.info('Copy Glib compiled schemas')
            schemas_target_dir = os.path.join(target_share_dir, 'glib-2.0/schemas')
            os.makedirs(schemas_target_dir)
            compiled_schemas_file = '/usr/share/glib-2.0/schemas/gschemas.compiled'
            if not os.path.isfile(compiled_schemas_file):
                raise NotImplementedError("Can't find the gschemas.compiled file at the standard location in this system")
            shutil.copy(compiled_schemas_file, schemas_target_dir)
            _log.info('Copy shared mime info cache')
            # Same for shared-mime-info cache
            mime_target_dir = os.path.join(target_share_dir, 'mime')
            os.makedirs(mime_target_dir)
            mime_cache = '/usr/share/mime/mime.cache'
            if not os.path.isfile(mime_cache):
                raise NotImplementedError("Can't find the mime.cache file from shared-mime-info at the standard location in this system")
            shutil.copy(mime_cache, mime_target_dir)
            _log.info('Generate fontconfig config')
            # Fontconfig fonts and config
            target_font_config_dir = os.path.join(target_share_dir, 'fonts/config')
            target_font_data_dir = os.path.join(target_share_dir, 'fonts/fonts')
            os.makedirs(target_font_config_dir)
            # We keep assuming standard locations for this files on the target system
            system_font_config_dir = '/etc/fonts/conf.d'
            system_font_data_dir = '/usr/share/fonts'
            if not (os.path.isdir(system_font_config_dir) and os.path.isdir(system_font_data_dir)):
                raise NotImplementedError("Can't find the fontconfig files at the standard location in this system")
            shutil.copytree(system_font_config_dir, os.path.join(target_font_config_dir, os.path.basename(system_font_config_dir)))
            self._generate_font_conf(target_font_config_dir)
            _log.info('Copy fonts')
            # We copy every font, so this may copy lot of data if you have lot of fonts installed.
            # So print a warning in that ase
            system_font_data_size = self._get_total_size_dir(system_font_data_dir)
            if system_font_data_size > 104857600:  # 100 MB
                _log.warning('Copying %d MB of font data from the system at /usr/share/fonts. This should be optimized.' % int(system_font_data_size / (1024*1024)))
            shutil.copytree(system_font_data_dir, target_font_data_dir)
            # Copy XKB data
            _log.info('Copy XKB data')
            system_xkb_data = '/usr/share/X11/xkb'
            if not (os.path.isdir(system_xkb_data)):
                raise NotImplementedError("Can't find the XKB data files at the standard location in this system")
            target_xkb_data = os.path.join(target_share_dir, 'xkb')
            shutil.copytree(system_xkb_data, target_xkb_data)
            # Flatpak configures gnutls to use the p11-kit trust store and then uses p11-kit to access the host SSL certificates.
            # - It starts a p11 kit server on the hosts and bind-mounting a socket that later is used from inside the sandbox via p11-kit-client.so
            # See: https://github.com/p11-glue/p11-kit/issues/68
            # Starting a p11-kit server is problematic for the use case of the bundle, and also will only work if the host is configured to use p11-kit
            # So we take a different approach here.
            # 1. We generate a bundle-ca-certificates.pem file with all the certs of the system were the bundle was generated and we ship this file
            # 2. At runtime we look for the system TLS ca-bundle.pem file and we use it. If we can't find it then we default to using the one shipped inside the bundle.
            # We pass the path to the TLS ca-bundle.pem file using the env var WEBKIT_TLS_CAFILE_PEM
            # The host p11-kit setup may still work if it is using the same paths for loading the p11-kit modules, but otherwise it will fail to load the modules.
            # That is ok, since we don't need anymore a working p11-kit setup. If it works it will be used, but if not we have a fallback.
            # To debug this is useful to export P11_KIT_DEBUG=all
            # This approach also allows this bundle script go generate a working bundle when gnutls is not configured to use p11-kit.
            _log.info('Generate TLS CA certificates file')
            certs_target_dir = os.path.join(target_share_dir, 'certs')
            os.makedirs(certs_target_dir)
            self._generate_tls_cafile(certs_target_dir)
            if self._platform == 'gtk':
                _log.info('Copy basic GTK icons and themes.')
                icons_target_dir = os.path.join(target_share_dir, 'icons')
                os.makedirs(icons_target_dir)
                for gtk_icon_dir in ['/usr/share/icons/Adwaita', '/usr/share/icons/hicolor']:
                    if not os.path.isdir(gtk_icon_dir):
                        raise NotImplementedError('Can not find the GTK icon theme at path "%s" in this system' % gtk_icon_dir)
                    shutil.copytree(gtk_icon_dir, os.path.join(icons_target_dir, os.path.basename(gtk_icon_dir)))

        # Finally generate the wrappers.
        self._generate_wrapper_script(interpreter, bundle_binary)
        if bundle_binary == 'MiniBrowser':
            self._generate_wrapper_script(interpreter, self._port_binary_preffix + 'WebDriver')
        self._generate_install_deps_script(system_packages_needed)
        # Clean remaining stuff
        if bundle_binary == 'MiniBrowser' and self._syslibs == 'bundle-all':
            self._dlopenwrap_make('clean')


class BundleUploader(object):

    def __init__(self, bundle_file_path, remote_config_file, bundle_type, platform, configuration, compression_type, revision, log_level):
        self._bundle_file_path = bundle_file_path
        self._remote_config_file = remote_config_file
        self._configuration = configuration
        self._revision = revision
        self._bundle_type = bundle_type
        self._platform = platform
        self._compression_type = compression_type
        self._sftp_quiet = log_level == 'quiet' or log_level == 'minimal'
        if not os.path.isfile(self._remote_config_file):
            raise ValueError('Can not find remote config file for upload at path %s' % self._remote_config_file)

    def _sha256sum(self, file):
        hash = hashlib.sha256()
        with open(file, 'rb') as f:
            for chunk in iter(lambda: f.read(4096), b''):
                hash.update(chunk)
        return hash.hexdigest()

    def _get_osidversion(self):
        with open('/etc/os-release', 'r') as osrelease_handle:
            for line in osrelease_handle.readlines():
                if line.startswith('ID='):
                    os_id = line.split('=')[1].strip().strip('"')
                if line.startswith('VERSION_ID='):
                    version_id = line.split('=')[1].strip().strip('"')
        assert(os_id)
        assert(version_id)
        osidversion = os_id + '-' + version_id
        assert(' ' not in osidversion)
        assert(len(osidversion) > 3)
        return osidversion

    # The expected format for --remote-config-file is something like:
    # {
    # "servername": "webkitgtk.org",
    # "serveraddress": "webkitgtk.intranet-address.local",
    # "serverport": "23",
    # "username": "upload-bot-64",
    # "baseurl": "https://webkitgtk.org/built-products",
    # "remotepath" : "x86_64/nightly/%(bundletype)s/%(distro_id_ver)s/%(bundletype)s_%(platform)s_%(configuration)s_r%(version)s.%(compression_type)s",
    # "sshkey": "output of the priv key in base64. E.g. cat ~/.ssh/id_rsa|base64 -w0"
    # }
    def upload(self):
        remote_data = json.load(open(self._remote_config_file))
        remote_file_bundle_path = remote_data['remotepath'] % { 'bundletype' : self._bundle_type,
                                                                'configuration' : self._configuration,
                                                                'compression_type' : self._compression_type,
                                                                'distro_id_ver' : self._get_osidversion().capitalize(),
                                                                'platform' : self._platform,
                                                                'version' : self._revision }
        with tempfile.NamedTemporaryFile(mode='w+b') as sshkeyfile, tempfile.NamedTemporaryFile(mode='w+') as hashcheckfile, \
             tempfile.NamedTemporaryFile(mode='w+') as lastisfile, tempfile.NamedTemporaryFile(mode='w+') as uploadinstructionsfile:

            # In theory NamedTemporaryFile() is already created 0600. But it don't hurts ensuring this again here.
            os.chmod(sshkeyfile.name, 0o600)
            sshkeyfile.write(base64.b64decode(remote_data['sshkey']))
            sshkeyfile.flush()
            # Generate and upload also a sha256 hash
            hashforbundle = self._sha256sum(self._bundle_file_path)
            os.chmod(hashcheckfile.name, 0o644)
            hashcheckfile.write('%s %s\n' % (hashforbundle, os.path.basename(remote_file_bundle_path)))
            hashcheckfile.flush()
            # A LAST-IS file for convenience
            os.chmod(lastisfile.name, 0o644)
            lastisfile.write('%s\n' % os.path.basename(remote_file_bundle_path))
            lastisfile.flush()
            # SFTP upload instructions file
            uploadinstructionsfile.write('progress\n')
            uploadinstructionsfile.write('put %s %s\n' % (self._bundle_file_path, remote_file_bundle_path))
            remote_file_bundle_path_no_ext, _ = os.path.splitext(remote_file_bundle_path)
            uploadinstructionsfile.write('put %s %s\n' % (hashcheckfile.name, remote_file_bundle_path_no_ext + '.sha256sum'))
            uploadinstructionsfile.write('put %s %s\n' % (lastisfile.name, os.path.join(os.path.dirname(remote_file_bundle_path), 'LAST-IS')))
            uploadinstructionsfile.write('quit\n')
            uploadinstructionsfile.flush()
            # The idea of this is to ensure scp doesn't ask any question (not even on the first run).
            # This should be secure enough according to https://www.gremwell.com/ssh-mitm-public-key-authentication
            sftpCommand = ['sftp',
                           '-o', 'StrictHostKeyChecking=no',
                           '-o', 'UserKnownHostsFile=/dev/null',
                           '-o', 'LogLevel=ERROR',
                           '-P', remote_data['serverport'],
                           '-i', sshkeyfile.name,
                           '-b', uploadinstructionsfile.name,
                           '%s@%s' % (remote_data['username'], remote_data['serveraddress'])]
            _log.info('Uploading bundle to %s as %s with sha256 hash %s' % (remote_data['servername'], remote_file_bundle_path, hashforbundle))
            sftp_out = subprocess.DEVNULL if self._sftp_quiet else sys.stdout
            if subprocess.call(sftpCommand, stdout=sftp_out, stderr=sftp_out) != 0:
                raise RuntimeError('The sftp command returned non-zero status')

        _log.log(LOG_MESSAGE, 'Done: archive sucesfully uploaded to %s/%s' % (remote_data['baseurl'], remote_file_bundle_path))
        return 0


def configure_logging(selected_log_level='info'):

    class LogHandler(logging.StreamHandler):
        def __init__(self, stream):
             super().__init__(stream)

        def format(self, record):
            if record.levelno > LOG_MESSAGE:
                return '%s: %s' % (record.levelname, record.getMessage())
            return record.getMessage()

    logging.addLevelName(LOG_MESSAGE, 'MESSAGE')
    if selected_log_level == 'debug':
        log_level = logging.DEBUG
    elif selected_log_level == 'info':
        log_level = logging.INFO
    elif selected_log_level == 'quiet':
        log_level = logging.NOTSET
    elif selected_log_level == 'minimal':
        log_level = logging.getLevelName(LOG_MESSAGE)

    handler = LogHandler(sys.stdout)
    logger = logging.getLogger()
    logger.addHandler(handler)
    logger.setLevel(log_level)
    return handler


def main():
    parser = argparse.ArgumentParser('usage: %prog [options]')
    configuration = parser.add_mutually_exclusive_group(required=True)
    configuration.add_argument('--debug', action='store_const', const='debug', dest='configuration')
    configuration.add_argument('--release', action='store_const', const='release', dest='configuration')
    parser.add_argument('--platform', dest='platform', choices=['gtk', 'wpe'], required=True,
                        help='The WebKit port to generate the bundle')
    parser.add_argument('--bundle', dest='bundle_binary', choices=['jsc', 'MiniBrowser', 'all'], required=True,
                        help='Select what main binary should be included in the bundle')
    parser.add_argument('--syslibs', dest='syslibs', choices=['bundle-all', 'generate-install-script'], default='generate-install-script',
                        help='If value is "bundle-all", the bundle will include _all_ the system libraries instead of a install-dependencies script.\n'
                        'If value is "generate-install-script", the system libraries will not be bundled and a install-dependencies script will be generated for this distribution.')
    parser.add_argument('--ldd', dest='ldd', default='ldd', help='Use alternative ldd (useful for non-native binaries')
    parser.add_argument('--compression', dest='compression', choices=['zip', 'tar.xz'], default='zip')
    parser.add_argument('--destination', action='store', dest='destination',
                        help='Optional path were to store the bundle')
    parser.add_argument('--no-strip', action='store_true', dest='no_strip',
                        help='Do not strip the binaries and libraries inside the bundle')
    parser.add_argument('--log-level', dest='log_level', choices=['quiet', 'minimal', 'info', 'debug'], default='info')
    parser.add_argument('--revision', action='store', dest='webkit_version')
    parser.add_argument('--builder-name', action='store', dest='builder_name')
    parser.add_argument('--remote-config-file', action='store', dest='remote_config_file',
                        help='Optional configuration file with the configuration needed to upload the generated the bundle to a remote server via sftp/ssh.')
    options = parser.parse_args()

    flatpakutils.run_in_sandbox_if_available([sys.argv[0], '--flatpak-' + options.platform] + sys.argv[1:])
    if not flatpakutils.is_sandboxed():
        jhbuildutils.enter_jhbuild_environment_if_available(options.platform)