path: root/scripts/generic/t/01-coveragerunner_testcocoon.t

#!/usr/bin/env perl
use 5.010;
use strict;
use warnings;
use utf8;
use Readonly;

=head1 NAME

01-coveragerunner_testcocoon.t - basic test for coveragerunner_testcocoon.pl

=head1 SYNOPSIS

  perl ./01-coveragerunner_testcocoon.t

This test will run the coveragerunner_testcocoon.pl script with a few different
types of subprocesses and verify that behavior is as expected.

=cut

use Encode;
use English qw( -no_match_vars );
use FindBin;
use Readonly;
use Test::More;
use Capture::Tiny qw( capture );
use Env::Path;
use File::Basename;
use File::Copy;
use File::Find::Rule;
use File::Path qw( mkpath );
use File::Spec::Functions;
use File::Temp qw( tempdir );

use lib "$FindBin::Bin/../../lib/perl5";
use QtQA::Test::More qw( is_or_like create_mock_command );

# Directory separator, quoted for regex
Readonly my $DS => ($OSNAME eq 'MSWin32') ? q{\\\\} : q{/};

# Tempdir testcocoon for tests template
Readonly my $TEMPDIR_TOPLEVEL => File::Spec->tmpdir;

Readonly my $TEMPDIR_TEMPLATE =>
    qr{\Q$TEMPDIR_TOPLEVEL\E${DS}testcocoon_runner\..{6}}sm;

# Preamble always printed out at the beginning
Readonly my $COVERAGERUNNER_PREAMBLE =>
    qr{Gather all library files covered in a global database\sList of all source files in coverage.*};

# Reused paths to csmes for the libraries
# Note for this (and others), the module/.. part is optional,
# because that may be redundant and File::Find may already clean up that
# before returning.
Readonly my $LIST_CSMESLIB =>
    qr{${TEMPDIR_TEMPLATE}${DS}(?:module${DS}\.\.${DS})?qtbase${DS}lib${DS}lib\.csmes.*
${TEMPDIR_TEMPLATE}${DS}(?:module${DS}\.\.${DS})?qtbase${DS}lib${DS}lib2\.csmes.*}sm;

# First merge command
Readonly my $COVERAGERUNNER_CMMERGE_SRC =>
    qr{\+ cmmerge --append --output=${TEMPDIR_TEMPLATE}${DS}module_coverage_src-[0-9]{8}-[0-9]{4}\.csmes ${TEMPDIR_TEMPLATE}${DS}(?:module${DS}\.\.${DS})?qtbase${DS}lib${DS}lib2\.csmes.*};

# Second merge command
Readonly my $COVERAGERUNNER_CMMERGE_GLOBAL =>
    qr{\+ cmmerge --append --output=${TEMPDIR_TEMPLATE}${DS}module_coverage_global-[0-9]{8}-[0-9]{4}\.csmes ${TEMPDIR_TEMPLATE}${DS}module_coverage_unittests-[0-9]{8}-[0-9]{4}\.csmes.*};

# Cmreport command
Readonly my $COVERAGERUNNER_CMREPORT =>
    qr{\+ cmreport --csmes=${TEMPDIR_TEMPLATE}${DS}module_coverage_global-[0-9]{8}-[0-9]{4}\.csmes --xml=${TEMPDIR_TEMPLATE}${DS}module_coverage_report-[0-9]{8}-[0-9]{4}\.xml --select=.* --source=all --source-sort=name --global=all.*};

# gzip command
Readonly my $COVERAGERUNNER_GZIP =>
    qr{\+ gzip ${TEMPDIR_TEMPLATE}${DS}module_coverage_global-[0-9]{8}-[0-9]{4}\.csmes.*};

# xml2html_testcocoon command
Readonly my $COVERAGERUNNER_XML2HTML =>
    qr{\+ xml2html_testcocoon --xml ${TEMPDIR_TEMPLATE}${DS}module_coverage_report-[0-9]{8}-[0-9]{4}\.xml --module module --output ${TEMPDIR_TEMPLATE}.*};

# Standard output up to the first merge (gathering the lib and plugins csmes)
Readonly my $FIRST_CMMERGE =>
    qr{$COVERAGERUNNER_PREAMBLE
$LIST_CSMESLIB
$COVERAGERUNNER_CMMERGE_SRC}sm;

# Standard output up to the second merge (gathering the lib and plugins csmes)
Readonly my $SECOND_CMMERGE =>
    qr{$FIRST_CMMERGE
cmmerge success lib.*
End of list.*
$COVERAGERUNNER_CMMERGE_GLOBAL}sm;

# Standard output until cmreport command
Readonly my $CMREPORT =>
    qr{$SECOND_CMMERGE
cmmerge success tests.*
$COVERAGERUNNER_CMREPORT}sm;

# Standard output until gzip
Readonly my $GZIP =>
    qr{$CMREPORT
cmreport success.*
$COVERAGERUNNER_GZIP}sm;

# Standard output until xml2html_testcocoon
Readonly my $XML2HTML =>
    qr{$GZIP
gzip success.*
$COVERAGERUNNER_XML2HTML}sm;

# Successful run
Readonly my $TESTSCRIPT_SUCCESS =>
    qr{$XML2HTML
xml2html_testcocoon success.*}sm;

# Invalid --qtcoverage-test-ouput required argument
Readonly my $INVALID_TESTS_OUTPUT =>
    qr{${TEMPDIR_TEMPLATE}${DS}tests_invalid\.csmes does not exist. Either the tests have not been run or coverage was not enabled at build time};

# Run coveragerunner_testcocoon
sub test_run
{
    my ($params_ref) = @_;

    my @args              = @{$params_ref->{ args }};
    my $expected_stdout   =   $params_ref->{ expected_stdout };
    my $expected_stderr   =   $params_ref->{ expected_stderr };
    my $expected_success  =   $params_ref->{ expected_success };
    my $testname          =   $params_ref->{ testname }          || q{};

    my $status;
    my ($output, $error) = capture {
        $status = system( 'perl', "$FindBin::Bin/../coveragerunner_testcocoon.pl", @args );
    };

    if ($expected_success) {
        is  ( $status, 0, "$testname exits zero" );
    } else {
        isnt( $status, 0, "$testname exits non-zero" );
    }

    is_or_like( $output, $expected_stdout, "$testname output looks correct" );
    is_or_like( $error,  $expected_stderr, "$testname error looks correct" );

    return;
}

sub test_success
{
    my $tempdir = tempdir( 'testcocoon_runner.XXXXXX', TMPDIR => 1, CLEANUP => 1 );
    init_test_env($tempdir);

    # Put our mock command first in PATH
    local $ENV{ PATH } = $ENV{ PATH };
    Env::Path->PATH->Prepend( $tempdir );

    create_mock_command(
        name        =>  'cmmerge',
        directory   =>  $tempdir,
        sequence    =>  [
            { exitcode => 0, stdout => "cmmerge success lib\n"},
            { exitcode => 0, stdout => "cmmerge success tests\n"},
        ],
    );

    create_mock_command(
        name        =>  'cmreport',
        directory   =>  $tempdir,
        sequence    =>  [
            { exitcode => 0, stdout => "cmreport success\n"},
        ],
    );

    create_mock_command(
        name        =>  'gzip',
        directory   =>  $tempdir,
        sequence    =>  [
            { exitcode => 0, stdout => "gzip success\n"},
        ],
    );

    local $ENV{ TESTING_COVERAGERUNNER } = 1;
    create_mock_command(
        name        =>  'xml2html_testcocoon',
        directory   =>  $tempdir,
        sequence    =>  [
            { exitcode => 0, stdout => "xml2html_testcocoon success\n"},
        ],
    );

    my $module_gitdir = catfile( $tempdir, 'module');
    my $test_csmes = catfile( $tempdir, 'tests.csmes' );

    test_run({
        args                =>  [ '--qt-gitmodule-dir', $module_gitdir, '--qt-gitmodule', 'module', '--qtcoverage-tests-output', $test_csmes, 'testname'],
        expected_stdout     =>  $TESTSCRIPT_SUCCESS,
        expected_stderr     =>  q{},
        expected_success    =>  1,
        testname            =>  'basic test success',
    });

    return;
}

sub test_first_merge_failure
{
    my $tempdir = tempdir( 'testcocoon_runner.XXXXXX', TMPDIR => 1, CLEANUP => 1 );
    init_test_env($tempdir);

    # Put our mock command first in PATH
    local $ENV{ PATH } = $ENV{ PATH };
    Env::Path->PATH->Prepend( $tempdir );

    create_mock_command(
        name        =>  'cmmerge',
        directory   =>  $tempdir,
        sequence    =>  [
            { exitcode => 1, stderr => "first cmmerge failure\n", stdout => ""},
        ],
    );

    my $module_gitdir = catfile( $tempdir, 'module');
    my $test_csmes = catfile( $tempdir, 'tests.csmes' );

    test_run({
        args                =>  [ '--qt-gitmodule-dir', $module_gitdir, '--qt-gitmodule', 'module', '--qtcoverage-tests-output', $test_csmes, 'testname'],
        expected_stdout     =>  $FIRST_CMMERGE,
        expected_stderr     =>  qr{first cmmerge failure\n}sm,
        expected_success    =>  0,
        testname            =>  'test first merge failed',
    });

    return;
}

sub test_second_merge_failure
{
    my $tempdir = tempdir( 'testcocoon_runner.XXXXXX', TMPDIR => 1, CLEANUP => 1 );
    init_test_env($tempdir);

    # Put our mock command first in PATH
    local $ENV{ PATH } = $ENV{ PATH };
    Env::Path->PATH->Prepend( $tempdir );

    create_mock_command(
        name        =>  'cmmerge',
        directory   =>  $tempdir,
        sequence    =>  [
            { exitcode => 0, stdout => "cmmerge success lib\n"},
            { exitcode => 2, stderr => "second cmmerge failure\n", stdout => ""},
        ],
    );

    my $module_gitdir = catfile( $tempdir, 'module');
    my $test_csmes = catfile( $tempdir, 'tests.csmes' );

    test_run({
        args                =>  [ '--qt-gitmodule-dir', $module_gitdir, '--qt-gitmodule', 'module', '--qtcoverage-tests-output', $test_csmes, 'testname'],
        expected_stdout     =>  $SECOND_CMMERGE,
        expected_stderr     =>  qr{second cmmerge failure\n}sm,
        expected_success    =>  0,
        testname            =>  'test second merge failed',
    });

    return;
}

sub test_cmreport_failure
{
    my $tempdir = tempdir( 'testcocoon_runner.XXXXXX', TMPDIR => 1, CLEANUP => 1 );
    init_test_env($tempdir);

    # Put our mock command first in PATH
    local $ENV{ PATH } = $ENV{ PATH };
    Env::Path->PATH->Prepend( $tempdir );

    create_mock_command(
        name        =>  'cmmerge',
        directory   =>  $tempdir,
        sequence    =>  [
            { exitcode => 0, stdout => "cmmerge success lib\n"},
            { exitcode => 0, stdout => "cmmerge success tests\n"},
        ],
    );

    create_mock_command(
        name        =>  'cmreport',
        directory   =>  $tempdir,
        sequence    =>  [
            { exitcode => 3, stderr => "cmreport failure\n", stdout => ""},
        ],
    );

    my $module_gitdir = catfile( $tempdir, 'module');
    my $test_csmes = catfile( $tempdir, 'tests.csmes' );

    test_run({
        args                =>  [ '--qt-gitmodule-dir', $module_gitdir, '--qt-gitmodule', 'module', '--qtcoverage-tests-output', $test_csmes, 'testname'],
        expected_stdout     =>  $CMREPORT,
        expected_stderr     =>  qr{cmreport failure\n}sm,
        expected_success    =>  0,
        testname            =>  'test cmreport failure',
    });

    return;
}

sub test_gzip_failure
{
    my $tempdir = tempdir( 'testcocoon_runner.XXXXXX', TMPDIR => 1, CLEANUP => 1 );
    init_test_env($tempdir);

    # Put our mock command first in PATH
    local $ENV{ PATH } = $ENV{ PATH };
    Env::Path->PATH->Prepend( $tempdir );

    create_mock_command(
        name        =>  'cmmerge',
        directory   =>  $tempdir,
        sequence    =>  [
            { exitcode => 0, stdout => "cmmerge success lib\n"},
            { exitcode => 0, stdout => "cmmerge success tests\n"},
        ],
    );

    create_mock_command(
        name        =>  'cmreport',
        directory   =>  $tempdir,
        sequence    =>  [
            { exitcode => 0, stdout => "cmreport success\n"},
        ],
    );

    create_mock_command(
        name        =>  'gzip',
        directory   =>  $tempdir,
        sequence    =>  [
            { exitcode => 4, stderr => "gzip failure\n", stdout => ""},
        ],
    );

    my $module_gitdir = catfile( $tempdir, 'module');
    my $test_csmes = catfile( $tempdir, 'tests.csmes' );

    test_run({
        args                =>  [ '--qt-gitmodule-dir', $module_gitdir, '--qt-gitmodule', 'module', '--qtcoverage-tests-output', $test_csmes, 'testname'],
        expected_stdout     =>  $GZIP,
        expected_stderr     =>  qr{gzip failure\n}sm,
        expected_success    =>  0,
        testname            =>  'test gzip failure',
    });

    return;
}

sub test_xml2html_failure
{
    my $tempdir = tempdir( 'testcocoon_runner.XXXXXX', TMPDIR => 1, CLEANUP => 1 );
    init_test_env($tempdir);

    # Put our mock command first in PATH
    local $ENV{ PATH } = $ENV{ PATH };
    Env::Path->PATH->Prepend( $tempdir );

    create_mock_command(
        name        =>  'cmmerge',
        directory   =>  $tempdir,
        sequence    =>  [
            { exitcode => 0, stdout => "cmmerge success lib\n"},
            { exitcode => 0, stdout => "cmmerge success tests\n"},
        ],
    );

    create_mock_command(
        name        =>  'cmreport',
        directory   =>  $tempdir,
        sequence    =>  [
            { exitcode => 0, stdout => "cmreport success\n"},
        ],
    );

    create_mock_command(
        name        =>  'gzip',
        directory   =>  $tempdir,
        sequence    =>  [
            { exitcode => 0, stdout => "gzip success\n"},
        ],
    );

    local $ENV{ TESTING_COVERAGERUNNER } = 1;
    create_mock_command(
        name        =>  'xml2html_testcocoon',
        directory   =>  $tempdir,
        sequence    =>  [
            { exitcode => 5, stderr => "xml2html_testcocoon failure\n"},
        ],
    );

    my $module_gitdir = catfile( $tempdir, 'module');
    my $test_csmes = catfile( $tempdir, 'tests.csmes' );

    test_run({
        args                =>  [ '--qt-gitmodule-dir', $module_gitdir, '--qt-gitmodule', 'module', '--qtcoverage-tests-output', $test_csmes, 'testname'],
        expected_stdout     =>  $XML2HTML,
        expected_stderr     =>  qr{xml2html_testcocoon failure\n}sm,
        expected_success    =>  0,
        testname            =>  'test xml2html failure',
    });

    return;
}

sub test_invalid_qtcoverage_tests_output
{
    my $tempdir = tempdir( 'testcocoon_runner.XXXXXX', TMPDIR => 1, CLEANUP => 1 );
    init_test_env($tempdir);

    my $test_invalid_csmes = catfile( $tempdir, 'tests_invalid.csmes' );

    test_run({
        args                =>  [ '--qt-gitmodule-dir', $tempdir, '--qt-gitmodule', 'module', '--qtcoverage-tests-output', $test_invalid_csmes, 'testname'],
        expected_stdout     =>  q{},
        expected_stderr     =>  $INVALID_TESTS_OUTPUT,
        expected_success    =>  0,
        testname            =>  'invalid csmes output file',
    });

    return;
}

sub init_test_env
{
    my ($tempdir) = @_;

    # Create qtbase path
    my $qtbase_path = catfile($tempdir, 'qtbase');
    if (! -d $qtbase_path && ! mkpath( $qtbase_path )) {
        die "mkpath $qtbase_path: $!";
    }

    # Create plugins path
    my $plugins_path = catfile($qtbase_path, 'plugins');
    if (! -d $plugins_path && ! mkpath( $plugins_path )) {
        die "mkpath $plugins_path: $!";
    }

    # Create "module" path
    my $module_gitdir = catfile( $tempdir, 'module');
    if (! -d $module_gitdir && ! mkpath( $module_gitdir )) {
        die "mkpath $module_gitdir: $!";
    }

    # Create lib path
    my $lib_path = catfile($qtbase_path, 'lib');
    if (! -d $lib_path && ! mkpath( $lib_path )) {
        die "mkpath $lib_path: $!";
    }
    # Create a csmes in lib folder
    open(my $lib, ">", "$lib_path/lib.csmes") or die $!;
    close($lib);
    open(my $lib2, ">", "$lib_path/lib2.csmes") or die $!;
    close($lib2);
    # Create a tests result database csmes
    open(my $test, ">", "$tempdir/tests.csmes") or die $!;
    close($test);

    return;
}

sub run
{
    test_success;
    test_first_merge_failure;
    test_second_merge_failure;
    test_cmreport_failure;
    test_gzip_failure;
    test_invalid_qtcoverage_tests_output;
    test_xml2html_failure;

    done_testing;

    return;
}

run if (!caller);
1;