path: root/doc/src/qtopiamail_performance.qdocperl

/*!

\perl{
use Carp;

# Try to use DB connection when possible.
eval q{
    use QaDb;
    $db = QaDb::database_connect;
    eval q{
        $branch = QaDb::QPE_BRANCH();
    };
    if ($@) {
        $branch = "4.5";
        $@ = undef;
    }
    sub unary_query {
        my $query = shift;
        my %attr;
        my @ary = @{$db->selectall_arrayref($query, \%attr, @_)};
        if (scalar @ary && scalar @{$ary[0]}) {
            return $ary[0]->[0];
        }
        else {
            return undef;
        }
    }

    $build_id = unary_query(q{
SELECT b.id,pv.timestamp
FROM   builds b
    JOIN performanceValues pv ON pv.builds_id = b.id
    JOIN performanceThings pt ON pv.performanceThings_id = pt.id
    JOIN qtopiaBranches qb ON b.qtopiaBranches_id = qb.id
WHERE pt.name = 'tst_messageserver:completeRetrievalImap:heap_usage:small_messages--100_1'
    AND qb.name = ?
ORDER BY b.qtopiaTimestamp DESC, pv.timestamp DESC LIMIT 1
    }, $branch);
    $commit = unary_query('SELECT qtopiaChange FROM builds WHERE id = ?', $build_id);

    sub result_from_db {
        my $thing = shift;
        my $result = unary_query(q{
SELECT AVG(pv.value)
FROM   performanceValues pv
    JOIN performanceThings pt ON pt.id = pv.performanceThings_id
    JOIN builds b ON b.id = pv.builds_id
    JOIN qtopiaBranches qb ON qb.id = b.qtopiaBranches_id
WHERE pt.name = ?
    AND qb.name = ?
GROUP BY b.id
ORDER BY b.qtopiaTimestamp DESC, pv.timestamp DESC LIMIT 1
        }, $thing, $branch);
        (defined $result) or return $result;
        return sprintf "%d", $result;
    }

    eval q{
        use Memoize;
        memoize('result_from_db');
        memoize('unary_query');
    };

};

if ($@) {
    # Failed to connect to database - fall back on hardcoded values
    $commit = "fb6155f2fed80d87a3c00c8b4c7dbe83faab4197";
    $branch = "4.5";
    $build_id = undef;
}

# Retrieve result for a particular test.
# Pulls results from QaDb or falls back to hardcoded results.
sub result {
    my $thing = shift;
    my $unit = shift;

    my $tst = 'tst_messageserver:completeRetrievalImap';
    my %results = (
"$tst:heap_usage:small_messages--100_1"  => 1422,
"$tst:heap_usage:small_messages--200_1"  => 1538,
"$tst:heap_usage:small_messages--500_1"  => 1960,
"$tst:heap_usage:small_messages--1000_1" => 2579,
"$tst:heap_usage:small_messages--2000_1" => 3840,
"$tst:heap_usage:small_messages--5000_1" => 7648,
"$tst:heap_usage:small_messages--10000_1" => 13993,

"$tst:heap_usage:big_messages_1" => 1285,

"$tst:disk_usage:small_messages--100_1"  => 774,
"$tst:disk_usage:small_messages--200_1"  => 1522,
"$tst:disk_usage:small_messages--500_1"  => 3742,
"$tst:disk_usage:small_messages--1000_1" => 7487,
"$tst:disk_usage:small_messages--2000_1" => 14940,
"$tst:disk_usage:small_messages--5000_1" => 37300,
"$tst:disk_usage:small_messages--10000_1" => 74375,

"$tst:disk_usage:big_messages_1" => 8157,

"$tst:walltime:small_messages--100_1"   => 7133,
"$tst:walltime:small_messages--200_1"   => 13898,
"$tst:walltime:small_messages--500_1"   => 35241,
"$tst:walltime:small_messages--1000_1"  => 70305,
"$tst:walltime:small_messages--2000_1"  => 157786,
"$tst:walltime:small_messages--5000_1"  => 467195,
"$tst:walltime:small_messages--10000_1" => 1044617,

"$tst:walltime:big_messages_1" => 3990,
    );

    $tst = 'tst_messageserver:removeMessages';
    %results = (%results, 
"$tst:heap_usage:big_messages_1"         =>        31,
"$tst:heap_usage:small_messages--100_1"  =>        71,
"$tst:heap_usage:small_messages--200_1"  =>       126,
"$tst:heap_usage:small_messages--500_1"  =>       276,
"$tst:disk_usage:small_messages--100_1"  =>        -651,
"$tst:disk_usage:small_messages--200_1"  =>       -1301,
"$tst:disk_usage:small_messages--500_1"  =>       -2148,
"$tst:walltime:big_messages_1"           =>        16,
"$tst:walltime:small_messages--100_1"   =>         33,
"$tst:walltime:small_messages--200_1"   =>         54,
"$tst:walltime:small_messages--500_1"    =>       120,
      );

    $tst = 'tst_messageserver:replaceMessages';
    %results = (%results, 
"$tst:disk_usage:big_messages_1"         =>       0,
"$tst:disk_usage:small_messages--100_1"  =>       0,
"$tst:disk_usage:small_messages--200_1"  =>       0,
"$tst:disk_usage:small_messages--500_1"  =>       0,
      );

    # Try to get result from database first.

    # We're paying for earlier poor design choices now... if the thing is particularly
    # small, it'll be recorded per 1M or 1K iterations and postfixed with M or K.
    # Check for that first.
    eval "\$out = result_from_db '${thing}M'";
    if (defined $out) {
        return ($out/1000000) . ($unit ? " $unit" : "");
    }

    eval "\$out = result_from_db '${thing}K'";
    if (defined $out) {
        return ($out/1000) . ($unit ? " $unit" : "");
    }

    eval "\$out = result_from_db '$thing'";
    if ($@) {
        if ($FORCE) { confess "Failed to get $thing result from database: ", $@; }
    }
    elsif (defined $out) {
        return $out . ($unit ? " $unit" : "");
    }

    # Failed... use hardcoded value.
    if (exists $results{$thing}) {
        return $results{$thing} . ($unit ? " $unit" : "");
    }
    else {
        return "N/A";
    }
}

sub dataset_size {
    my $thing = shift;
    my $unit = shift;

    my %size = (
        "small_messages--100"   => 646,
        "small_messages--200"   => 1331,
        "small_messages--500"   => 3174,
        "small_messages--1000"  => 6451,
        "small_messages--2000"  => 12800,
        "small_messages--5000"  => 31948,
        "small_messages--10000" => 63897,
        "big_messages"          => 7987,
    );
    if (exists $size{$thing}) {
        return $size{$thing} . ($unit ? " $unit" : "");
    }
    else {
        return "N/A";
    }
}

# Returns a new xy plot object.
sub chart {
    eval 'use Chart::LinesPoints';
    if ($@) {
        if ($FORCE) { confess $@; }
        return undef;
    }
    my $ch = Chart::LinesPoints->new(700, 500);
    $ch->set('legend'         => 'none');
    $ch->set('xy_plot'        => 'true');
    $ch->set('skip_x_ticks'   => 100);
    $ch->set('skip_y_ticks'   => 20);
    $ch->set('y_grid_lines'   => 'true');
    $ch->set('transparent'    => 'true');
    $ch->set('precision'      => '0');
    return $ch;
}

"";
}

\page qtopiamail_performance.html
\title Qt Extended Messaging Performance

\section1 Introduction
The performance of the Qt Extended Messaging library is tested using the reference \c{messageserver} implementation included with Qt Extended.  Various tasks are performed using different amounts and sizes of message to determine the effect these parameters have on the library's performance.

\section1 Tests

A series of tests are defined and each test has several metrics measured during execution.
\table
\header \o Test         \o Description
\row
    \o completeRetrievalImap
    \o Retrieval of the full content of a set of messages using IMAP.
\row
    \o removeMessages
    \o Deleting a set of messages from local storage.
\row
    \o replaceMessages
    \o Deleting a set of messages from local storage, then redownloading the same set of messages.
\endtable

\table
\header \o Metric       \o Description
\row
    \o Heap usage
    \o The peak amount of memory allocated on the heap during the test.
\row
    \o Filesystem usage
    \o The increase in filesystem usage during the test, in the directory used to store mail data.
\row
    \o Execution time
    \o The amount of time taken to perform the test.
\endtable

\section1 Testdata

The following table describes the testdata used during the tests.
The total size refers to the sizes of all messages added together, which can be significantly
less than the storage size on disk - for example, if stored in maildir format on an ext2
filesystem, there may be as much as 4 KB of additional wasted space for each stored message.

In all cases, where portions of the message are Base64 encoded, the size refers to the encoded size.

All of the messages in the various \c{small_messages} datasets are extracted from the same corpus.

\table
\header \o Name         \o Description  \o Total size
\row
    \o small_messages--100
    \o 100 plaintext e-mails of typical length with no attachments.
    \o \perl dataset_size('small_messages--100','KB')
\row
    \o small_messages--200
    \o 200 plaintext e-mails.
    \o \perl dataset_size('small_messages--200','KB')
\row
    \o small_messages--500
    \o 500 plaintext e-mails.
    \o \perl dataset_size('small_messages--500','KB')
\row
    \o small_messages--1000
    \o 1000 plaintext e-mails.
    \o \perl dataset_size('small_messages--1000','KB')
\row
    \o small_messages--2000
    \o 2000 plaintext e-mails.
    \o \perl dataset_size('small_messages--2000','KB')
\row
    \o small_messages--5000
    \o 5000 plaintext e-mails.
    \o \perl dataset_size('small_messages--5000','KB')
\row
    \o small_messages--10000
    \o 10000 plaintext e-mails.
    \o \perl dataset_size('small_messages--10000','KB')
\row
    \o big_messages
    \o 3 e-mails with large attachments, including plaintext and binary.
    \o \perl dataset_size('big_messages','KB')
\endtable

\section1 Results

Last test: Qt Extended \perl{$branch}, commit \perl{$commit}.

\section2 completeRetrievalImap

\table
\header \o Dataset         \o Heap usage   \o Filesystem usage \o Execution time
\perl { $tst = 'tst_messageserver:completeRetrievalImap'; ""; }
\row
    \o small_messages--100
    \o \perl result("$tst:heap_usage:small_messages--100_1","KB")
    \o \perl result("$tst:disk_usage:small_messages--100_1","KB")
    \o \perl result("$tst:walltime:small_messages--100_1","ms")
\row
    \o small_messages--200
    \o \perl result("$tst:heap_usage:small_messages--200_1","KB")
    \o \perl result("$tst:disk_usage:small_messages--200_1","KB")
    \o \perl result("$tst:walltime:small_messages--200_1","ms")
\row
    \o small_messages--500
    \o \perl result("$tst:heap_usage:small_messages--500_1","KB")
    \o \perl result("$tst:disk_usage:small_messages--500_1","KB")
    \o \perl result("$tst:walltime:small_messages--500_1","ms")
\row
    \o small_messages--1000
    \o \perl result("$tst:heap_usage:small_messages--1000_1","KB")
    \o \perl result("$tst:disk_usage:small_messages--1000_1","KB")
    \o \perl result("$tst:walltime:small_messages--1000_1","ms")
\row
    \o small_messages--2000
    \o \perl result("$tst:heap_usage:small_messages--2000_1","KB")
    \o \perl result("$tst:disk_usage:small_messages--2000_1","KB")
    \o \perl result("$tst:walltime:small_messages--2000_1","ms")
\row
    \o small_messages--5000
    \o \perl result("$tst:heap_usage:small_messages--5000_1","KB")
    \o \perl result("$tst:disk_usage:small_messages--5000_1","KB")
    \o \perl result("$tst:walltime:small_messages--5000_1","ms")
\row
    \o small_messages--10000
    \o \perl result("$tst:heap_usage:small_messages--10000_1","KB")
    \o \perl result("$tst:disk_usage:small_messages--10000_1","KB")
    \o \perl result("$tst:walltime:small_messages--10000_1","ms")
\row
    \o big_messages
    \o \perl result("$tst:heap_usage:big_messages_1","KB")
    \o \perl result("$tst:disk_usage:big_messages_1","KB")
    \o \perl result("$tst:walltime:big_messages_1","ms")
\endtable

\perl{
    my $ch = chart();
    $ch or return "(chart not generated: Chart.pm is not available and -force was not specified)";

    $ch->set('title'          => "Small messages retrieval");
    $ch->set('sub_title'      => "Heap usage");
    $ch->set('x_label'        => "Message count");
    $ch->set('y_label'        => "Heap size (KB)");
    $ch->add_dataset(100, 200, 500, 1000, 2000, 5000, 10000);
    {
        my $r = sub {
            my $num = shift;
            return result("tst_messageserver:completeRetrievalImap:heap_usage:small_messages--${num}_1");
        };

        $ch->add_dataset($r->(100), $r->(200), $r->(500),
            $r->(1000), $r->(2000), $r->(5000), $r->(10000));
    }

    my $imgname = "perfchart_heap_usage_completeRetrieval_small_messages.png";
    mkdir "$GENERATED_SOURCE/images";
    $ch->png("$GENERATED_SOURCE/images/$imgname");

    "\\image $imgname";
}

\perl{
    my $ch = chart();
    $ch or return "(chart not generated: Chart.pm is not available and -force was not specified)";

    $ch->set('title'          => "Small messages retrieval");
    $ch->set('sub_title'      => "Filesystem wastage");
    $ch->set('x_label'        => "Message count");
    $ch->set('y_label'        => "Size (KB)");
    $ch->set('legend'         => 'right');
    $ch->set('legend_labels'  => ["Message size", "Size on disk"]);
    {
        my $msg = sub {
            my $num = shift;
            return dataset_size("small_messages--${num}");
        };
        my $fs = sub {
            my $num = shift;
            return result("tst_messageserver:completeRetrievalImap:disk_usage:small_messages--${num}_1");
        };

        $ch->add_dataset(100, 200, 500,
            1000, 2000, 5000, 10000);
        $ch->add_dataset($msg->(100), $msg->(200), $msg->(500),
            $msg->(1000), $msg->(2000), $msg->(5000), $msg->(10000));
        $ch->add_dataset($fs->(100),  $fs->(200),  $fs->(500),
            $fs->(1000), $fs->(2000), $fs->(5000), $fs->(10000));
    }

    my $imgname = "perfchart_disk_usage_completeRetrieval_small_messages.png";
    mkdir "$GENERATED_SOURCE/images";
    $ch->png("$GENERATED_SOURCE/images/$imgname");

    "\\image $imgname";
}

\perl{
    my $ch = chart();
    $ch or return "(chart not generated: Chart.pm is not available and -force was not specified)";

    $ch->set('title'          => "Small messages retrieval");
    $ch->set('sub_title'      => "Execution time");
    $ch->set('x_label'        => "Message count");
    $ch->set('y_label'        => "Time (s)");
    $ch->add_dataset(100, 200, 500,
        1000, 2000, 5000, 10000);
    {
        my $r = sub {
            my $num = shift;
            return result("tst_messageserver:completeRetrievalImap:walltime:small_messages--${num}_1")/1000;
        };

        $ch->add_dataset($r->(100), $r->(200), $r->(500),
            $r->(1000), $r->(2000), $r->(5000), $r->(10000));
    }

    my $imgname = "perfchart_walltime_completeRetrieval_small_messages.png";
    mkdir "$GENERATED_SOURCE/images";
    $ch->png("$GENERATED_SOURCE/images/$imgname");

    "\\image $imgname";
}


\section2 removeMessages

Results for messages >= 1000 are blocked by task 236253.

\table
\header \o Dataset         \o Heap usage   \o Filesystem usage \o Execution time
\perl { $tst = 'tst_messageserver:removeMessages'; ""; }
\row
    \o small_messages--100
    \o \perl result("$tst:heap_usage:small_messages--100_1","KB")
    \o \perl result("$tst:disk_usage:small_messages--100_1","KB")
    \o \perl result("$tst:walltime:small_messages--100_1","ms")
\row
    \o small_messages--200
    \o \perl result("$tst:heap_usage:small_messages--200_1","KB")
    \o \perl result("$tst:disk_usage:small_messages--200_1","KB")
    \o \perl result("$tst:walltime:small_messages--200_1","ms")
\row
    \o small_messages--500
    \o \perl result("$tst:heap_usage:small_messages--500_1","KB")
    \o \perl result("$tst:disk_usage:small_messages--500_1","KB")
    \o \perl result("$tst:walltime:small_messages--500_1","ms")
\row
    \o big_messages
    \o \perl result("$tst:heap_usage:big_messages_1","KB")
    \o \perl result("$tst:disk_usage:big_messages_1","KB")
    \o \perl result("$tst:walltime:big_messages_1","ms")
\endtable

\perl{
    my $ch = chart();
    $ch or return "(chart not generated: Chart.pm is not available and -force was not specified)";

    $ch->set('title'          => "Small messages removal");
    $ch->set('sub_title'      => "Filesystem wastage");
    $ch->set('x_label'        => "Message count");
    $ch->set('y_label'        => "Size (KB)");
    $ch->set('legend'         => 'right');
    $ch->set('legend_labels'  => ["Space used by download", "Space freed by removal"]);
    {
        my $fs = sub {
            my $num = shift;
            return result("tst_messageserver:completeRetrievalImap:disk_usage:small_messages--${num}_1");
        };
        my $free = sub {
            my $num = shift;
            return -result("tst_messageserver:removeMessages:disk_usage:small_messages--${num}_1");
        };

        $ch->add_dataset(100, 200, 500);
        $ch->add_dataset($fs->(100),   $fs->(200),   $fs->(500));
        $ch->add_dataset($free->(100), $free->(200), $free->(500));
    }

    my $imgname = "perfchart_disk_usage_removeMessages_small_messages.png";
    mkdir "$GENERATED_SOURCE/images";
    $ch->png("$GENERATED_SOURCE/images/$imgname");

    "\\image $imgname";
}

\section2 replaceMessages

The intention of this test is to demonstrate that, although removing messages does not free all of the space used to store metadata about those messages, the space will be reused when messages are next downloaded.
Therefore, expected results are 0 KB for all datasets.

Results for messages >= 1000 are blocked by task 236253.

\table
\header \o Dataset         \o Filesystem usage
\perl { $tst = 'tst_messageserver:replaceMessages'; ""; }
\row
    \o small_messages--100
    \o \perl result("$tst:disk_usage:small_messages--100_1","KB")
\row
    \o small_messages--200
    \o \perl result("$tst:disk_usage:small_messages--200_1","KB")
\row
    \o small_messages--500
    \o \perl result("$tst:disk_usage:small_messages--500_1","KB")
\row
    \o big_messages
    \o \perl result("$tst:disk_usage:big_messages_1","KB")
\endtable


*/