summaryrefslogtreecommitdiffstats
path: root/scripts/jenkins/gerrit-notify-jenkins.pl
blob: 2a17ca91e3393f040e8a856580014a060837d48c (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
#!/usr/bin/env perl
# Copyright (C) 2017 The Qt Company Ltd.
# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0

=head1 NAME

gerrit-notify-jenkins.pl - notify Jenkins about Gerrit updates, without polling

=head1 SYNOPSIS

  ./gerrit-notify-jenkins.pl \
    --gerrit-url ssh://gerrit.example.com:29418/ \
    --jenkins-url http://jenkins.example.com/jenkins

Connects to 'gerrit stream-events' on gerrit.example.com port 29418 and invokes
the git SCM plugin's notifyCommit URL on jenkins.example.com each time a ref
is updated.

This script runs an infinite loop and will attempt to re-connect to gerrit any
time an error occurs.

Logging of this script may be configured by the PERL_ANYEVENT_VERBOSE and
PERL_ANYEVENT_LOG environment variables (see 'perldoc AnyEvent::Log')

=head2 OPTIONS

=over

=item --gerrit-url <URL>

The ssh URL for gerrit.

It must be possible to invoke 'gerrit stream-events' over ssh to this host and port.

=item --jenkins-url <URL>

Base URL of Jenkins.

=back

=head2 NOTIFIED URLS

The Jenkins notifyCommit mechanism expects the following URL to be activated when
changes occur to some relevant git repository:

  <jenkins_url>/git/notifyCommit?url=<git_url>

However, there are many possible URLs referring to the same git repository;
for example, an ssh URL with or without hostname, or a URL using an alias set up
in .ssh/config or .gitconfig, or http vs https vs ssh URLs for the same repository.

This script has no way of knowing which git URLs are being tracked by Jenkins.
Therefore, it notifies of all commonly used git URL styles for gerrit, including:

  ssh://<gerrit_host>:<gerrit_port>/<gerrit_project>
  ssh://<gerrit_host>:<gerrit_port>/<gerrit_project>.git
  ssh://<gerrit_host>/<gerrit_project>
  ssh://<gerrit_host>/<gerrit_project>.git
  http://<gerrit_host>/p/<gerrit_project>
  http://<gerrit_host>/p/<gerrit_project>.git
  https://<gerrit_host>/p/<gerrit_project>
  https://<gerrit_host>/p/<gerrit_project>.git

Apart from the minor additional network traffic, it is harmless to notify for
unused git URLs.

If the notifications appear to be not working, check that the relevant Jenkins
projects are using a URL matching one of the above forms.

=head2 JENKINS SETUP

Jenkins must be set up using the Git SCM plugin (at least version 1.1.14) and
SCM polling must be enabled. The notification mechanism works by activating the
polling, so it won't do anything if polling is disabled. Of course, the poll
frequency should be low, otherwise there is little benefit from using this script.

It is recommended not to rely on this script as the sole mechanism for triggering
Jenkins builds, since it is always possible for events to be lost (e.g. if the
connection to gerrit or Jenkins is temporarily interrupted). A poll schedule
like the following is a good compromise:

  H */2 * * *

This will cause Jenkins to poll the repository once every two hours (at a random
minute of the hour). Therefore, in the unusual case of events being lost, Jenkins
would still determine that a change has occurred within a maximum of two hours.

=cut

package QtQA::App::GerritNotifyJenkins;
use strict;
use warnings;

use AnyEvent::HTTP;
use AnyEvent::Handle;
use AnyEvent::Util;
use Coro::AnyEvent;
use Coro;
use Data::Dumper;
use English qw( -no_match_vars );
use File::Spec::Functions;
use FindBin;
use Getopt::Long qw( GetOptionsFromArray );
use Pod::Usage;
use URI;

use lib catfile( $FindBin::Bin, qw(.. lib perl5) );
use QtQA::Gerrit;

# Given a gerrit $project (e.g. 'qt/qtbase'), returns a list of all git URLs commonly
# used to refer to that project (e.g. ssh with port number, ssh without port number,
# http, https, ...)
sub generate_urls
{
    my ($self, $project) = @_;

    my $base = URI->new( $self->{ gerrit_url } );

    my @out;

    # ssh without port
    push @out, 'ssh://' . $base->host() . $base->path() . "/$project";

    # ssh with port
    if ($base->port()) {
        push @out, 'ssh://' . $base->host() . ':' . $base->port() . $base->path() . "/$project";
    }

    # http
    push @out, 'http://' . $base->host() . $base->path() . '/p/' . $project;

    # https
    push @out, 'https://' . $base->host() . $base->path() . '/p/' . $project;

    @out = (
        @out,
        map { "$_.git" } @out,
    );

    return @out;
}

# Try hard to do a successful http_get to $url.
#
# Most kinds of errors will cause the request to be retried, repeatedly.
# Will eventually die if not successful.
#
# Blocking; expected to be called from within a coro.
#
sub robust_http_get
{
    my ($url) = @_;

    my $MAX_ATTEMPTS = 8;
    my $MAX_SLEEP = 60;

    my $attempt = 1;
    my $sleep = 2;

    while (1) {
        http_get( $url, Coro::rouse_cb() );
        my (undef, $headers) = Coro::rouse_wait();
        if ($headers->{ Status } =~ m{^2}) {
            # success!
            last;
        }

        my $error = "[attempt $attempt]: $headers->{ Status } $headers->{ Reason }";
        ++$attempt;

        if ($attempt > $MAX_ATTEMPTS) {
            die "failed after repeated attempts. Last error: $error\n";
        }

        AE::log(warn => "$error, trying again in $sleep seconds");

        Coro::AnyEvent::sleep( $sleep );

        $sleep *= 2;
        if ($sleep > $MAX_SLEEP) {
            $sleep = $MAX_SLEEP;
        }
    }

    return;
}

# Notify Jenkins of updates to $project.
#
# This will (asychronously) hit all URLs returned by generate_urls.
#
sub do_notify_commit
{
    my ($self, $project) = @_;

    my @gerrit_urls = $self->generate_urls( $project );
    my $notify_commit_url = URI->new( $self->{ jenkins_url } . '/git/notifyCommit' );

    # spawn all HTTP requests async, don't bother waiting for them
    foreach my $gerrit_url (@gerrit_urls) {
        async {
            my $url = $notify_commit_url->clone();
            $url->query_form( url => $gerrit_url );

            eval {
                robust_http_get( $url->as_string() );
            };
            if (my $error = $EVAL_ERROR) {
                AE::log(warn => "notify to $url failed: $error\n");
            } else {
                AE::log(debug => "notified $url");
            }
        }
    }

    return;
}

# Process an $event seen from gerrit stream-events.
#
# The $event has already been parsed from JSON into perl data (a hashref is expected).
#
sub handle_event
{
    my ($self, $event) = @_;

    # only hashes are expected
    if (ref($event) ne 'HASH') {
        AE::log(warn => 'unexpected gerrit event: ' . Dumper( $event ) . "\n");
        return;
    }

    # ref-updated is the only interesting event for us
    if ($event->{ type } ne 'ref-updated') {
        return;
    }

    my $project = $event->{ refUpdate }{ project };
    my $ref = $event->{ refUpdate }{ refName };

    AE::log(debug => "$ref updated on $project, spawning notifyCommit");

    $self->do_notify_commit( $project );

    return;
}

# Main loop.
#
# Connect to gerrit stream-events and process the events.
#
# This should never exit. It will repeatedly re-connect to gerrit if the connection is disrupted.
sub do_stream_events
{
    my ($self) = @_;

    my $watcher = QtQA::Gerrit::stream_events(
        url => $self->{ gerrit_url },
        on_event => sub {
            my (undef, $data) = @_;
            $self->handle_event( $data );
        },
    );

    # In normal usage, this is the only output.
    # This is just to give some confidence to the user that we're doing anything at all...
    print "Entering main loop.\n";

    AE::cv()->recv();

    AE::log(error => 'internal error: main loop unexpectedly finished');
    return;
}

# Entry point.
sub run
{
    my ($self, @args) = @_;

    GetOptionsFromArray(
        \@args,
        'help|h' => sub { pod2usage(2) },
        'gerrit-url=s' => \$self->{ gerrit_url },
        'jenkins-url=s' => \$self->{ jenkins_url },
    ) || die $!;

    if (!$self->{ gerrit_url }) {
        die "Missing mandatory --gerrit-url argument\n";
    }
    $self->{ gerrit_url } =~ s{/+\z}{};

    if (!$self->{ jenkins_url }) {
        die "Missing mandatory --jenkins-url argument\n";
    }
    $self->{ jenkins_url } =~ s{/+\z}{};

    local $OUTPUT_AUTOFLUSH = 1;

    $self->do_stream_events();

    return;
}

sub new
{
    my ($class) = @_;
    return bless {}, $class;
}


QtQA::App::GerritNotifyJenkins->new( )->run( @ARGV ) unless caller;
1;