Skip to content

Commit c20c58e

Browse files
author
[email protected]
committed
[build.webkit.org] Move python code to load config from master.cfg in separate file
https://bugs.webkit.org/show_bug.cgi?id=180278 Reviewed by Daniel Bates. * BuildSlaveSupport/build.webkit.org-config/master.cfg: Moved appropriate code to loadConfig.py * BuildSlaveSupport/build.webkit.org-config/loadConfig.py: Ditto. * BuildSlaveSupport/build.webkit.org-config/steps.py: Removed extra import. git-svn-id: http://svn.webkit.org/repository/webkit/trunk@225433 268f45cc-cd09-0410-ab3c-d52691b4dbfc
1 parent 9377950 commit c20c58e

File tree

4 files changed

+142
-109
lines changed

4 files changed

+142
-109
lines changed

‎Tools/BuildSlaveSupport/build.webkit.org-config/loadConfig.py

+130
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
# Copyright (C) 2017 Apple Inc. All rights reserved.
2+
#
3+
# Redistribution and use in source and binary forms, with or without
4+
# modification, are permitted provided that the following conditions
5+
# are met:
6+
# 1. Redistributions of source code must retain the above copyright
7+
# notice, this list of conditions and the following disclaimer.
8+
# 2. Redistributions in binary form must reproduce the above copyright
9+
# notice, this list of conditions and the following disclaimer in the
10+
# documentation and/or other materials provided with the distribution.
11+
#
12+
# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
13+
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14+
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15+
# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
16+
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
17+
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
18+
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
19+
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
20+
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
21+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22+
23+
from buildbot.buildslave import BuildSlave
24+
from buildbot.scheduler import AnyBranchScheduler, Triggerable
25+
from buildbot.schedulers.forcesched import FixedParameter, ForceScheduler, StringParameter, BooleanParameter
26+
from buildbot.schedulers.filter import ChangeFilter
27+
from buildbot.process import buildstep, factory, properties
28+
29+
from factories import *
30+
31+
import re
32+
import json
33+
import operator
34+
35+
from committer_auth import CommitterAuth
36+
import wkbuild
37+
38+
trunk_filter = ChangeFilter(branch=["trunk", None])
39+
40+
41+
def pickLatestBuild(builder, requests):
42+
return max(requests, key=operator.attrgetter("submittedAt"))
43+
44+
45+
def loadBuilderConfig(c):
46+
# FIXME: These file handles are leaked.
47+
passwords = json.load(open('passwords.json'))
48+
config = json.load(open('config.json'))
49+
50+
c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']]
51+
52+
c['schedulers'] = []
53+
for scheduler in config['schedulers']:
54+
if "change_filter" in scheduler:
55+
scheduler["change_filter"] = globals()[scheduler["change_filter"]]
56+
kls = globals()[scheduler.pop('type')]
57+
# Python 2.6 can't handle unicode keys as keyword arguments:
58+
# http://bugs.python.org/issue2646. Modern versions of json return
59+
# unicode strings from json.load, so we map all keys to str objects.
60+
scheduler = dict(map(lambda key_value_pair: (str(key_value_pair[0]), key_value_pair[1]), scheduler.items()))
61+
62+
c['schedulers'].append(kls(**scheduler))
63+
64+
forceScheduler = ForceScheduler(
65+
name="force",
66+
builderNames=[str(builder['name']) for builder in config['builders']],
67+
reason=StringParameter(name="reason", default="", size=40),
68+
69+
# Validate SVN revision: number or empty string
70+
revision=StringParameter(name="revision", default="", regex=re.compile(r'^(\d*)$')),
71+
72+
# Disable default enabled input fields: branch, repository, project, additional properties
73+
branch=FixedParameter(name="branch"),
74+
repository=FixedParameter(name="repository"),
75+
project=FixedParameter(name="project"),
76+
properties=[BooleanParameter(name="is_clean", label="Force Clean build")]
77+
)
78+
c['schedulers'].append(forceScheduler)
79+
80+
c['builders'] = []
81+
for builder in config['builders']:
82+
for slaveName in builder['slavenames']:
83+
for slave in config['slaves']:
84+
if slave['name'] != slaveName or slave['platform'] == '*':
85+
continue
86+
87+
if slave['platform'] != builder['platform']:
88+
raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform'])
89+
90+
break
91+
92+
platform = builder['platform']
93+
94+
builderType = builder.pop('type')
95+
factory = globals()["%sFactory" % builderType]
96+
factorykwargs = {}
97+
for key in "platform", "configuration", "architectures", "triggers", "additionalArguments", "SVNMirror":
98+
value = builder.pop(key, None)
99+
if value:
100+
factorykwargs[key] = value
101+
102+
builder["factory"] = factory(**factorykwargs)
103+
104+
if platform.startswith('mac'):
105+
builder["category"] = 'AppleMac'
106+
elif platform.startswith('ios'):
107+
builder['category'] = 'iOS'
108+
elif platform == 'win':
109+
builder["category"] = 'AppleWin'
110+
elif platform.startswith('gtk'):
111+
builder["category"] = 'GTK'
112+
elif platform.startswith('wpe'):
113+
builder["category"] = 'WPE'
114+
else:
115+
builder["category"] = 'misc'
116+
117+
if (builder['category'] in ('AppleMac', 'AppleWin', 'iOS')) and builderType != 'Build':
118+
builder['nextBuild'] = pickLatestBuild
119+
120+
c['builders'].append(builder)
121+
122+
123+
class PlatformSpecificScheduler(AnyBranchScheduler):
124+
def __init__(self, platform, branch, **kwargs):
125+
self.platform = platform
126+
filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter)
127+
AnyBranchScheduler.__init__(self, name=platform, change_filter=filter, **kwargs)
128+
129+
def filter(self, change):
130+
return wkbuild.should_build(self.platform, change.files)

‎Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg

+1-107
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,13 @@
11
# -*- python -*-
22
# ex: set syntax=python:
33

4-
from buildbot.buildslave import BuildSlave
54
from buildbot.changes.pb import PBChangeSource
6-
from buildbot.scheduler import AnyBranchScheduler, Triggerable
7-
from buildbot.schedulers.forcesched import FixedParameter, ForceScheduler, StringParameter, BooleanParameter
8-
from buildbot.schedulers.filter import ChangeFilter
95
from buildbot.status import html
106
from buildbot.status.web.authz import Authz
11-
from buildbot.process import buildstep, factory, properties
12-
13-
from factories import *
14-
from twisted.internet import defer
15-
16-
import re
17-
import json
18-
import operator
197

208
from committer_auth import CommitterAuth
21-
import wkbuild
9+
from loadConfig import *
2210

23-
APPLE_WEBKIT_AWS_PROXY = "http://54.190.50.182:873"
24-
S3URL = "https://s3-us-west-2.amazonaws.com/"
2511

2612
c = BuildmasterConfig = {}
2713

@@ -57,96 +43,4 @@ c['eventHorizon'] = 200
5743
c['buildCacheSize'] = 60
5844

5945

60-
def pickLatestBuild(builder, requests):
61-
return max(requests, key=operator.attrgetter("submittedAt"))
62-
63-
64-
class PlatformSpecificScheduler(AnyBranchScheduler):
65-
def __init__(self, platform, branch, **kwargs):
66-
self.platform = platform
67-
filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter)
68-
AnyBranchScheduler.__init__(self, name=platform, change_filter=filter, **kwargs)
69-
70-
def filter(self, change):
71-
return wkbuild.should_build(self.platform, change.files)
72-
73-
trunk_filter = ChangeFilter(branch=["trunk", None])
74-
75-
def loadBuilderConfig(c):
76-
# FIXME: These file handles are leaked.
77-
passwords = json.load(open('passwords.json'))
78-
config = json.load(open('config.json'))
79-
80-
c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']]
81-
82-
c['schedulers'] = []
83-
for scheduler in config['schedulers']:
84-
if "change_filter" in scheduler:
85-
scheduler["change_filter"] = globals()[scheduler["change_filter"]]
86-
kls = globals()[scheduler.pop('type')]
87-
# Python 2.6 can't handle unicode keys as keyword arguments:
88-
# http://bugs.python.org/issue2646. Modern versions of json return
89-
# unicode strings from json.load, so we map all keys to str objects.
90-
scheduler = dict(map(lambda key_value_pair: (str(key_value_pair[0]), key_value_pair[1]), scheduler.items()))
91-
92-
c['schedulers'].append(kls(**scheduler))
93-
94-
forceScheduler = ForceScheduler(
95-
name="force",
96-
builderNames=[str(builder['name']) for builder in config['builders']],
97-
reason=StringParameter(name="reason", default="", size=40),
98-
99-
# Validate SVN revision: number or empty string
100-
revision=StringParameter(name="revision", default="", regex=re.compile(r'^(\d*)$')),
101-
102-
# Disable default enabled input fields: branch, repository, project, additional properties
103-
branch=FixedParameter(name="branch"),
104-
repository=FixedParameter(name="repository"),
105-
project=FixedParameter(name="project"),
106-
properties=[BooleanParameter(name="is_clean", label="Force Clean build")]
107-
)
108-
c['schedulers'].append(forceScheduler)
109-
110-
c['builders'] = []
111-
for builder in config['builders']:
112-
for slaveName in builder['slavenames']:
113-
for slave in config['slaves']:
114-
if slave['name'] != slaveName or slave['platform'] == '*':
115-
continue
116-
117-
if slave['platform'] != builder['platform']:
118-
raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform'])
119-
120-
break
121-
122-
platform = builder['platform']
123-
124-
builderType = builder.pop('type')
125-
factory = globals()["%sFactory" % builderType]
126-
factorykwargs = {}
127-
for key in "platform", "configuration", "architectures", "triggers", "additionalArguments", "SVNMirror":
128-
value = builder.pop(key, None)
129-
if value:
130-
factorykwargs[key] = value
131-
132-
builder["factory"] = factory(**factorykwargs)
133-
134-
if platform.startswith('mac'):
135-
builder["category"] = 'AppleMac'
136-
elif platform.startswith('ios'):
137-
builder['category'] = 'iOS'
138-
elif platform == 'win':
139-
builder["category"] = 'AppleWin'
140-
elif platform.startswith('gtk'):
141-
builder["category"] = 'GTK'
142-
elif platform.startswith('wpe'):
143-
builder["category"] = 'WPE'
144-
else:
145-
builder["category"] = 'misc'
146-
147-
if (builder['category'] in ('AppleMac', 'AppleWin', 'iOS')) and builderType != 'Build':
148-
builder['nextBuild'] = pickLatestBuild
149-
150-
c['builders'].append(builder)
151-
15246
loadBuilderConfig(c)

‎Tools/BuildSlaveSupport/build.webkit.org-config/steps.py

-2
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@
2424
from buildbot.steps import master, shell, source, transfer, trigger
2525
from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED, EXCEPTION
2626

27-
from factories import *
28-
2927
from twisted.internet import defer
3028

3129
import os

‎Tools/ChangeLog

+11
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,14 @@
1+
2017-12-01 Aakash Jain <[email protected]>
2+
3+
[build.webkit.org] Move python code to load config from master.cfg in separate file
4+
https://bugs.webkit.org/show_bug.cgi?id=180278
5+
6+
Reviewed by Daniel Bates.
7+
8+
* BuildSlaveSupport/build.webkit.org-config/master.cfg: Moved appropriate code to loadConfig.py
9+
* BuildSlaveSupport/build.webkit.org-config/loadConfig.py: Ditto.
10+
* BuildSlaveSupport/build.webkit.org-config/steps.py: Removed extra import.
11+
112
2017-12-01 Brian Burg <[email protected]>
213

314
Web Inspector: move Inspector::Protocol::Array<T> to JSON namespace

0 commit comments

Comments
 (0)