Skip to content

Commit 35ac3f3

Browse files
tboschtbosch
committed
feat(perf): measure error and stop automatically when the numbers are good enough.
1 parent e5dbc69 commit 35ac3f3

File tree

2 files changed

+171
-107
lines changed

2 files changed

+171
-107
lines changed

‎protractor-perf-shared.js‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,26 @@ var config = exports.config = {
33
specs: ['modules/*/test/**/*_perf.js'],
44

55
params: {
6-
// number test iterations to warm up the browser
7-
warmupCount: 10,
8-
// number test iterations to measure
9-
measureCount: 10,
10-
// TODO(tbosch): remove this and provide a proper protractor integration
11-
sleepInterval: process.env.TRAVIS ? 5000 : 1000,
6+
// size of the sample to take
7+
sampleSize: 10,
8+
// error to be used for early exit
9+
exitOnErrorLowerThan: 4,
10+
// maxium number times the benchmark gets repeated before we output the stats
11+
// of the best sample
12+
maxRepeatCount: 30
1213
},
1314

1415
// Disable waiting for Angular as we don't have an integration layer yet...
1516
// TODO(tbosch): Implement a proper debugging API for Ng2.0, remove this here
1617
// and the sleeps in all tests.
1718
onPrepare: function() {
1819
browser.ignoreSynchronization = true;
20+
var _get = browser.get;
21+
var sleepInterval = process.env.TRAVIS ? 5000 : 1000;
22+
browser.get = function() {
23+
browser.sleep(sleepInterval);
24+
return _get.apply(this, arguments);
25+
}
1926
},
2027

2128
jasmineNodeOpts: {

‎tools/perf/util.js‎

Lines changed: 158 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,92 @@ var webdriver = require('protractor/node_modules/selenium-webdriver');
22

33
module.exports = {
44
perfLogs: perfLogs,
5-
sumTimelineStats: sumTimelineStats,
5+
sumTimelineRecords: sumTimelineRecords,
66
runSimpleBenchmark: runSimpleBenchmark,
77
verifyNoErrors: verifyNoErrors,
88
printObjectAsMarkdown: printObjectAsMarkdown
99
};
1010

11+
// TODO: rename into runSimpleBenchmark
12+
function runSimpleBenchmark(config) {
13+
// TODO: move this into the tests!
14+
browser.get(config.url);
15+
16+
var buttons = config.buttons.map(function(selector) {
17+
return $(selector);
18+
});
19+
var globalParams = browser.params;
20+
21+
// empty perflogs queue and gc
22+
gc();
23+
perfLogs();
24+
var sampleQueue = [];
25+
var bestSampleStats = null;
26+
27+
loop(globalParams.maxRepeatCount).then(function(stats) {
28+
printObjectAsMarkdown(config.name, stats);
29+
});
30+
31+
function loop(count) {
32+
if (!count) {
33+
return bestSampleStats;
34+
}
35+
return webdriver.promise.all(buttons.map(function(button) {
36+
// Note: even though we remove the gc time from the script time,
37+
// we still get a high standard devication if we don't gc after every click...
38+
return button.click().then(gc);
39+
})).then(function() {
40+
return perfLogs();
41+
}).then(function(logs) {
42+
var stats = calculateStatsBasedOnLogs(logs);
43+
if (stats) {
44+
if (stats.script.error < globalParams.exitOnErrorLowerThan) {
45+
return stats;
46+
}
47+
if (!bestSampleStats || stats.script.error < bestSampleStats.script.error) {
48+
bestSampleStats = stats;
49+
}
50+
}
51+
return loop(count-1);
52+
});
53+
}
54+
55+
function calculateStatsBasedOnLogs(logs) {
56+
sampleQueue.push(sumTimelineRecords(logs['Timeline.eventRecorded']));
57+
if (sampleQueue.length >= globalParams.sampleSize) {
58+
sampleQueue.splice(0, sampleQueue.length - globalParams.sampleSize);
59+
// TODO: gc numbers don't have much meaning right now,
60+
// as a benchmark run destroys everything.
61+
// We need to measure the heap size after gc as well!
62+
return calculateObjectSampleStats(sampleQueue, ['script', 'render', 'gcTime', 'gcAmount']);
63+
}
64+
return null;
65+
}
66+
}
67+
68+
function gc() {
69+
// TODO(tbosch): this only works on chrome, and we actually should
70+
// extend chromedriver to use the Debugger.CollectGarbage call of the
71+
// remote debugger protocol.
72+
// See http://src.chromium.org/viewvc/blink/trunk/Source/devtools/protocol.json
73+
// For iOS Safari we need an extension to appium that uses
74+
// the webkit remote debug protocol. See
75+
// https://github.com/WebKit/webkit/blob/master/Source/WebInspectorUI/Versions/Inspector-iOS-8.0.json
76+
return browser.executeScript('window.gc()');
77+
}
78+
79+
function verifyNoErrors() {
80+
browser.manage().logs().get('browser').then(function(browserLog) {
81+
var filteredLog = browserLog.filter(function(logEntry) {
82+
return logEntry.level.value > webdriver.logging.Level.WARNING.value;
83+
});
84+
expect(filteredLog.length).toEqual(0);
85+
if (filteredLog.length) {
86+
console.log('browser console errors: ' + require('util').inspect(filteredLog));
87+
}
88+
});
89+
}
90+
1191
function perfLogs() {
1292
return plainLogs('performance').then(function(entries) {
1393
var entriesByMethod = {};
@@ -34,115 +114,56 @@ function plainLogs(type) {
34114
};
35115

36116

37-
function sumTimelineStats(messages) {
117+
function sumTimelineRecords(messages) {
38118
var recordStats = {
39119
script: 0,
40-
gc: {
41-
time: 0,
42-
amount: 0
43-
},
120+
gcTime: 0,
121+
gcAmount: 0,
44122
render: 0
45123
};
46124
messages.forEach(function(message) {
47-
sumTimelineRecordStats(message.record, recordStats);
125+
processRecord(message.record, recordStats);
48126
});
49127
return recordStats;
50-
}
51-
52-
function sumTimelineRecordStats(record, result) {
53-
var summedChildrenDuration = 0;
54-
if (record.children) {
55-
record.children.forEach(function(child) {
56-
summedChildrenDuration += sumTimelineRecordStats(child, result);
57-
});
58-
}
59-
// in case a script forced a gc or a reflow
60-
// we need to substract the gc time / reflow time
61-
// from the script time!
62-
var recordDuration = (record.endTime ? record.endTime - record.startTime : 0)
63-
- summedChildrenDuration;
64-
65-
var recordSummed = true;
66-
if (record.type === 'FunctionCall') {
67-
result.script += recordDuration;
68-
} else if (record.type === 'GCEvent') {
69-
result.gc.time += recordDuration;
70-
result.gc.amount += record.data.usedHeapSizeDelta;
71-
} else if (record.type === 'RecalculateStyles' ||
72-
record.type === 'Layout' ||
73-
record.type === 'UpdateLayerTree' ||
74-
record.type === 'Paint' ||
75-
record.type === 'Rasterize' ||
76-
record.type === 'CompositeLayers') {
77-
result.render += recordDuration;
78-
} else {
79-
recordSummed = false;
80-
}
81-
if (recordSummed) {
82-
return recordDuration;
83-
} else {
84-
return summedChildrenDuration;
85-
}
86-
}
87-
88-
function runSimpleBenchmark(config) {
89-
var url = config.url;
90-
var buttonSelectors = config.buttons;
91-
// TODO: Don't use a fixed number of warmup / measure iterations,
92-
// but make this dependent on the variance of the test results!
93-
var warmupCount = browser.params.warmupCount;
94-
var measureCount = browser.params.measureCount;
95-
var name = config.name;
96-
97-
browser.get(url);
98-
// TODO(tbosch): replace this with a proper protractor/ng2.0 integration
99-
// and remove this function as well as all method calls.
100-
browser.sleep(browser.params.sleepInterval)
101-
102-
var btns = buttonSelectors.map(function(selector) {
103-
return $(selector);
104-
});
105128

106-
multiClick(btns, warmupCount);
107-
gc();
108-
// empty perflogs queue
109-
perfLogs();
110-
111-
multiClick(btns, measureCount);
112-
gc();
113-
return perfLogs().then(function(logs) {
114-
var stats = sumTimelineStats(logs['Timeline.eventRecorded']);
115-
printObjectAsMarkdown(name, stats);
116-
return stats;
117-
});
118-
}
119-
120-
function gc() {
121-
// TODO(tbosch): this only works on chrome.
122-
// For iOS Safari we need an extension to appium...
123-
browser.executeScript('window.gc()');
124-
}
125-
126-
function multiClick(buttons, count) {
127-
var actions = browser.actions();
128-
for (var i=0; i<count; i++) {
129-
buttons.forEach(function(button) {
130-
actions.click(button);
131-
});
132-
}
133-
actions.perform();
134-
}
129+
function processRecord(record, recordStats) {
130+
var summedChildrenDuration = 0;
131+
if (record.children) {
132+
record.children.forEach(function(child) {
133+
summedChildrenDuration += processRecord(child, recordStats);
134+
});
135+
}
135136

136-
function verifyNoErrors() {
137-
browser.manage().logs().get('browser').then(function(browserLog) {
138-
var filteredLog = browserLog.filter(function(logEntry) {
139-
return logEntry.level.value > webdriver.logging.Level.WARNING.value;
140-
});
141-
expect(filteredLog.length).toEqual(0);
142-
if (filteredLog.length) {
143-
console.log('browser console errors: ' + require('util').inspect(filteredLog));
137+
var recordDuration;
138+
var recordUsed = false;
139+
if (recordStats) {
140+
// we need to substract the time of child records
141+
// that have been added to the stats from this record.
142+
// E.g. for a script record that triggered a gc or reflow while executing.
143+
recordDuration = (record.endTime ? record.endTime - record.startTime : 0)
144+
- summedChildrenDuration;
145+
if (record.type === 'FunctionCall') {
146+
if (!record.data || record.data.scriptName !== 'InjectedScript') {
147+
// ignore scripts that were injected by Webdriver (e.g. calculation of element positions, ...)
148+
recordStats.script += recordDuration;
149+
recordUsed = true;
150+
}
151+
} else if (record.type === 'GCEvent') {
152+
recordStats.gcTime += recordDuration;
153+
recordStats.gcAmount += record.data.usedHeapSizeDelta;
154+
recordUsed = true;
155+
} else if (record.type === 'RecalculateStyles' ||
156+
record.type === 'Layout' ||
157+
record.type === 'UpdateLayerTree' ||
158+
record.type === 'Paint' ||
159+
record.type === 'Rasterize' ||
160+
record.type === 'CompositeLayers') {
161+
recordStats.render += recordDuration;
162+
recordUsed = true;
163+
}
144164
}
145-
});
165+
return recordUsed ? recordDuration : summedChildrenDuration;
166+
}
146167
}
147168

148169
function printObjectAsMarkdown(name, obj) {
@@ -176,4 +197,40 @@ function printObjectAsMarkdown(name, obj) {
176197
}
177198
}
178199
}
179-
}
200+
}
201+
202+
function calculateObjectSampleStats(objectSamples, properties) {
203+
var result = {};
204+
properties.forEach(function(prop) {
205+
var samples = objectSamples.map(function(objectSample) {
206+
return objectSample[prop];
207+
});
208+
var mean = calculateMean(samples);
209+
var error = calculateCoefficientOfVariation(samples, mean);
210+
result[prop] = {
211+
mean: mean,
212+
error: error
213+
};
214+
});
215+
return result;
216+
}
217+
218+
function calculateCoefficientOfVariation(sample, mean) {
219+
return calculateStandardDeviation(sample, mean) / mean * 100;
220+
}
221+
222+
function calculateMean(sample) {
223+
var total = 0;
224+
sample.forEach(function(x) { total += x; });
225+
return total / sample.length;
226+
}
227+
228+
function calculateStandardDeviation(sample, mean) {
229+
var deviation = 0;
230+
sample.forEach(function(x) {
231+
deviation += Math.pow(x - mean, 2);
232+
});
233+
deviation = deviation / (sample.length -1);
234+
deviation = Math.sqrt(deviation);
235+
return deviation;
236+
};

0 commit comments

Comments
 (0)