Skip to content

Commit 1803c08

Browse files
krausestkrausest
committed
Allow for a second run to check results
1 parent 5c5cacf commit 1803c08

File tree

8 files changed

+229
-52
lines changed

8 files changed

+229
-52
lines changed

‎copy.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,14 @@ var path = require('path');
55

66
if (fs.existsSync("dist")) fs.removeSync("dist");
77
fs.mkdirSync("dist");
8-
fs.mkdirSync("dist"+path.sep+"webdriver-java");
9-
fs.copySync("webdriver-ts"+path.sep+"table.html", "dist"+path.sep+"webdriver-java"+path.sep+"table.html");
8+
fs.mkdirSync("dist"+path.sep+"webdriver-ts");
9+
fs.copySync("webdriver-ts"+path.sep+"table.html", "dist"+path.sep+"webdriver-ts"+path.sep+"table.html");
1010

1111
fs.copySync("index.html", "dist"+path.sep+"index.html");
1212
fs.copySync("css", "dist"+path.sep+"css");
1313

1414
_.each(fs.readdirSync('.'), function(name) {
15-
if(fs.statSync(name).isDirectory() && name[0] !== '.' && name !== 'css' && name !== 'node_modules' && name !== 'webdriver-java' && name !== 'dist') {
15+
if(fs.statSync(name).isDirectory() && name[0] !== '.' && name !== 'css' && name !== 'node_modules' && name !== 'webdriver-ts' && name !== 'dist') {
1616
console.log("dist"+path.sep+name);
1717
fs.mkdirSync("dist"+path.sep+name);
1818
fs.mkdirSync("dist"+path.sep+name+path.sep+"dist");

‎webdriver-ts/check.dot‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
<!DOCTYPE html>
2+
<html lang="en">
3+
<meta charset="utf-8" />
4+
<head>
5+
<title>Table Report</title>
6+
<link rel="stylesheet" href="../css/github-markdown.css">
7+
<style>
8+
body {
9+
overflow:scroll;
10+
}
11+
table.results {
12+
font-size: 10px;
13+
table-layout: fixed;
14+
width: 1px;
15+
overflow:visible;
16+
display: table;
17+
}
18+
table.results th {
19+
width: 60px;
20+
}
21+
table.results th.benchname {
22+
width: 100px;
23+
}
24+
table.results td {
25+
padding: 3px;
26+
}
27+
table.results th {
28+
padding: 3px;
29+
}
30+
.top1 {
31+
background-color: #63BF7C;
32+
}
33+
.top2 {
34+
background-color: #D1D580;
35+
}
36+
.top3 {
37+
background-color: #FFEC84;
38+
}
39+
.top4 {
40+
background-color: #FCAA78;
41+
}
42+
.top5 {
43+
background-color: #F9696C;
44+
}
45+
td {
46+
text-align: center;
47+
}
48+
.rowCount {
49+
font-size: 8px;
50+
}
51+
.deviation {
52+
font-size: 8px;
53+
padding-left: 5px;
54+
}
55+
.deviation:before {
56+
content: '± ';
57+
}
58+
.factor {
59+
font-size: 8px;
60+
}
61+
</style>
62+
</head>
63+
<body class="markdown-body">
64+
<h3>Results Check</h3>
65+
<table class="results">
66+
<thead>
67+
<tr>
68+
<th class='benchname'></th>
69+
{{~it.frameworks :value}}
70+
<th>{{=value}}</th>
71+
{{~}}
72+
</tr>
73+
</thead>
74+
<tbody>
75+
{{~it.resultRow :bench}}
76+
<tr>
77+
<th class='benchname'>{{=bench.name}}
78+
<div class="rowCount">{{=bench.description}}</div>
79+
</th>
80+
{{~bench.tests :test}}
81+
<td style="background-color:{{=test.styleClass}}"><span class="mean">{{=test.result}}</span><span class="deviation">{{=test.origTime}}</span><span class="deviation">{{=test.checkTime}}</span></td>
82+
{{~}}
83+
</tr>
84+
{{~}}
85+
</table>
86+
</body>
87+
</html>

‎webdriver-ts/src/benchmarkRunner.ts‎

Lines changed: 16 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
import * as chrome from 'selenium-webdriver/chrome'
22
import {Builder, WebDriver, promise, logging} from 'selenium-webdriver'
3-
import {BenchmarkType, Benchmark, benchmarks} from './benchmarks'
3+
import {BenchmarkType, Benchmark, benchmarks, fileName} from './benchmarks'
44
import {forProm} from './webdriverAccess'
55

66
import * as fs from 'fs';
77
import * as yargs from 'yargs';
8-
import {JSONResult, config} from './common'
8+
import {JSONResult, config, FrameworkData, frameworks} from './common'
99
var chromedriver:any = require('chromedriver');
1010
var jStat:any = require('jstat').jStat;
1111

@@ -55,42 +55,6 @@ function readLogs(driver: WebDriver): promise.Promise<Timingresult[]> {
5555
});
5656
}
5757

58-
interface FrameworkData {
59-
name: string;
60-
uri: string;
61-
}
62-
63-
function f(name: string, uri: string = null): FrameworkData
64-
{
65-
return {name, uri: uri? uri : name};
66-
}
67-
68-
let frameworks = [
69-
f("angular-v1.5.8"),
70-
f("angular-v2.0.0-rc5"),
71-
f("aurelia-v1.0.0", "aurelia-v1.0.0/dist"),
72-
f("bobril-v4.44.1"),
73-
f("cyclejs-v7.0.0"),
74-
f("domvm-v1.2.10"),
75-
f("inferno-v0.7.26"),
76-
f("kivi-v1.0.0-rc0"),
77-
f("mithril-v0.2.5"),
78-
f("mithril-v1.0.0-alpha"),
79-
f("plastiq-v1.33.0"),
80-
f("preact-v5.7.0"),
81-
f("ractive-v0.7.3"),
82-
f("ractive-edge"),
83-
f("react-lite-v0.15.17"),
84-
f("react-v15.3.1"),
85-
f("react-v15.3.1-mobX-v2.5.0"),
86-
f("riot-v2.6.1"),
87-
f("tsers-v1.0.0"),
88-
f("vanillajs"),
89-
f("vidom-v0.3.18"),
90-
f("vue-v1.0.26"),
91-
f("vue-v2.0.0-beta1")
92-
];
93-
9458
function buildDriver() {
9559
let logPref = new logging.Preferences();
9660
logPref.setLevel(logging.Type.PERFORMANCE, logging.Level.ALL);
@@ -149,7 +113,7 @@ interface Result {
149113
benchmark: Benchmark
150114
}
151115

152-
function writeResult(res: Result) {
116+
function writeResult(res: Result, dir: string) {
153117
let benchmark = res.benchmark;
154118
let framework = res.framework.name;
155119
let data = res.results;
@@ -167,10 +131,10 @@ function writeResult(res: Result) {
167131
"geometricMean": s.geomean(),
168132
"standardDeviation": s.stdev()
169133
}
170-
fs.writeFileSync(`results/${framework}_${benchmark.id}.json`, JSON.stringify(result), {encoding: "utf8"});
134+
fs.writeFileSync(`${dir}/${fileName(framework, benchmark)}`, JSON.stringify(result), {encoding: "utf8"});
171135
}
172136

173-
function runBench(frameworkNames: string[], benchmarkNames: string[]): promise.Promise<any> {
137+
function runBench(frameworkNames: string[], benchmarkNames: string[], dir: string): promise.Promise<any> {
174138
let runFrameworks = frameworks.filter(f => frameworkNames.some(name => f.name.indexOf(name)>-1));
175139
let runBenchmarks = benchmarks.filter(b => benchmarkNames.some(name => b.id.toLowerCase().indexOf(name)>-1));
176140
console.log("Frameworks that will be benchmarked", runFrameworks);
@@ -196,7 +160,7 @@ function runBench(frameworkNames: string[], benchmarkNames: string[]): promise.P
196160
})
197161
.then(results => reduceBenchmarkResults(benchmark, results))
198162
.then(results => {
199-
writeResult({framework: framework, results: results, benchmark: benchmark});
163+
writeResult({framework: framework, results: results, benchmark: benchmark}, dir);
200164
})
201165
.thenFinally(() => {console.log("QUIT"); driver.quit();})
202166
});
@@ -205,17 +169,24 @@ function runBench(frameworkNames: string[], benchmarkNames: string[]): promise.P
205169
let args = yargs(process.argv)
206170
.usage("$0 [--framework Framework1,Framework2,...] [--benchmark Benchmark1,Benchmark2,...]")
207171
.help('help')
172+
.default('check','false')
208173
.array("framework").array("benchmark").argv;
209174

175+
console.log(args);
176+
210177
let runBenchmarks = args.benchmark && args.benchmark.length>0 ? args.benchmark : [""];
211178
let runFrameworks = args.framework && args.framework.length>0 ? args.framework : [""];
212179

213-
if (!fs.existsSync("results"))
214-
fs.mkdirSync("results");
180+
let dir = args.check === 'true' ? "results_check" : "results"
181+
182+
console.log("target directory", dir);
183+
184+
if (!fs.existsSync(dir))
185+
fs.mkdirSync(dir);
215186

216187
if (args.help) {
217188
yargs.showHelp();
218189
} else {
219-
runBench(runFrameworks, runBenchmarks);
190+
runBench(runFrameworks, runBenchmarks, dir);
220191
}
221192

‎webdriver-ts/src/benchmarks.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -204,4 +204,8 @@ export let benchmarks : [ Benchmark ] = [
204204
benchClear2nd,
205205
benchReadyMemory,
206206
benchRunMemory
207-
];
207+
];
208+
209+
export function fileName(framework: string, benchmark: Benchmark) {
210+
return `${framework}_${benchmark.id}.json`;
211+
}

‎webdriver-ts/src/common.ts‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,40 @@ export let config = {
1717
LOG_PROGRESS: true,
1818
LOG_DETAILS: false,
1919
LOG_DEBUG: false
20-
}
20+
}
21+
22+
export interface FrameworkData {
23+
name: string;
24+
uri: string;
25+
}
26+
27+
function f(name: string, uri: string = null): FrameworkData
28+
{
29+
return {name, uri: uri? uri : name};
30+
}
31+
32+
export let frameworks = [
33+
f("angular-v1.5.8"),
34+
f("angular-v2.0.0-rc5"),
35+
f("aurelia-v1.0.0", "aurelia-v1.0.0/dist"),
36+
f("bobril-v4.44.1"),
37+
f("cyclejs-v7.0.0"),
38+
f("domvm-v1.2.10"),
39+
f("inferno-v0.7.26"),
40+
f("kivi-v1.0.0-rc0"),
41+
f("mithril-v0.2.5"),
42+
f("mithril-v1.0.0-alpha"),
43+
f("plastiq-v1.33.0"),
44+
f("preact-v5.7.0"),
45+
f("ractive-v0.7.3"),
46+
f("ractive-edge"),
47+
f("react-lite-v0.15.17"),
48+
f("react-v15.3.1"),
49+
f("react-v15.3.1-mobX-v2.5.0"),
50+
f("riot-v2.6.1"),
51+
f("tsers-v1.0.0"),
52+
f("vanillajs"),
53+
f("vidom-v0.3.18"),
54+
f("vue-v1.0.26"),
55+
f("vue-v2.0.0-beta1")
56+
];

‎webdriver-ts/src/runCheck.ts‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
import * as fs from 'fs';
2+
import {BenchmarkType, Benchmark, benchmarks, fileName} from './benchmarks'
3+
import {JSONResult, config, FrameworkData, frameworks} from './common'
4+
5+
const dots = require('dot').process({
6+
path: './'
7+
});
8+
9+
function parse(fileName: string) {
10+
return <JSONResult>JSON.parse(fs.readFileSync(fileName, {
11+
encoding:'utf-8'
12+
}));
13+
}
14+
15+
class ResultCheck {
16+
constructor(public result: string, public styleClass: string, public origTime: string, public checkTime: string) {}
17+
}
18+
19+
class CheckResultList {
20+
tests: Array<ResultCheck>;
21+
name: string;
22+
description: string;
23+
24+
constructor(benchmark: Benchmark, tests: Array<ResultCheck>) {
25+
this.tests = tests;
26+
this.name = benchmark.label;
27+
this.description = benchmark.description;
28+
}
29+
}
30+
31+
function color(factor:number): string {
32+
if (factor < 0.1) {
33+
let a = factor / 0.1;
34+
let r = (1.0-a)* 99 + a * 255;
35+
let g = (1.0-a)* 191 + a * 236;
36+
let b = (1.0-a)* 124 + a * 132;
37+
return `rgb(${r.toFixed(0)}, ${g.toFixed(0)}, ${b.toFixed(0)})`
38+
} else {
39+
let a = Math.min(factor - 0.1, 1.0);
40+
let r = (1.0-a)* 255 + a * 249;
41+
let g = (1.0-a)* 236 + a * 105;
42+
let b = (1.0-a)* 132 + a * 108;
43+
return `rgb(${r.toFixed(0)}, ${g.toFixed(0)}, ${b.toFixed(0)})`
44+
}
45+
}
46+
47+
let resultRow : Array<CheckResultList> = [];
48+
for (let i= 0;i<benchmarks.length;i++) {
49+
let benchmark = benchmarks[i];
50+
let row : Array<ResultCheck> = [];
51+
for (let j= 0;j<frameworks.length;j++) {
52+
let framework = frameworks[j];
53+
let origRunName = `results/${fileName(framework.name, benchmark)}`;
54+
let checkRunName = `results_check/${fileName(framework.name, benchmark)}`;
55+
if (fs.existsSync(origRunName) && fs.existsSync(checkRunName))
56+
{
57+
let origRun = parse(origRunName);
58+
let checkRun = parse(checkRunName);
59+
let difference = (checkRun.mean - origRun.mean) / ((origRun.mean + checkRun.mean) * 0.5);
60+
console.log("both files exist for ",framework.name, benchmark.id, difference, origRun.mean, checkRun.mean);
61+
row.push(new ResultCheck((difference * 100).toFixed(1)+"%", color(Math.abs(difference)), origRun.mean.toFixed(0)+"±"+origRun.standardDeviation.toFixed(1), checkRun.mean.toFixed(0)+"±"+checkRun.standardDeviation.toFixed(1)));
62+
} else if (fs.existsSync(origRunName)) {
63+
row.push(new ResultCheck("no file in results_check", "#f00", "", ""));
64+
} else if (fs.existsSync(checkRunName)) {
65+
row.push(new ResultCheck("no file in results", "#f00", "", ""));
66+
} else {
67+
row.push(new ResultCheck("no file at all", "#f00", "", ""));
68+
}
69+
}
70+
resultRow.push(new CheckResultList(benchmark, row));
71+
}
72+
73+
fs.writeFileSync('./check.html', dots.check({
74+
frameworks: frameworks.map(framework => framework.name.replace('-v', ' v')),
75+
resultRow,
76+
}), {
77+
encoding: 'utf8'
78+
})

‎webdriver-ts/table.html‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎webdriver-ts/tsconfig.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@
1515
"./src/benchmarks.ts",
1616
"./src/webdriverAccess.ts",
1717
"./src/benchmarkRunner.ts",
18-
"./src/createResultTable.ts"
18+
"./src/createResultTable.ts",
19+
"./src/runCheck.ts"
1920
],
2021
"exclude": [
2122
"node_modules"

0 commit comments

Comments
 (0)