added t-test for results

krausest · krausest · commit 2abf9e409f15 · 2017-06-18T22:02:06.000+02:00
diff --git a/webdriver-ts-results/package.json b/webdriver-ts-results/package.json
@@ -42,6 +42,7 @@
   },
   "dependencies": {
     "babel-polyfill": "^6.23.0",
+    "jStat": "^1.6.2",
     "react": "15.5.4",
     "react-dom": "15.5.4"
   }
diff --git a/webdriver-ts-results/src/App.tsx b/webdriver-ts-results/src/App.tsx
@@ -21,6 +21,7 @@ interface State {
   separateKeyedAndNonKeyed: boolean;
   resultTables: Array<ResultTableData>;
   sortKey: string;
+  compareWith: Framework | undefined;
 }
 
 let allBenchmarks = () => benchmarks.reduce((set, b) => set.add(b), new Set() );
@@ -37,13 +38,13 @@ class App extends React.Component<{}, State> {
         event.preventDefault();
         let set = this.state.selectedBenchmarks;
         benchmarks.forEach(b => {if ((b.type === BenchmarkType.MEM) === memBenchmarks) set.add(b);});
-        this.setState({selectedBenchmarks: set, resultTables: this.updateResultTable(set, this.state.selectedFrameworks, this.state.separateKeyedAndNonKeyed, this.state.sortKey)});
+        this.setState({selectedBenchmarks: set, resultTables: this.updateResultTable(set, this.state.selectedFrameworks, this.state.separateKeyedAndNonKeyed, this.state.sortKey, this.state.compareWith)});
       },
       selectNone: (event: React.SyntheticEvent<any>) => {    
         event.preventDefault();
         let set = this.state.selectedBenchmarks;
         benchmarks.forEach(b => {if ((b.type === BenchmarkType.MEM) === memBenchmarks) set.delete(b);});
-        this.setState({selectedBenchmarks: set, sortKey: SORT_BY_NAME, resultTables: this.updateResultTable(set, this.state.selectedFrameworks, this.state.separateKeyedAndNonKeyed, SORT_BY_NAME)});
+        this.setState({selectedBenchmarks: set, sortKey: SORT_BY_NAME, resultTables: this.updateResultTable(set, this.state.selectedFrameworks, this.state.separateKeyedAndNonKeyed, SORT_BY_NAME, this.state.compareWith)});
       },
       areAllSelected: () => benchmarks.filter(b => memBenchmarks ? b.type === BenchmarkType.MEM : b.type !== BenchmarkType.MEM)
                               .every(b => this.state.selectedBenchmarks.has(b)),
@@ -56,13 +57,13 @@ class App extends React.Component<{}, State> {
         event.preventDefault();
         let set = this.state.selectedFrameworks;
         frameworks.forEach(framework => {if (framework.nonKeyed === nonKeyed && !set.has(framework)) set.add(framework);});
-        this.setState({selectedFrameworks: set, resultTables: this.updateResultTable(this.state.selectedBenchmarks, set, this.state.separateKeyedAndNonKeyed, this.state.sortKey)});
+        this.setState({selectedFrameworks: set, resultTables: this.updateResultTable(this.state.selectedBenchmarks, set, this.state.separateKeyedAndNonKeyed, this.state.sortKey, this.state.compareWith)});
       },
       selectNone: (event: React.SyntheticEvent<any>) => {    
         event.preventDefault();
         let set = this.state.selectedFrameworks;
         set.forEach(framework => {if (framework.nonKeyed === nonKeyed) set.delete(framework);});
-        this.setState({selectedFrameworks: set, resultTables: this.updateResultTable(this.state.selectedBenchmarks, set, this.state.separateKeyedAndNonKeyed, this.state.sortKey)});
+        this.setState({selectedFrameworks: set, resultTables: this.updateResultTable(this.state.selectedBenchmarks, set, this.state.separateKeyedAndNonKeyed, this.state.sortKey, this.state.compareWith)});
       },
       areAllSelected: () => frameworks.filter(f => f.nonKeyed===nonKeyed).every(f => this.state.selectedFrameworks.has(f)),
       isNoneSelected: () => frameworks.filter(f => f.nonKeyed===nonKeyed).every(f => !this.state.selectedFrameworks.has(f)),
@@ -84,8 +85,9 @@ class App extends React.Component<{}, State> {
                   selectedBenchmarks: _allBenchmarks,
                   selectedFrameworks: _allFrameworks,
                   separateKeyedAndNonKeyed: true,
-                  resultTables: this.updateResultTable(_allBenchmarks, _allFrameworks, true, SORT_BY_NAME),
-                  sortKey: SORT_BY_NAME     
+                  resultTables: this.updateResultTable(_allBenchmarks, _allFrameworks, true, SORT_BY_NAME, undefined),
+                  sortKey: SORT_BY_NAME,
+                  compareWith: undefined                  
                 };
   }
   selectBenchmark = (benchmark: Benchmark, value: boolean) => {
@@ -97,29 +99,39 @@ class App extends React.Component<{}, State> {
     let setIds = new Set();
     set.forEach(b => setIds.add(b.id))
     if ((sortKey!=SORT_BY_NAME && sortKey!=SORT_BY_GEOMMEAN) && !setIds.has(sortKey)) sortKey = SORT_BY_NAME;
-    this.setState({selectedBenchmarks: set, sortKey, resultTables: this.updateResultTable(set, this.state.selectedFrameworks, this.state.separateKeyedAndNonKeyed, sortKey)});
+    this.setState({selectedBenchmarks: set, sortKey, resultTables: this.updateResultTable(set, this.state.selectedFrameworks, this.state.separateKeyedAndNonKeyed, sortKey, this.state.compareWith)});
   }
   selectFramework = (framework: Framework, value: boolean): void => {
     let set = new Set();
     this.state.selectedFrameworks.forEach(framework => set.add(framework));
     if (set.has(framework)) set.delete(framework);
     else set.add(framework);
-    this.setState({selectedFrameworks: set, resultTables: this.updateResultTable(this.state.selectedBenchmarks, set, this.state.separateKeyedAndNonKeyed, this.state.sortKey)});
+    this.setState({selectedFrameworks: set, resultTables: this.updateResultTable(this.state.selectedBenchmarks, set, this.state.separateKeyedAndNonKeyed, this.state.sortKey, this.state.compareWith)});
   }  
   selectSeparateKeyedAndNonKeyed = (value: boolean): void => {
-    this.setState({separateKeyedAndNonKeyed: value, resultTables: this.updateResultTable(this.state.selectedBenchmarks, this.state.selectedFrameworks, value, this.state.sortKey)});
+    this.setState({separateKeyedAndNonKeyed: value, resultTables: this.updateResultTable(this.state.selectedBenchmarks, this.state.selectedFrameworks, value, this.state.sortKey, this.state.compareWith)});
   }
-  updateResultTable(selectedBenchmarks: Set<Benchmark>, selectedFrameworks: Set<Framework>, separateKeyedAndNonKeyed: boolean, sortKey: string) {
+  updateResultTable(selectedBenchmarks: Set<Benchmark>, selectedFrameworks: Set<Framework>, separateKeyedAndNonKeyed: boolean, sortKey: string, compareWith: Framework|undefined) {
     if (separateKeyedAndNonKeyed) {
-      return [new ResultTableData(frameworks, benchmarks, resultLookup, selectedFrameworks, selectedBenchmarks, false, sortKey),
-              new ResultTableData(frameworks, benchmarks, resultLookup, selectedFrameworks, selectedBenchmarks, true, sortKey)]      
+      return [new ResultTableData(frameworks, benchmarks, resultLookup, selectedFrameworks, selectedBenchmarks, false, sortKey, compareWith),
+              new ResultTableData(frameworks, benchmarks, resultLookup, selectedFrameworks, selectedBenchmarks, true, sortKey, compareWith)]      
     } else {
-      return [new ResultTableData(frameworks, benchmarks, resultLookup, selectedFrameworks, selectedBenchmarks, undefined, sortKey)]
+      return [new ResultTableData(frameworks, benchmarks, resultLookup, selectedFrameworks, selectedBenchmarks, undefined, sortKey, compareWith)]
     }
   }
+  selectComparison = (framework: string): void => {
+    let compareWith: Framework | undefined = undefined;
+    compareWith = this.state.frameworksKeyed.find((f) => f.name === framework);
+    if (!compareWith) {
+      compareWith = this.state.frameworksNonKeyed.find((f) => f.name === framework);
+    }
+    console.log("compareWith", compareWith);
+    this.setState({compareWith:compareWith, resultTables: this.updateResultTable(this.state.selectedBenchmarks, this.state.selectedFrameworks, this.state.separateKeyedAndNonKeyed, this.state.sortKey, compareWith)});
+  }
+
   sortBy = (sortkey: string, tableIdx: number): void => {
     this.state.resultTables[tableIdx].sortBy(sortkey);
-    this.setState({sortKey:sortkey, resultTables: this.updateResultTable(this.state.selectedBenchmarks, this.state.selectedFrameworks, this.state.separateKeyedAndNonKeyed, sortkey)});
+    this.setState({sortKey:sortkey, resultTables: this.updateResultTable(this.state.selectedBenchmarks, this.state.selectedFrameworks, this.state.separateKeyedAndNonKeyed, sortkey, this.state.compareWith)});
   }
   render() {
     let disclaimer = (false) ? (<div>
@@ -143,7 +155,15 @@ class App extends React.Component<{}, State> {
                     selectBenchmark={this.selectBenchmark}
                     selectFramework={this.selectFramework}
                     selectSeparateKeyedAndNonKeyed={this.selectSeparateKeyedAndNonKeyed}
-                    separateKeyedAndNonKeyed={this.state.separateKeyedAndNonKeyed}/>
+                    separateKeyedAndNonKeyed={this.state.separateKeyedAndNonKeyed}
+                    compareWith={this.state.compareWith}
+                    selectComparison={this.selectComparison}
+                    />
+          {!this.state.compareWith ? null :           
+          (<p style={{marginTop:'10px'}}>In comparison mode white cells mean there's no statistically significant difference. 
+            Green cells are significantly faster than the comparison and red cells are slower. 
+            The test is performed as a one sided t-test. The significance level is 10%. The darker the color the lower the p-Value.</p>
+          )}
           <ResultTable currentSortKey={this.state.sortKey} data={this.state.resultTables} separateKeyedAndNonKeyed={this.state.separateKeyedAndNonKeyed} sortBy={this.sortBy}/>
       </div>
     );
diff --git a/webdriver-ts-results/src/Common.tsx b/webdriver-ts-results/src/Common.tsx
@@ -1,4 +1,5 @@
 import * as React from 'react';
+var jStat:any = require('jstat').jStat;
 
 export interface Framework {
     name: string;
@@ -22,6 +23,7 @@ export interface Result {
     min: number;
     max: number;
     mean: number;
+    count?: number;
     geometricMean: number;
     standardDeviation: number;
 }
@@ -59,11 +61,13 @@ let computeColor = function(factor: number): string {
 
 export class TableResultValueEntry implements TableResultEntry {
     color: string;
-    constructor(public key:string, public mean: number, public standardDeviation: number, public factor: number) {
+    constructor(public key:string, public mean: number, public standardDeviation: number, public factor: number, public statisticallySignificantFactor: string|number|undefined, public statisticalCol: [string,string]|undefined) {
         this.color = computeColor(factor);
     }
     render() {
-        return (<td key={this.key} style={{backgroundColor:this.color}}>
+        let col = this.statisticalCol === undefined ? this.color : this.statisticalCol[0];
+        let textCol = this.statisticalCol === undefined ? '#000' : this.statisticalCol[1];
+        return (<td key={this.key} style={{backgroundColor:col, color: textCol}}>
                     <span className="mean">{this.mean.toFixed(1)}</span>
                     <span className="deviation">{this.standardDeviation.toFixed(1)}</span>
                     <br />
@@ -101,6 +105,25 @@ export function convertToMap(results: Array<Result>): ResultLookup {
     }
 }
 
+let statisticComputeColor = function(sign: number, pValue: number): [string, string] {
+    if (pValue < 0.90) {
+        return ['#fff','#000'];
+    } 
+    if (sign < 0) {
+        let a = (pValue - 0.9) * 10.0;
+        let r = 0;
+        let g = (1.0-a)* 255 + a * 160;
+        let b = 0;
+        return [`rgb(${r.toFixed(0)}, ${g.toFixed(0)}, ${b.toFixed(0)})`, '#fff'];
+    } else  {
+        let a = (pValue - 0.9) * 10.0;
+        let r = (1.0-a)* 255 + a * 160;
+        let g = 0;
+        let b = 0;
+        return [`rgb(${r.toFixed(0)}, ${g.toFixed(0)}, ${b.toFixed(0)})`, '#fff'];
+    }
+}
+
 export class ResultTableData {
     // Rows
     benchmarksCPU: Array<Benchmark>;
@@ -113,7 +136,8 @@ export class ResultTableData {
     resultsMEM: Array<Array<TableResultValueEntry|null>>;
 
     constructor(public allFrameworks: Array<Framework>, public allBenchmarks: Array<Benchmark>, public results: ResultLookup, 
-        public selectedFrameworks: Set<Framework>, public selectedBenchmarks: Set<Benchmark>, nonKeyed: boolean|undefined, sortKey: string) {
+        public selectedFrameworks: Set<Framework>, public selectedBenchmarks: Set<Benchmark>, nonKeyed: boolean|undefined, sortKey: string,
+        public compareWith: Framework|undefined) {
         this.frameworks = this.allFrameworks.filter(framework => (nonKeyed===undefined || framework.nonKeyed === nonKeyed) && selectedFrameworks.has(framework));
         this.update(sortKey);
     }
@@ -176,6 +200,7 @@ export class ResultTableData {
     }
     computeFactors(benchmark: Benchmark, clamp: boolean): Array<TableResultValueEntry|null> {
         let benchmarkResults = this.frameworks.map(f => this.results(benchmark, f));
+        let compareWithResults = this.compareWith ? this.results(benchmark, this.compareWith) : undefined;
         let min = benchmarkResults.reduce((min, result) => result===null ? min : Math.min(min,result.mean), Number.POSITIVE_INFINITY);
         return this.frameworks.map(f => {
             let result = this.results(benchmark, f);
@@ -184,7 +209,35 @@ export class ResultTableData {
                 let mean = result.mean;
                 let factor = clamp ? Math.max(16, result.mean) / Math.max(16, min) : result.mean/min;
                 let standardDeviation = result.standardDeviation;
-                return new TableResultValueEntry(f.name, mean, standardDeviation, factor);
+
+                let statisticalResult = undefined;
+                let statisticalCol = undefined;
+                // X1,..,Xn: this Framework, Y1, ..., Yn: selected Framework
+                // https://de.wikipedia.org/wiki/Zweistichproben-t-Test
+                if (compareWithResults) {
+                    let n = result.count || 20;
+                    let m = result.count || 20;
+                    let s2 = (n-1)*Math.pow(result.standardDeviation,2) + (m-1)*Math.pow(compareWithResults.standardDeviation,2) / (n+m-2)
+                    let s = Math.sqrt(s2);
+                    let t = Math.sqrt((n * m) / (n + m))*(result.mean - compareWithResults.mean - 0)/s;                            
+                    let talpha = jStat.studentt.inv( 0.975, n + m - 2);
+                    statisticalResult = t.toFixed(3) + ": " +talpha.toFixed(3);
+                    let p = jStat.studentt.cdf( Math.abs(t), n + m -2 );
+                    statisticalCol = statisticComputeColor(t, p);
+                    statisticalResult += ": " + p.toFixed(3) + ";" + statisticalCol;
+                    if (result.mean < compareWithResults.mean) {
+                        // H0 X >= Y, H1 X < Y
+                        if (t < -talpha) {
+                            statisticalResult += " faster";
+                        }
+                    } else {
+                        // H0 X <= Y, H1 X > Y
+                        if (t > talpha) {
+                            statisticalResult += " slower";
+                        }
+                    }
+                }
+                return new TableResultValueEntry(f.name, mean, standardDeviation, factor, statisticalResult, statisticalCol);
             }
         });
     }
diff --git a/webdriver-ts-results/src/SelectBar.tsx b/webdriver-ts-results/src/SelectBar.tsx
@@ -19,6 +19,8 @@ export interface Props {
   frameworksNonKeyed: Array<Framework>;
   benchmarksCPU: Array<Benchmark>;
   benchmarksMEM: Array<Benchmark>;
+  compareWith: Framework | undefined;
+  selectComparison: (framework: string) => void;
 }
 
 export class SelectBar extends React.Component<Props, void> {
@@ -34,7 +36,10 @@ export class SelectBar extends React.Component<Props, void> {
           frameworksKeyed,
           frameworksNonKeyed,
           benchmarksCPU,
-          benchmarksMEM} = this.props;
+          benchmarksMEM,
+          compareWith,
+          selectComparison
+      } = this.props;
         return (
           <div>
             <DropDown label="Which frameworks?" width='1024px'>
@@ -73,6 +78,29 @@ export class SelectBar extends React.Component<Props, void> {
                   Separate keyed and non-keyed
                 </label>
             </div>
+            <div className="hspan"/>
+            <form className="form-inline" style={{display:"inline-block"}}>
+              <div className="form-group">
+                  <div className="hspan"/>
+                  <select className="form-control" value={compareWith ? compareWith.name : ''} onChange={(evt) => selectComparison(evt.target.value)}>
+                    <option value=''>Compare with ...</option>                        
+                    <optgroup label="Keyed">
+                      {
+                        frameworksKeyed.map(f => 
+                            <option key={f.name} value={f.name}>{f.name}</option>                        
+                        )
+                      }
+                    </optgroup>
+                    <optgroup label="Non-keyed">
+                      {
+                        frameworksNonKeyed.map(f => 
+                            <option key={f.name} value={f.name}>{f.name}</option>                        
+                        )
+                      }
+                    </optgroup>
+                  </select>
+              </div>
+            </form>
           </div>
         );
     }
diff --git a/webdriver-ts-results/table.html b/webdriver-ts-results/table.html

Original file line number	Diff line number	Diff line change
`@@ -42,6 +42,7 @@`
`42`	`42`	`},`
`43`	`43`	`"dependencies": {`
`44`	`44`	`"babel-polyfill": "^6.23.0",`
	`45`	`+ "jStat": "^1.6.2",`
`45`	`46`	`"react": "15.5.4",`
`46`	`47`	`"react-dom": "15.5.4"`
`47`	`48`	`}`