improved batching

Author: krausest

webdriver-ts-results/src/results.ts

@@ -13,17 +13,17 @@ import { resolve } from 'dns';
 function forkAndCallBenchmark(frameworks: FrameworkData[], frameworkName: string, keyed: boolean, benchmarkName: string, benchmarkOptions: BenchmarkOptions): Promise<ErrorAndWarning> {
     return new Promise((resolve, reject) => {
         const forked = fork('dist/forkedBenchmarkRunner.js');
-                if (config.LOG_DEBUG) console.log("forked child process");
+                console.log("FORKING:  forked child process");
                 forked.send({ config, frameworks, keyed, frameworkName, benchmarkName, benchmarkOptions });
                 forked.on('message', async (msg: ErrorAndWarning) => {
-                    if (config.LOG_DEBUG) console.log("main process got message from child", msg);
+                    console.log("FORKING: main process got message from child", msg);
                     resolve(msg);
                 });
                 forked.on('close', (msg) => {
-                    if (config.LOG_DEBUG) console.log("child closed", msg);
+                    console.log("FORKING: child closed", msg);
                 });
                 forked.on('error', (msg) => {
-                    if (config.LOG_DEBUG) console.log("child error", msg);
+                    console.log("FORKING: child error", msg);
                     reject(msg);
                 });
                 forked.on('exit', (code, signal) => {
@@ -45,20 +45,28 @@ async function runBenchmakLoop(frameworks: FrameworkData[], frameworkName: strin
         let warnings : String[] = [];
         let errors : String[] = [];
 
-        let results: Array<number|LighthouseData> = [];
+        let results: Array<number[]|LighthouseData> = [];
         let count = 0;
+
         if (benchmark.type == BenchmarkType.CPU) {
             count = benchmarkOptions.numIterationsForCPUBenchmarks;
+            benchmarkOptions.batchSize = config.ALLOW_BATCHING && benchmark.allowBatching ? count : 1;
         } else if (benchmark.type == BenchmarkType.MEM) {
             count = benchmarkOptions.numIterationsForMemBenchmarks;
+            benchmarkOptions.batchSize = 1;
         } else {
             count = benchmarkOptions.numIterationsForStartupBenchmark
+            benchmarkOptions.batchSize = 1;
         }
 
-        for (let i = 0; i < count; i++) {
+
+        while (results.length < count) {
+            benchmarkOptions.batchSize = Math.min(benchmarkOptions.batchSize, count-results.length);
+            console.log("FORKING: ", benchmark.id, " BatchSize ", benchmarkOptions.batchSize);
             let res = await forkAndCallBenchmark(frameworks, frameworkName, keyed, benchmarkName, benchmarkOptions);
             if (res.result) {
-                results.push(res.result);
+                if (Array.isArray(res.result)) { results = results.concat(res.result)}
+                else results.push(res.result);
             }
             warnings = warnings.concat(res.warnings);
             if (res.error) {
@@ -96,14 +104,15 @@ async function runBench(runFrameworks: FrameworkData[], benchmarkNames: string[]
         chromeBinaryPath: args.chromeBinary,
         numIterationsForCPUBenchmarks: config.REPEAT_RUN,
         numIterationsForMemBenchmarks: config.REPEAT_RUN_MEM,
-        numIterationsForStartupBenchmark: config.REPEAT_RUN_STARTUP
+        numIterationsForStartupBenchmark: config.REPEAT_RUN_STARTUP,
+        batchSize: 1
     }
 
     for (let i = 0; i < runFrameworks.length; i++) {
         for (let j = 0; j < runBenchmarks.length; j++) {
             try {
                 let result = await runBenchmakLoop(runFrameworks, runFrameworks[i].name, runFrameworks[i].keyed, runBenchmarks[j].id, benchmarkOptions);
-                errors = warnings.concat(result.errors);
+                errors = errors.concat(result.errors);
                 warnings = warnings.concat(result.warnings);
             } catch (e) {
                 console.log("UNHANDELED ERROR", e);

webdriver-ts-results/table.html

@@ -11,7 +11,8 @@ export interface BenchmarkInfo {
     type: BenchmarkType;
     label: string;
     description: string;
-    throttleCPU?: number
+    throttleCPU?: number;
+    allowBatching: boolean;
 }
 
 export abstract class Benchmark {
@@ -20,13 +21,15 @@ export abstract class Benchmark {
     label: string;
     description: string;
     throttleCPU?: number;
+    allowBatching: boolean;
 
     constructor(public benchmarkInfo: BenchmarkInfo) {
         this.id = benchmarkInfo.id;
         this.type = benchmarkInfo.type;
         this.label = benchmarkInfo.label;
         this.description = benchmarkInfo.description;
         this.throttleCPU = benchmarkInfo.throttleCPU;
+        this.allowBatching = benchmarkInfo.allowBatching;
     }
     abstract init(driver: WebDriver, framework: FrameworkData): Promise<any>;
     abstract run(driver: WebDriver, framework: FrameworkData): Promise<any>;
@@ -54,7 +57,8 @@ const benchRun = new class extends Benchmark {
             id: "01_run1k",
             label: "create rows",
             description: "creating 1,000 rows",
-            type: BenchmarkType.CPU
+            type: BenchmarkType.CPU,
+            allowBatching: true
         })
     }
     async init(driver: WebDriver) { await testElementLocatedById(driver, "add", SHORT_TIMEOUT); }
@@ -70,7 +74,8 @@ const benchReplaceAll = new class extends Benchmark {
             id: "02_replace1k",
             label: "replace all rows",
             description: "updating all 1,000 rows (" + config.WARMUP_COUNT + " warmup runs).",
-            type: BenchmarkType.CPU
+            type: BenchmarkType.CPU,
+            allowBatching: true
         })
     }
     async init(driver: WebDriver) {
@@ -93,7 +98,8 @@ const benchUpdate = new class extends Benchmark {
             label: "partial update",
             description: "updating every 10th row for 1,000 rows (3 warmup runs). 16x CPU slowdown.",
             type: BenchmarkType.CPU,
-            throttleCPU: 16
+            throttleCPU: 16,
+            allowBatching: true
         })
     }
     async init(driver: WebDriver) {
@@ -118,14 +124,15 @@ const benchSelect = new class extends Benchmark {
             label: "select row",
             description: "highlighting a selected row. (" + config.WARMUP_COUNT + " warmup runs). 16x CPU slowdown.",
             type: BenchmarkType.CPU,
-            throttleCPU: 16
+            throttleCPU: 16,
+            allowBatching: true
         })
     }
     async init(driver: WebDriver) {
         await testElementLocatedById(driver, "run", SHORT_TIMEOUT);
         await clickElementById(driver, 'run');
         await testElementLocatedByXpath(driver, "//tbody/tr[1]/td[2]/a");
-        for (let i = 0; i <= config.WARMUP_COUNT; i++) {
+        for (let i = 0; i <= 10; i++) {
             await clickElementByXPath(driver, `//tbody/tr[${i + 1}]/td[2]/a`);
         }
     }
@@ -142,7 +149,8 @@ const benchSwapRows = new class extends Benchmark {
             label: "swap rows",
             description: "swap 2 rows for table with 1,000 rows. (" + config.WARMUP_COUNT + " warmup runs). 4x CPU slowdown.",
             type: BenchmarkType.CPU,
-            throttleCPU: 4
+            throttleCPU: 4,
+            allowBatching: true
         })
     }
     async init(driver: WebDriver) {
@@ -168,7 +176,8 @@ const benchRemove = new class extends Benchmark {
             id: "06_remove-one-1k",
             label: "remove row",
             description: "removing one row. (" + config.WARMUP_COUNT + " warmup runs).",
-            type: BenchmarkType.CPU
+            type: BenchmarkType.CPU,
+            allowBatching: true
         })
     }
     async init(driver: WebDriver) {
@@ -201,7 +210,8 @@ const benchRunBig = new class extends Benchmark {
             id: "07_create10k",
             label: "create many rows",
             description: "creating 10,000 rows",
-            type: BenchmarkType.CPU
+            type: BenchmarkType.CPU,
+            allowBatching: false
         })
     }
     async init(driver: WebDriver) {
@@ -220,7 +230,8 @@ const benchAppendToManyRows = new class extends Benchmark {
             label: "append rows to large table",
             description: "appending 1,000 to a table of 10,000 rows. 2x CPU slowdown",
             type: BenchmarkType.CPU,
-            throttleCPU: 2
+            throttleCPU: 2,
+            allowBatching: false
         })
     }
     async init(driver: WebDriver) {
@@ -241,7 +252,8 @@ const benchClear = new class extends Benchmark {
             label: "clear rows",
             description: "clearing a table with 1,000 rows. 8x CPU slowdown",
             type: BenchmarkType.CPU,
-            throttleCPU: 8
+            throttleCPU: 8,
+            allowBatching: true
         })
     }
     async init(driver: WebDriver) {
@@ -262,6 +274,7 @@ const benchReadyMemory = new class extends Benchmark {
             label: "ready memory",
             description: "Memory usage after page load.",
             type: BenchmarkType.MEM,
+            allowBatching: true
         })
     }
     async init(driver: WebDriver) {
@@ -283,6 +296,7 @@ const benchRunMemory = new class extends Benchmark {
             label: "run memory",
             description: "Memory usage after adding 1000 rows.",
             type: BenchmarkType.MEM,
+            allowBatching: true
         })
     }
     async init(driver: WebDriver) {
@@ -301,6 +315,7 @@ const benchUpdate5Memory = new class extends Benchmark {
             label: "update eatch 10th row for 1k rows (5 cycles)",
             description: "Memory usage after clicking update every 10th row 5 times",
             type: BenchmarkType.MEM,
+            allowBatching: true
         })
     }
     async init(driver: WebDriver) {
@@ -322,6 +337,7 @@ const benchReplace5Memory = new class extends Benchmark {
             label: "replace 1k rows (5 cycles)",
             description: "Memory usage after clicking create 1000 rows 5 times",
             type: BenchmarkType.MEM,
+            allowBatching: true
         })
     }
     async init(driver: WebDriver) {
@@ -342,6 +358,7 @@ const benchCreateClear5Memory = new class extends Benchmark {
             label: "creating/clearing 1k rows (5 cycles)",
             description: "Memory usage after creating and clearing 1000 rows 5 times",
             type: BenchmarkType.MEM,
+            allowBatching: true
         })
     }
     async init(driver: WebDriver) {
@@ -362,31 +379,35 @@ const benchStartupConsistentlyInteractive: StartupBenchmarkResult = {
     label: "consistently interactive",
     description: "a pessimistic TTI - when the CPU and network are both definitely very idle. (no more CPU tasks over 50ms)",
     type: BenchmarkType.STARTUP,
-    property: "TimeToConsistentlyInteractive"
+    property: "TimeToConsistentlyInteractive",
+    allowBatching: true
 }
 
 const benchStartupBootup: StartupBenchmarkResult = {
     id: "32_startup-bt",
     label: "script bootup time",
     description: "the total ms required to parse/compile/evaluate all the page's scripts",
     type: BenchmarkType.STARTUP,
-    property: "ScriptBootUpTtime"
+    property: "ScriptBootUpTtime",
+    allowBatching: true
 }
 
 const benchStartupMainThreadWorkCost: StartupBenchmarkResult = {
     id: "33_startup-mainthreadcost",
     label: "main thread work cost",
     description: "total amount of time spent doing work on the main thread. includes style/layout/etc.",
     type: BenchmarkType.STARTUP,
-    property: "MainThreadWorkCost"
+    property: "MainThreadWorkCost",
+    allowBatching: true
 }
 
 const benchStartupTotalBytes: StartupBenchmarkResult = {
     id: "34_startup-totalbytes",
     label: "total kilobyte weight",
     description: "network transfer cost (post-compression) of all the resources loaded into the page.",
     type: BenchmarkType.STARTUP,
-    property: "TotalKiloByteWeight"
+    property: "TotalKiloByteWeight",
+    allowBatching: true
 }
 
 class BenchStartup extends Benchmark {
@@ -396,6 +417,7 @@ class BenchStartup extends Benchmark {
             label: "startup time",
             description: "Time for loading, parsing and starting up",
             type: BenchmarkType.STARTUP,
+            allowBatching: true
         })
     }
     async init(driver: WebDriver) { // not used with lighthouse

webdriver-ts/results.json

@@ -14,7 +14,7 @@ export type TBenchmarkStatus = 'OK'|'TEST_FAILED'|'TECHNICAL_ERROR';
 export interface ErrorAndWarning {
     error: String;
     warnings: String[];
-    result?: number|LighthouseData;
+    result?: number[]|LighthouseData;
 }
 
 export interface BenchmarkDriverOptions {
@@ -26,17 +26,19 @@ export interface BenchmarkDriverOptions {
 
 export interface BenchmarkOptions extends BenchmarkDriverOptions {
     port: string;
+    batchSize: number;
     numIterationsForCPUBenchmarks: number;
     numIterationsForMemBenchmarks: number;
     numIterationsForStartupBenchmark: number;
+
 }
 
 export let config = {
     PORT: 8080,
     REMOTE_DEBUGGING_PORT: 9999,
     CHROME_PORT: 9998,
     REPEAT_RUN: 10,
-    REPEAT_RUN_MEM: 5,
+    REPEAT_RUN_MEM: 1,
     REPEAT_RUN_STARTUP: 4,
     DROP_WORST_RUN: 0,
     WARMUP_COUNT: 5,
@@ -50,7 +52,8 @@ export let config = {
     STARTUP_SLEEP_DURATION: 1000,
     FORK_CHROMEDRIVER: true,
     WRITE_RESULTS: true,
-    RESULTS_DIRECTORY: "results"
+    RESULTS_DIRECTORY: "results",
+    ALLOW_BATCHING: false
 }
 export type TConfig = typeof config;