Merge branch 'master' of github.com:krausest/js-framework-benchmark

Author: krausest

webdriver-ts/package-lock.json

@@ -18,35 +18,35 @@
   "author": "",
   "license": "Apache-2.0",
   "devDependencies": {
-    "@types/lodash": "4.14.191",
-    "@types/node": "18.14.6",
-    "@types/ramda": "0.28.23",
-    "@types/selenium-webdriver": "4.1.12",
+    "@types/lodash": "4.14.194",
+    "@types/node": "18.16.1",
+    "@types/ramda": "0.29.0",
+    "@types/selenium-webdriver": "4.1.13",
     "@types/semver": "7.3.13",
     "@types/underscore": "1.11.4",
-    "@types/yargs": "17.0.22",
-    "@typescript-eslint/eslint-plugin": "5.54.0",
-    "@typescript-eslint/parser": "5.54.0",
-    "dockerode": "3.3.4",
-    "eslint": "8.35.0",
+    "@types/yargs": "17.0.24",
+    "@typescript-eslint/eslint-plugin": "5.59.1",
+    "@typescript-eslint/parser": "5.59.1",
+    "dockerode": "3.3.5",
+    "eslint": "8.39.0",
     "typescript": "4.9.5"
   },
   "dependencies": {
-    "axios": "1.3.4",
-    "chromedriver": "110.0.0",
+    "axios": "1.3.6",
+    "chromedriver": "112.0.0",
     "cross-env": "7.0.3",
     "dot": "1.1.3",
     "jstat": "1.9.6",
-    "lighthouse": "10.0.2",
+    "lighthouse": "10.1.1",
     "lodash": "4.17.21",
-    "npm-check-updates": "16.7.10",
-    "playwright": "1.31.2",
-    "playwright-firefox": "1.31.2",
-    "playwright-webkit": "1.31.2",
-    "puppeteer-core": "19.7.2",
-    "ramda": "0.28.0",
-    "selenium-webdriver": "4.8.1",
-    "semver": "7.3.8",
+    "npm-check-updates": "16.10.9",
+    "playwright": "1.33.0",
+    "playwright-firefox": "1.33.0",
+    "playwright-webkit": "1.33.0",
+    "puppeteer-core": "19.11.1",
+    "ramda": "0.29.0",
+    "selenium-webdriver": "4.9.0",
+    "semver": "7.5.0",
     "underscore": "1.13.6",
     "yargs": "17.7.1"
   }

webdriver-ts/package.json

@@ -0,0 +1,21 @@
+import yargs from 'yargs';
+
+// FIXME: Clean up args.
+// What works: npm run bench keyed/react, npm run bench -- keyed/react, npm run bench -- keyed/react --count 1 --benchmark 01_
+// What doesn't work (keyed/react becomes an element of argument benchmark): npm run bench -- --count 1 --benchmark 01_ keyed/react
+
+export let args: any = yargs(process.argv)
+  .usage(
+    "$0 [--framework Framework1 Framework2 ...] [--benchmark Benchmark1 Benchmark2 ...] [--chromeBinary path] \n or: $0 [directory1] [directory2] .. [directory3]"
+  )
+  .help("help")
+  .boolean("headless").default("headless", false)
+  .boolean("smoketest")
+  .boolean("nothrottling").default("nothrottling", false)
+  .string("runner").default("runner","puppeteer")
+  .string("browser").default("browser","chrome")
+  .array("framework")
+  .array("benchmark")
+  .string("chromeBinary").argv;
+
+console.log("args", args);

webdriver-ts/src/benchmarkArgs.ts

@@ -1,16 +1,16 @@
+import {args} from './benchmarkArgs.js';
+import { BenchmarkOptions, BENCHMARK_RUNNER, config, ErrorAndWarning, FrameworkData, initializeFrameworks } from "./common.js";
 import { fork } from "child_process";
 import * as fs from "fs";
-import yargs from 'yargs';
 import { BenchmarkInfo, benchmarkInfos, BenchmarkType, CPUBenchmarkInfo, MemBenchmarkInfo, StartupBenchmarkInfo } from "./benchmarksCommon.js";
 import { StartupBenchmarkResult } from "./benchmarksLighthouse.js";
-import { BenchmarkOptions, BENCHMARK_RUNNER, config, ErrorAndWarning, FrameworkData, initializeFrameworks } from "./common.js";
 import { writeResults } from "./writeResults.js";
 
 function forkAndCallBenchmark(
   framework: FrameworkData,
   benchmarkInfo: BenchmarkInfo,
   benchmarkOptions: BenchmarkOptions
-): Promise<ErrorAndWarning> {
+): Promise<ErrorAndWarning<number|StartupBenchmarkResult>> {
   return new Promise((resolve, reject) => {
     let forkedRunner = null;
     if (benchmarkInfo.type === BenchmarkType.STARTUP_MAIN) {
@@ -35,7 +35,7 @@ function forkAndCallBenchmark(
       benchmarkId: benchmarkInfo.id,
       benchmarkOptions,
     });
-    forked.on("message", (msg: ErrorAndWarning) => {
+    forked.on("message", (msg: ErrorAndWarning<number|StartupBenchmarkResult>) => {
       if (config.LOG_DETAILS) console.log("FORKING: main process got message from child", msg);
       resolve(msg);
     });
@@ -243,23 +243,6 @@ async function runBench(runFrameworks: FrameworkData[], benchmarkInfos: Benchmar
   }
 }
 
-// FIXME: Clean up args.
-// What works: npm run bench keyed/react, npm run bench -- keyed/react, npm run bench -- keyed/react --count 1 --benchmark 01_
-// What doesn't work (keyed/react becomes an element of argument benchmark): npm run bench -- --count 1 --benchmark 01_ keyed/react
-
-let args: any = yargs(process.argv)
-  .usage(
-    "$0 [--framework Framework1 Framework2 ...] [--benchmark Benchmark1 Benchmark2 ...] [--chromeBinary path] \n or: $0 [directory1] [directory2] .. [directory3]"
-  )
-  .help("help")
-  .boolean("headless").default("headless", false)
-  .boolean("smoketest")
-  .string("runner").default("runner",config.BENCHMARK_RUNNER)
-  .string("browser").default("browser",config.BROWSER)
-  .array("framework")
-  .array("benchmark")
-  .string("chromeBinary").argv;
-
 let runner = args.runner;
 if ([BENCHMARK_RUNNER.WEBDRIVER_CDP,BENCHMARK_RUNNER.WEBDRIVER,BENCHMARK_RUNNER.WEBDRIVER_AFTERFRAME,
   BENCHMARK_RUNNER.PLAYWRIGHT,
@@ -305,7 +288,9 @@ async function main() {
     config.EXIT_ON_ERROR = true;
     console.log('Using smoketest config ', JSON.stringify(config));
   }
-    
+  if (config.BENCHMARK_RUNNER == BENCHMARK_RUNNER.WEBDRIVER_AFTERFRAME) {
+    config.RESULTS_DIRECTORY = "results_client_"+config.BROWSER;
+  }    
   if (!fs.existsSync(config.RESULTS_DIRECTORY)) fs.mkdirSync(config.RESULTS_DIRECTORY);
   if (!fs.existsSync(config.TRACES_DIRECTORY)) fs.mkdirSync(config.TRACES_DIRECTORY);
 

webdriver-ts/src/benchmarkRunner.ts

@@ -1,4 +1,5 @@
 import { FrameworkData, config } from "./common.js";
+import { args } from "./benchmarkArgs.js";
 
 export enum BenchmarkType {
   CPU,
@@ -96,7 +97,8 @@ const slowDownsOSX: ISlowDowns = {
 
 const slowDownsLinux: ISlowDowns = slowDownsOSX;
 
-const slowDowns: ISlowDowns = process.platform == "darwin" ? slowDownsOSX : slowDownsLinux;
+
+const slowDowns: ISlowDowns = args.nothrottling ? {} : (process.platform == "darwin" ? slowDownsOSX : slowDownsLinux);
 
 export function slowDownNote(benchmark: TBenchmarkID): string {
   return slowDowns[benchmark] ? " " + slowDowns[benchmark] + "x CPU slowdown." : "";

webdriver-ts/src/benchmarksCommon.ts

@@ -1,5 +1,4 @@
 import axios from "axios";
-import { StartupBenchmarkResult } from "./benchmarksLighthouse.js";
 
 
 export interface JSONResult {
@@ -18,10 +17,10 @@ export interface JSONResult {
 
 export type TBenchmarkStatus = "OK" | "TEST_FAILED" | "TECHNICAL_ERROR";
 
-export interface ErrorAndWarning {
+export interface ErrorAndWarning<T> {
   error: string;
   warnings: string[];
-  result?: number[] | StartupBenchmarkResult[];
+  result?: T[];
 }
 
 export interface BenchmarkDriverOptions {

webdriver-ts/src/common.ts

@@ -89,7 +89,7 @@ async function runStartupBenchmark(
   framework: FrameworkData,
   benchmark: BenchmarkLighthouse,
   benchmarkOptions: BenchmarkOptions
-): Promise<ErrorAndWarning> {
+): Promise<ErrorAndWarning<StartupBenchmarkResult>> {
   console.log("benchmarking startup", framework, benchmark.benchmarkInfo.id);
 
   let error: string = undefined;
@@ -106,13 +106,13 @@ export async function executeBenchmark(
   framework: FrameworkData,
   benchmarkId: string,
   benchmarkOptions: BenchmarkOptions
-): Promise<ErrorAndWarning> {
+): Promise<ErrorAndWarning<StartupBenchmarkResult>> {
   let runBenchmarks: Array<BenchmarkLighthouse> = benchmarks.filter(b => benchmarkId === b.benchmarkInfo.id && b instanceof BenchmarkLighthouse) as Array<BenchmarkLighthouse>;
   if (runBenchmarks.length != 1) throw `Benchmark name ${benchmarkId} is not unique (lighthouse)`;
 
   let benchmark = runBenchmarks[0];
 
-  let errorAndWarnings: ErrorAndWarning;
+  let errorAndWarnings: ErrorAndWarning<StartupBenchmarkResult>;
     errorAndWarnings = await runStartupBenchmark(framework, benchmark, benchmarkOptions);
   if (config.LOG_DEBUG) console.log("benchmark finished - got errors promise", errorAndWarnings);
   return errorAndWarnings;

webdriver-ts/src/forkedBenchmarkRunnerLighthouse.ts

@@ -53,7 +53,7 @@ async function forceGC(page: Page, client: CDPSession) {
   }
 }
 
-async function runCPUBenchmark(framework: FrameworkData, benchmark: CPUBenchmarkPlaywright, benchmarkOptions: BenchmarkOptions): Promise<ErrorAndWarning>
+async function runCPUBenchmark(framework: FrameworkData, benchmark: CPUBenchmarkPlaywright, benchmarkOptions: BenchmarkOptions): Promise<ErrorAndWarning<number>>
 {
     let error: string = undefined;
     let warnings: string[] = [];
@@ -153,7 +153,7 @@ async function runMemBenchmark(
   framework: FrameworkData,
   benchmark: MemBenchmarkPlaywright,
   benchmarkOptions: BenchmarkOptions
-): Promise<ErrorAndWarning> {
+): Promise<ErrorAndWarning<number>> {
   let error: string = undefined;
   let warnings: string[] = [];
   let results: number[] = [];
@@ -217,12 +217,12 @@ export async function executeBenchmark(
   framework: FrameworkData,
   benchmarkId: string,
   benchmarkOptions: BenchmarkOptions
-): Promise<ErrorAndWarning> {
+): Promise<ErrorAndWarning<number>> {
   let runBenchmarks: Array<TBenchmarkPlaywright> = benchmarks.filter(b => benchmarkId === b.benchmarkInfo.id && (b instanceof CPUBenchmarkPlaywright || b instanceof MemBenchmarkPlaywright) ) as Array<TBenchmarkPlaywright>;
 
   let benchmark = runBenchmarks[0];
 
-  let errorAndWarnings: ErrorAndWarning;
+  let errorAndWarnings: ErrorAndWarning<number>;
   if (benchmark.type == BenchmarkType.CPU) {
     errorAndWarnings = await runCPUBenchmark(framework, benchmark as CPUBenchmarkPlaywright, benchmarkOptions);
   } else {

webdriver-ts/src/forkedBenchmarkRunnerPlaywright.ts

@@ -50,7 +50,7 @@ async function forceGC(page: Page, client: CDPSession) {
   }
 }
 
-async function runCPUBenchmark(framework: FrameworkData, benchmark: CPUBenchmarkPuppeteer, benchmarkOptions: BenchmarkOptions): Promise<ErrorAndWarning>
+async function runCPUBenchmark(framework: FrameworkData, benchmark: CPUBenchmarkPuppeteer, benchmarkOptions: BenchmarkOptions): Promise<ErrorAndWarning<number>>
 {
     let error: string = undefined;
     let warnings: string[] = [];
@@ -165,7 +165,7 @@ async function runMemBenchmark(
   framework: FrameworkData,
   benchmark: MemBenchmarkPuppeteer,
   benchmarkOptions: BenchmarkOptions
-): Promise<ErrorAndWarning> {
+): Promise<ErrorAndWarning<number>> {
   let error: string = undefined;
   let warnings: string[] = [];
   let results: number[] = [];
@@ -235,13 +235,13 @@ export async function executeBenchmark(
   framework: FrameworkData,
   benchmarkId: string,
   benchmarkOptions: BenchmarkOptions
-): Promise<ErrorAndWarning> {
+): Promise<ErrorAndWarning<number>> {
   let runBenchmarks: Array<TBenchmarkPuppeteer> = benchmarks.filter(b => benchmarkId === b.benchmarkInfo.id && (b instanceof CPUBenchmarkPuppeteer || b instanceof MemBenchmarkPuppeteer) ) as Array<TBenchmarkPuppeteer>;
   if (runBenchmarks.length != 1) throw `Benchmark name ${benchmarkId} is not unique (puppeteer)`;
 
   let benchmark = runBenchmarks[0];
 
-  let errorAndWarnings: ErrorAndWarning;
+  let errorAndWarnings: ErrorAndWarning<number>;
   if (benchmark.type == BenchmarkType.CPU) {
     errorAndWarnings = await runCPUBenchmark(framework, benchmark as CPUBenchmarkPuppeteer, benchmarkOptions);
   } else {

webdriver-ts/src/forkedBenchmarkRunnerPuppeteer.ts

@@ -253,7 +253,7 @@ async function runCPUBenchmark(
   framework: FrameworkData,
   benchmark: CPUBenchmarkWebdriver,
   benchmarkOptions: BenchmarkOptions
-): Promise<ErrorAndWarning> {
+): Promise<ErrorAndWarning<number>> {
   let error: string = undefined;
   let warnings: string[] = [];
 
@@ -286,11 +286,11 @@ async function runCPUBenchmark(
       await driver.executeScript("console.timeStamp('runBenchmark')");
       await runBenchmark(driver, benchmark, framework);
       if (benchmark.benchmarkInfo.throttleCPU) {
-        console.log("resetting CPU slowdown");
-        await (driver as any).sendDevToolsCommand("Emulation.setCPUThrottlingRate", { rate: 1 });
+          console.log("resetting CPU slowdown");
+          await (driver as any).sendDevToolsCommand("Emulation.setCPUThrottlingRate", { rate: 1 });
+        }
+        await driver.executeScript("console.timeStamp('finishedBenchmark')");
       }
-      await driver.executeScript("console.timeStamp('finishedBenchmark')");
-    }
     let result = await computeResultsCPU(driver, benchmarkOptions, framework, benchmark, warnings, benchmarkOptions.batchSize);
     await driver.close();
     await driver.quit();
@@ -314,13 +314,13 @@ export async function executeBenchmark(
   framework: FrameworkData,
   benchmarkId: string,
   benchmarkOptions: BenchmarkOptions
-): Promise<ErrorAndWarning> {
+): Promise<ErrorAndWarning<number>> {
   let runBenchmarks: Array<CPUBenchmarkWebdriver> = benchmarks.filter(b => benchmarkId === b.benchmarkInfo.id && b instanceof CPUBenchmarkWebdriver) as Array<CPUBenchmarkWebdriver>;
   if (runBenchmarks.length != 1) throw `Benchmark name ${benchmarkId} is not unique (webdriver)`;
 
   let benchmark = runBenchmarks[0];
 
-  let errorAndWarnings: ErrorAndWarning;
+  let errorAndWarnings: ErrorAndWarning<number>;
   if (benchmark.benchmarkInfo.type == BenchmarkType.CPU) {
     errorAndWarnings = await runCPUBenchmark(framework, benchmark, benchmarkOptions);
   }