add JS only duration for Playwright, add dummies for the others

krausest · krausest · commit c84a2c84705d · 2023-05-04T18:34:44.000+02:00
diff --git a/webdriver-ts/src/benchmarksWebdriverAfterframe.ts b/webdriver-ts/src/benchmarksWebdriverAfterframe.ts
@@ -18,7 +18,7 @@ const SHORT_TIMEOUT = 20 * 1000;
 let durations: Array<number> = [];
 
 export function getAfterframeDurations() {
-  return durations;
+  return durations.map(d => ({total:d, script:0}));
 }
 
 export async  function initMeasurement(driver: WebDriver) {
diff --git a/webdriver-ts/src/forkedBenchmarkRunnerPlaywright.ts b/webdriver-ts/src/forkedBenchmarkRunnerPlaywright.ts
@@ -1,5 +1,5 @@
 import { Browser, Page, CDPSession } from "playwright-core";
-import { BenchmarkType, slowDownFactor } from "./benchmarksCommon.js";
+import { BenchmarkType, CPUBenchmarkResult, slowDownFactor } from "./benchmarksCommon.js";
 import { benchmarks, CPUBenchmarkPlaywright, fileNameTrace, MemBenchmarkPlaywright, TBenchmarkPlaywright } from "./benchmarksPlaywright.js";
 import { BenchmarkOptions, config as defaultConfig, ErrorAndWarning, FrameworkData, TConfig } from "./common.js";
 import { startBrowser } from "./playwrightAccess.js";
@@ -53,11 +53,11 @@ async function forceGC(page: Page, client: CDPSession) {
   }
 }
 
-async function runCPUBenchmark(framework: FrameworkData, benchmark: CPUBenchmarkPlaywright, benchmarkOptions: BenchmarkOptions): Promise<ErrorAndWarning<number>>
+async function runCPUBenchmark(framework: FrameworkData, benchmark: CPUBenchmarkPlaywright, benchmarkOptions: BenchmarkOptions): Promise<ErrorAndWarning<CPUBenchmarkResult>>
 {
     let error: string = undefined;
     let warnings: string[] = [];
-    let results: number[] = [];
+    let results: CPUBenchmarkResult[] = [];
 
     console.log("benchmarking ", framework, benchmark.benchmarkInfo.id);
     let browser : Browser = null;
@@ -72,6 +72,7 @@ async function runCPUBenchmark(framework: FrameworkData, benchmark: CPUBenchmark
             });
         // }
         let client = await page.context().newCDPSession(page);
+        await client.send('Performance.enable');
         for (let i = 0; i <benchmarkOptions.batchSize; i++) {
             await page.goto(`http://${benchmarkOptions.host}:${benchmarkOptions.port}/${framework.uri}/index.html`, {waitUntil: "networkidle"});
 
@@ -112,16 +113,28 @@ async function runCPUBenchmark(framework: FrameworkData, benchmark: CPUBenchmark
                 screenshots: false,
                 categories:categories
             });
+            let m1 = (await client.send('Performance.getMetrics')).metrics;
+            let m1_val = m1.find(m => m.name === 'ScriptDuration').value;
+            let m1_Timestamp = m1.find(m => m.name === 'Timestamp').value;
+            console.log("m1", m1, m1_val);
             console.log("runBenchmark Playwright");
             await runBenchmark(browser, page, benchmark, framework);
 
             await wait(40);
+            let m2 = (await client.send('Performance.getMetrics')).metrics;
+            let m2_val = m2.find(m => m.name === 'ScriptDuration').value;
+            let m2_Timestamp = m2.find(m => m.name === 'Timestamp').value;
+            console.log("m2", m2, m2_val);
             await browser.stopTracing();
             if (throttleCPU) {
               await client.send('Emulation.setCPUThrottlingRate', { rate: 1 });            
           }  
             let result = await computeResultsCPU(config, fileNameTrace(framework, benchmark.benchmarkInfo, i, benchmarkOptions), benchmark.benchmarkInfo.durationMeasurementMode);
-            results.push(result);
+            let resultScript = (m2_val - m1_val)*1000.0;
+            console.log("**** resultScript = ", resultScript);
+            if (m2_Timestamp == m1_Timestamp) throw new Error("Page metrics timestamp didn't change");
+
+            results.push({total: result, script: resultScript});
             console.log(`duration for ${framework.name} and ${benchmark.benchmarkInfo.id}: ${result}`);
             if (result < 0)
                 throw new Error(`duration ${result} < 0`);                
@@ -218,12 +231,12 @@ export async function executeBenchmark(
   framework: FrameworkData,
   benchmarkId: string,
   benchmarkOptions: BenchmarkOptions
-): Promise<ErrorAndWarning<number>> {
+): Promise<ErrorAndWarning<number|CPUBenchmarkResult>> {
   let runBenchmarks: Array<TBenchmarkPlaywright> = benchmarks.filter(b => benchmarkId === b.benchmarkInfo.id && (b instanceof CPUBenchmarkPlaywright || b instanceof MemBenchmarkPlaywright) ) as Array<TBenchmarkPlaywright>;
 
   let benchmark = runBenchmarks[0];
 
-  let errorAndWarnings: ErrorAndWarning<number>;
+  let errorAndWarnings: ErrorAndWarning<number|CPUBenchmarkResult>;
   if (benchmark.type == BenchmarkType.CPU) {
     errorAndWarnings = await runCPUBenchmark(framework, benchmark as CPUBenchmarkPlaywright, benchmarkOptions);
   } else {
diff --git a/webdriver-ts/src/forkedBenchmarkRunnerPuppeteer.ts b/webdriver-ts/src/forkedBenchmarkRunnerPuppeteer.ts
@@ -139,7 +139,8 @@ async function runCPUBenchmark(framework: FrameworkData, benchmark: CPUBenchmark
             // let result = (m2.TaskDuration - m1.TaskDuration)*1000.0; //await computeResultsCPU(fileNameTrace(framework, benchmark, i), benchmarkOptions, framework, benchmark, warnings, benchmarkOptions.batchSize);
             let result = await computeResultsCPU(config, fileNameTrace(framework, benchmark.benchmarkInfo, i, benchmarkOptions), benchmark.benchmarkInfo.durationMeasurementMode);
             let resultScript = (m2.ScriptDuration - m1.ScriptDuration)*1000.0;
-            console.log("**** resultJS = ", resultScript);
+            console.log("**** resultScript = ", resultScript);
+            if (m2.Timestamp == m1.Timestamp) throw new Error("Page metrics timestamp didn't change");
             results.push({total:result, script: resultScript});
             console.log(`duration for ${framework.name} and ${benchmark.benchmarkInfo.id}: ${result}`);
             if (result < 0)
diff --git a/webdriver-ts/src/forkedBenchmarkRunnerWebdriver.ts b/webdriver-ts/src/forkedBenchmarkRunnerWebdriver.ts
@@ -4,7 +4,7 @@ import { setUseShadowRoot, buildDriver, setUseRowShadowRoot, setShadowRootName,
 
 import { TConfig, config as defaultConfig, FrameworkData, ErrorAndWarning, BenchmarkOptions } from "./common.js";
 import * as R from "ramda";
-import { BenchmarkType, DurationMeasurementMode, slowDownFactor } from "./benchmarksCommon.js";
+import { BenchmarkType, CPUBenchmarkResult, DurationMeasurementMode, slowDownFactor } from "./benchmarksCommon.js";
 
 let config: TConfig = defaultConfig;
 
@@ -126,7 +126,7 @@ async function computeResultsCPU(
   benchmark: CPUBenchmarkWebdriver,
   warnings: string[],
   expcectedResultCount: number
-): Promise<number[]> {
+): Promise<CPUBenchmarkResult[]> {
   let entriesBrowser = await driver.manage().logs().get(logging.Type.BROWSER);
   if (config.LOG_DEBUG) console.log("browser entries", entriesBrowser);
   const perfLogEvents = await fetchEventsFromPerformanceLog(driver);
@@ -135,7 +135,7 @@ async function computeResultsCPU(
   // if (config.LOG_DEBUG) console.log("filteredEvents ", asString(filteredEvents));
 
   let remaining = R.dropWhile(type_eq("initBenchmark"))(filteredEvents);
-  let results: number[] = [];
+  let results: CPUBenchmarkResult[] = [];
 
   while (remaining.length > 0) {
     let evts = R.splitWhen(type_eq("finishedBenchmark"))(remaining);
@@ -196,7 +196,8 @@ async function computeResultsCPU(
         console.log("soundness check failed. reported duration is bigger than whole benchmark duration", asString(eventsDuringBenchmark));
         throw "soundness check failed. reported duration is bigger than whole benchmark duration";
       }
-      results.push(duration);
+      // script is currently not implemented
+      results.push({total:duration, script:0});
     }
     remaining = R.drop(1, evts[1]);
   }
@@ -253,7 +254,7 @@ async function runCPUBenchmark(
   framework: FrameworkData,
   benchmark: CPUBenchmarkWebdriver,
   benchmarkOptions: BenchmarkOptions
-): Promise<ErrorAndWarning<number>> {
+): Promise<ErrorAndWarning<number|CPUBenchmarkResult>> {
   let error: string = undefined;
   let warnings: string[] = [];
 
@@ -315,13 +316,13 @@ export async function executeBenchmark(
   framework: FrameworkData,
   benchmarkId: string,
   benchmarkOptions: BenchmarkOptions
-): Promise<ErrorAndWarning<number>> {
+): Promise<ErrorAndWarning<number|CPUBenchmarkResult>> {
   let runBenchmarks: Array<CPUBenchmarkWebdriver> = benchmarks.filter(b => benchmarkId === b.benchmarkInfo.id && b instanceof CPUBenchmarkWebdriver) as Array<CPUBenchmarkWebdriver>;
   if (runBenchmarks.length != 1) throw `Benchmark name ${benchmarkId} is not unique (webdriver)`;
 
   let benchmark = runBenchmarks[0];
 
-  let errorAndWarnings: ErrorAndWarning<number>;
+  let errorAndWarnings: ErrorAndWarning<number|CPUBenchmarkResult>;
   if (benchmark.benchmarkInfo.type == BenchmarkType.CPU) {
     errorAndWarnings = await runCPUBenchmark(framework, benchmark, benchmarkOptions);
   }
diff --git a/webdriver-ts/src/forkedBenchmarkRunnerWebdriverAfterframe.ts b/webdriver-ts/src/forkedBenchmarkRunnerWebdriverAfterframe.ts
@@ -3,7 +3,7 @@ import { CPUBenchmarkWebdriver, benchmarks } from "./benchmarksWebdriverAfterfra
 import { setUseShadowRoot, buildDriver, setUseRowShadowRoot, setShadowRootName, setButtonsInShadowRoot } from "./webdriverAccess.js";
 
 import { TConfig, config as defaultConfig, FrameworkData, ErrorAndWarning, BenchmarkOptions } from "./common.js";
-import { BenchmarkType } from "./benchmarksCommon.js";
+import { BenchmarkType, CPUBenchmarkResult } from "./benchmarksCommon.js";
 import { getAfterframeDurations, initMeasurement } from "./benchmarksWebdriverAfterframe.js";
 
 let config: TConfig = defaultConfig;
@@ -46,10 +46,10 @@ async function runCPUBenchmark(
   framework: FrameworkData,
   benchmark: CPUBenchmarkWebdriver,
   benchmarkOptions: BenchmarkOptions
-): Promise<ErrorAndWarning<number>> {
+): Promise<ErrorAndWarning<CPUBenchmarkResult>> {
   let error: string = undefined;
   let warnings: string[] = [];
-  let results: number[] = [];
+  let results: CPUBenchmarkResult[] = [];
 
   console.log("benchmarking ", framework, benchmark.benchmarkInfo.id);
   let driver: WebDriver = null;
@@ -102,13 +102,13 @@ export async function executeBenchmark(
   framework: FrameworkData,
   benchmarkId: string,
   benchmarkOptions: BenchmarkOptions
-): Promise<ErrorAndWarning<number>> {
+): Promise<ErrorAndWarning<number|CPUBenchmarkResult>> {
   let runBenchmarks: Array<CPUBenchmarkWebdriver> = benchmarks.filter(b => benchmarkId === b.benchmarkInfo.id && b instanceof CPUBenchmarkWebdriver) as Array<CPUBenchmarkWebdriver>;
   if (runBenchmarks.length != 1) throw `Benchmark name ${benchmarkId} is not unique (webdriver)`;
 
   let benchmark = runBenchmarks[0];
 
-  let errorAndWarnings: ErrorAndWarning<number>;
+  let errorAndWarnings: ErrorAndWarning<number|CPUBenchmarkResult>;
   if (benchmark.benchmarkInfo.type == BenchmarkType.CPU) {
     errorAndWarnings = await runCPUBenchmark(framework, benchmark, benchmarkOptions);
   }
diff --git a/webdriver-ts/src/forkedBenchmarkRunnerWebdriverCDP.ts b/webdriver-ts/src/forkedBenchmarkRunnerWebdriverCDP.ts
@@ -1,6 +1,6 @@
 import * as fs from "fs/promises";
 import { WebDriver } from "selenium-webdriver";
-import { BenchmarkType, slowDownFactor } from "./benchmarksCommon.js";
+import { BenchmarkType, CPUBenchmarkResult, slowDownFactor } from "./benchmarksCommon.js";
 import { benchmarks, CPUBenchmarkWebdriverCDP, fileNameTrace } from "./benchmarksWebdriverCDP.js";
 import { BenchmarkOptions, config as defaultConfig, ErrorAndWarning, FrameworkData, TConfig } from "./common.js";
 import { computeResultsCPU } from "./timeline.js";
@@ -58,10 +58,10 @@ async function runCPUBenchmark(
   framework: FrameworkData,
   benchmark: CPUBenchmarkWebdriverCDP,
   benchmarkOptions: BenchmarkOptions
-): Promise<ErrorAndWarning<number>> {
+): Promise<ErrorAndWarning<CPUBenchmarkResult>> {
   let error: string = undefined;
   let warnings: string[] = [];
-  let results: number[] = [];
+  let results: CPUBenchmarkResult[] = [];
 
   console.log("benchmarking ", framework, benchmark.benchmarkInfo.id, "with webdriver (tracing via CDP Connection)");
   let driver: WebDriver = null;
@@ -134,7 +134,7 @@ async function runCPUBenchmark(
       await p;
 
       let result = await computeResultsCPU(config, fileNameTrace(framework, benchmark.benchmarkInfo, i, benchmarkOptions), benchmark.benchmarkInfo.durationMeasurementMode);
-      results.push(result);
+      results.push({total:result, script: 0});
       console.log(`duration for ${framework.name} and ${benchmark.benchmarkInfo.id}: ${result}`);
       if (result < 0)
           throw new Error(`duration ${result} < 0`);                
@@ -162,13 +162,13 @@ export async function executeBenchmark(
   framework: FrameworkData,
   benchmarkId: string,
   benchmarkOptions: BenchmarkOptions
-): Promise<ErrorAndWarning<number>> {
+): Promise<ErrorAndWarning<number|CPUBenchmarkResult>> {
   let runBenchmarks: Array<CPUBenchmarkWebdriverCDP> = benchmarks.filter(b => benchmarkId === b.benchmarkInfo.id && b instanceof CPUBenchmarkWebdriverCDP) as Array<CPUBenchmarkWebdriverCDP>;
   if (runBenchmarks.length != 1) throw `Benchmark name ${benchmarkId} is not unique (webdriver)`;
 
   let benchmark = runBenchmarks[0];
 
-  let errorAndWarnings: ErrorAndWarning<number>;
+  let errorAndWarnings: ErrorAndWarning<number|CPUBenchmarkResult>;
   if (benchmark.benchmarkInfo.type == BenchmarkType.CPU) {
     errorAndWarnings = await runCPUBenchmark(framework, benchmark, benchmarkOptions);
   }
diff --git a/webdriver-ts/src/writeResults.ts b/webdriver-ts/src/writeResults.ts
@@ -41,7 +41,6 @@ export function writeResults(resultDir: string, res: ResultLightHouse|ResultCPU|
     }
       break;
     case BenchmarkType.CPU:
-      console.log("writeResults", res);
       createResultFile(resultDir, ({total: res.results.map(r=>r.total), script: res.results.map(r=>r.script)}), res.framework, res.benchmark);
       break;
     case BenchmarkType.MEM:
@@ -51,7 +50,6 @@ export function writeResults(resultDir: string, res: ResultLightHouse|ResultCPU|
 }
 
 function createResultFile(resultDir: string, data: number[]|{[key:string]: number[]}, framework: FrameworkData, benchmark: BenchmarkInfo) {
-  console.log("createResultFile", data);
   let type = "";
   switch (benchmark.type) {
     case BenchmarkType.CPU:
@@ -91,7 +89,6 @@ function createResultFile(resultDir: string, data: number[]|{[key:string]: numbe
   } else {
     let values: {[k: string]: JSONResultData} = {};
     for (let key of Object.keys(data)) {
-      console.log("converting", key, data[key], data);
       values[key] = convertResult(key, data[key]);
     }
     let result: JSONResult = {

Original file line number	Diff line number	Diff line change
`@@ -18,7 +18,7 @@ const SHORT_TIMEOUT = 20 * 1000;`
`18`	`18`	`let durations: Array<number> = [];`
`19`	`19`
`20`	`20`	`export function getAfterframeDurations() {`
`21`		`- return durations;`
	`21`	`+ return durations.map(d => ({total:d, script:0}));`
`22`	`22`	`}`
`23`	`23`
`24`	`24`	`export async function initMeasurement(driver: WebDriver) {`