diff --git a/.gitignore b/.gitignore index 0f481fd..15a686e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ node_modules/ local.log errorShots -.DS_Store \ No newline at end of file +.DS_Store +*.log +performance-report*.json +package-lock.json \ No newline at end of file diff --git a/android/examples/run-parallel-test/moduleA/specs/test.js b/android/examples/run-parallel-test/moduleA/specs/test.js new file mode 100644 index 0000000..fd0be5a --- /dev/null +++ b/android/examples/run-parallel-test/moduleA/specs/test.js @@ -0,0 +1,81 @@ +const assert = require('assert'); + +describe("BStackDemo Tests Module A", () => { + before(async () => { + var skipButton = await $('android=new UiSelector().resourceId("org.wikipedia.alpha:id/fragment_onboarding_skip_button")'); + await skipButton.waitForDisplayed({ timeout: 30000 }); + await skipButton.click(); + + var searchSelector = await $(`~Search Wikipedia`); + await searchSelector.waitForDisplayed({ timeout: 30000 }); + await searchSelector.click(); + }); + + it("flaky test - random product selection", async () => { + // this.tags = ['regression', 'p1']; + const selector = Math.random() > 0.5 ? 'android=new UiSelector().resourceId("org.wikipedia.alpha:id/search_src_text")' : 'falseSelector'; + const insertTextSelector = await $(selector); + await insertTextSelector.waitForDisplayed({ timeout: 30000 }); + + // Randomly choose a term that will always have results or never have results + const randomTerm = Math.random() > 0.5 ? "BrowserStack" : "akjsdhfakjsdhf"; // nonsense string + await insertTextSelector.addValue(randomTerm); + await browser.pause(5000); + + const allResults = await $$(`android.widget.TextView`); + // This assertion will sometimes fail if randomTerm is the nonsense string + assert(allResults.length > 0); + }); + + it("always failing test - missing element 1", async () => { + // Try to click a non-existent element, which should fail + const nonExistent = await $(`~non-existent-1`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example C", async () => { + assert.equal(true, true); + }); + + it("always failing test - same stacktrace 1", async () => { + // Try to click a non-existent element, which should fail (same selector as below) + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always failing test - same stacktrace 2", async () => { + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example D", async () => { + assert.equal(true, true); + }); + + it("always passing test - example A", async () => { + assert.strictEqual(1 + 1, 2, 'This test should always pass'); + }); + + it("Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("Another Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("always passing test - example B", async () => { + assert.strictEqual("Browser" + "Stack", "BrowserStack", 'This test should always pass'); + }); +}); \ No newline at end of file diff --git a/android/examples/run-parallel-test/moduleB/specs/test.js b/android/examples/run-parallel-test/moduleB/specs/test.js new file mode 100644 index 0000000..9edd5fd --- /dev/null +++ b/android/examples/run-parallel-test/moduleB/specs/test.js @@ -0,0 +1,89 @@ +const assert = require('assert'); + +describe("BStackDemo Tests Module B", () => { + before(async () => { + var skipButton = await $('android=new UiSelector().resourceId("org.wikipedia.alpha:id/fragment_onboarding_skip_button")'); + await skipButton.waitForDisplayed({ timeout: 30000 }); + await skipButton.click(); + + var searchSelector = await $(`~Search Wikipedia`); + await searchSelector.waitForDisplayed({ timeout: 30000 }); + await searchSelector.click(); + + // var insertTextSelector = await $('android=new UiSelector().resourceId("org.wikipedia.alpha:id/search_src_text")'); + // await insertTextSelector.waitForDisplayed({ timeout: 30000 }); + + // await insertTextSelector.addValue("BrowserStack"); + // await browser.pause(5000); + + // var allProductsName = await $$(`android.widget.TextView`); + // assert(allProductsName.length > 0, 'Search results should be present'); + }); + + it("flaky test - random product selection", async () => { + const selector = Math.random() > 0.5 ? 'android=new UiSelector().resourceId("org.wikipedia.alpha:id/search_src_text")' : 'falseSelector'; + const insertTextSelector = await $(selector); + await insertTextSelector.waitForDisplayed({ timeout: 30000 }); + + // Randomly choose a term that will always have results or never have results + const randomTerm = Math.random() > 0.5 ? "BrowserStack" : "akjsdhfakjsdhf"; // nonsense string + await insertTextSelector.addValue(randomTerm); + await browser.pause(5000); + + const allResults = await $$(`android.widget.TextView`); + // This assertion will sometimes fail if randomTerm is the nonsense string + assert(allResults.length > 0); + }); + + it("always failing test - missing element 1", async () => { + // Try to click a non-existent element, which should fail + const nonExistent = await $(`~non-existent-1`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example C", async () => { + assert.equal(true, true); + }); + + it("always failing test - same stacktrace 1", async () => { + // Try to click a non-existent element, which should fail (same selector as below) + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always failing test - same stacktrace 2", async () => { + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example D", async () => { + assert.equal(true, true); + }); + + it("always passing test - example A", async () => { + assert.strictEqual(1 + 1, 2, 'This test should always pass'); + }); + + it("Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("Another Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("always passing test - example B", async () => { + assert.strictEqual("Browser" + "Stack", "BrowserStack", 'This test should always pass'); + }); +}); \ No newline at end of file diff --git a/android/examples/run-parallel-test/moduleC/specs/test.js b/android/examples/run-parallel-test/moduleC/specs/test.js new file mode 100644 index 0000000..6fbfb4d --- /dev/null +++ b/android/examples/run-parallel-test/moduleC/specs/test.js @@ -0,0 +1,89 @@ +const assert = require('assert'); + +describe("BStackDemo Tests Module C", () => { + before(async () => { + var skipButton = await $('android=new UiSelector().resourceId("org.wikipedia.alpha:id/fragment_onboarding_skip_button")'); + await skipButton.waitForDisplayed({ timeout: 30000 }); + await skipButton.click(); + + var searchSelector = await $(`~Search Wikipedia`); + await searchSelector.waitForDisplayed({ timeout: 30000 }); + await searchSelector.click(); + + // var insertTextSelector = await $('android=new UiSelector().resourceId("org.wikipedia.alpha:id/search_src_text")'); + // await insertTextSelector.waitForDisplayed({ timeout: 30000 }); + + // await insertTextSelector.addValue("BrowserStack"); + // await browser.pause(5000); + + // var allProductsName = await $$(`android.widget.TextView`); + // assert(allProductsName.length > 0, 'Search results should be present'); + }); + + it("flaky test - random product selection", async () => { + const selector = Math.random() > 0.5 ? 'android=new UiSelector().resourceId("org.wikipedia.alpha:id/search_src_text")' : 'falseSelector'; + const insertTextSelector = await $(selector); + await insertTextSelector.waitForDisplayed({ timeout: 30000 }); + + // Randomly choose a term that will always have results or never have results + const randomTerm = Math.random() > 0.5 ? "BrowserStack" : "akjsdhfakjsdhf"; // nonsense string + await insertTextSelector.addValue(randomTerm); + await browser.pause(5000); + + const allResults = await $$(`android.widget.TextView`); + // This assertion will sometimes fail if randomTerm is the nonsense string + assert(allResults.length > 0); + }); + + it("always failing test - missing element 1", async () => { + // Try to click a non-existent element, which should fail + const nonExistent = await $(`~non-existent-1`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example C", async () => { + assert.equal(true, true); + }); + + it("always passing test - example D", async () => { + assert.equal(true, true); + }); + + it("always failing test - same stacktrace 1", async () => { + // Try to click a non-existent element, which should fail (same selector as below) + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always failing test - same stacktrace 2", async () => { + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example A", async () => { + assert.strictEqual(1 + 1, 2, 'This test should always pass'); + }); + + it("Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("Another Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("always passing test - example B", async () => { + assert.strictEqual("Browser" + "Stack", "BrowserStack", 'This test should always pass'); + }); +}); \ No newline at end of file diff --git a/android/examples/run-parallel-test/parallel.conf.js b/android/examples/run-parallel-test/parallel.conf.js index 345e76b..7941dad 100644 --- a/android/examples/run-parallel-test/parallel.conf.js +++ b/android/examples/run-parallel-test/parallel.conf.js @@ -13,6 +13,10 @@ exports.config = { browserstackLocal: true, opts: { forcelocal: false, localIdentifier: "webdriverio-appium-app-browserstack-repo" }, app: process.env.BROWSERSTACK_APP_PATH || './examples/WikipediaSample.apk', + testObservability: true, + testObservabilityOptions: { + buildTag: ['bstack_sample'], + } } ] ], @@ -44,7 +48,7 @@ exports.config = { updateJob: false, specs: [ - './specs/single_test.js' + './m*/**' ], exclude: [], diff --git a/ios/examples/run-parallel-test/moduleA/specs/test.js b/ios/examples/run-parallel-test/moduleA/specs/test.js new file mode 100644 index 0000000..ae1cae1 --- /dev/null +++ b/ios/examples/run-parallel-test/moduleA/specs/test.js @@ -0,0 +1,74 @@ +const assert = require('assert'); + +describe("BStackDemo Tests Module A", () => { + before(async () => { + var textButton = await $(`~Text Button`); + await textButton.waitForDisplayed({ timeout: 30000 }); + await textButton.click(); + + var textInput = await $(`~Text Input`); + await textInput.waitForDisplayed({ timeout: 30000 }); + await textInput.click() + await textInput.addValue("hello@browserstack.com"+"\n"); + + var textOutput = await $(`~Text Output`); + await textOutput.waitForDisplayed({ timeout: 30000 }); + var value = await textOutput.getText(); + }); + + it("flaky test - passes and fails intermittently", async () => { + assert(Math.random() > 0.3, 'Flaky test failed'); + }); + + it("always failing test - missing element 1", async () => { + // Try to click a non-existent element, which should fail + const nonExistent = await $(`~non-existent-1`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example C", async () => { + assert.equal(true, true); + }); + + it("always failing test - same stacktrace 1", async () => { + // Try to click a non-existent element, which should fail (same selector as below) + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always failing test - same stacktrace 2", async () => { + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example D", async () => { + assert.equal(true, true); + }); + + it("always passing test - example A", async () => { + assert.strictEqual(1 + 1, 2, 'This test should always pass'); + }); + + it("Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("Another Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("always passing test - example B", async () => { + assert.strictEqual("Browser" + "Stack", "BrowserStack", 'This test should always pass'); + }); +}); diff --git a/ios/examples/run-parallel-test/moduleB/specs/test.js b/ios/examples/run-parallel-test/moduleB/specs/test.js new file mode 100644 index 0000000..2c7cf0c --- /dev/null +++ b/ios/examples/run-parallel-test/moduleB/specs/test.js @@ -0,0 +1,74 @@ +const assert = require('assert'); + +describe("BStackDemo Tests Module B", () => { + before(async () => { + var textButton = await $(`~Text Button`); + await textButton.waitForDisplayed({ timeout: 30000 }); + await textButton.click(); + + var textInput = await $(`~Text Input`); + await textInput.waitForDisplayed({ timeout: 30000 }); + await textInput.click() + await textInput.addValue("hello@browserstack.com"+"\n"); + + var textOutput = await $(`~Text Output`); + await textOutput.waitForDisplayed({ timeout: 30000 }); + var value = await textOutput.getText(); + }); + + it("flaky test - passes and fails intermittently", async () => { + assert(Math.random() > 0.3, 'Flaky test failed'); + }); + + it("always failing test - missing element 1", async () => { + // Try to click a non-existent element, which should fail + const nonExistent = await $(`~non-existent-1`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example C", async () => { + assert.equal(true, true); + }); + + it("always failing test - same stacktrace 1", async () => { + // Try to click a non-existent element, which should fail (same selector as below) + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always failing test - same stacktrace 2", async () => { + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example D", async () => { + assert.equal(true, true); + }); + + it("always passing test - example A", async () => { + assert.strictEqual(1 + 1, 2, 'This test should always pass'); + }); + + it("Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("Another Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("always passing test - example B", async () => { + assert.strictEqual("Browser" + "Stack", "BrowserStack", 'This test should always pass'); + }); +}); diff --git a/ios/examples/run-parallel-test/moduleC/specs/test.js b/ios/examples/run-parallel-test/moduleC/specs/test.js new file mode 100644 index 0000000..5b0f46a --- /dev/null +++ b/ios/examples/run-parallel-test/moduleC/specs/test.js @@ -0,0 +1,74 @@ +const assert = require('assert'); + +describe("BStackDemo Tests Module C", () => { + before(async () => { + var textButton = await $(`~Text Button`); + await textButton.waitForDisplayed({ timeout: 30000 }); + await textButton.click(); + + var textInput = await $(`~Text Input`); + await textInput.waitForDisplayed({ timeout: 30000 }); + await textInput.click() + await textInput.addValue("hello@browserstack.com"+"\n"); + + var textOutput = await $(`~Text Output`); + await textOutput.waitForDisplayed({ timeout: 30000 }); + var value = await textOutput.getText(); + }); + + it("flaky test - passes and fails intermittently", async () => { + assert(Math.random() > 0.3, 'Flaky test failed'); + }); + + it("always failing test - missing element 1", async () => { + // Try to click a non-existent element, which should fail + const nonExistent = await $(`~non-existent-1`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example C", async () => { + assert.equal(true, true); + }); + + it("always passing test - example D", async () => { + assert.equal(true, true); + }); + + it("always failing test - same stacktrace 1", async () => { + // Try to click a non-existent element, which should fail (same selector as below) + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always failing test - same stacktrace 2", async () => { + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example A", async () => { + assert.strictEqual(1 + 1, 2, 'This test should always pass'); + }); + + it("Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("Another Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("always passing test - example B", async () => { + assert.strictEqual("Browser" + "Stack", "BrowserStack", 'This test should always pass'); + }); +}); diff --git a/ios/examples/run-parallel-test/parallel.conf.js b/ios/examples/run-parallel-test/parallel.conf.js index 0dec3e2..e5441f5 100644 --- a/ios/examples/run-parallel-test/parallel.conf.js +++ b/ios/examples/run-parallel-test/parallel.conf.js @@ -10,7 +10,11 @@ exports.config = { buildIdentifier: '${BUILD_NUMBER}', browserstackLocal: true, opts: { forcelocal: false, localIdentifier: "webdriverio-appium-app-browserstack-repo" }, - app: process.env.BROWSERSTACK_APP_PATH || './examples/BStackSampleApp.ipa' + app: process.env.BROWSERSTACK_APP_PATH || './examples/BStackSampleApp.ipa', + testObservability: true, + testObservabilityOptions: { + buildTag: ['bstack_sample'], + } } ] ], @@ -42,7 +46,7 @@ exports.config = { updateJob: false, specs: [ - './specs/single_test.js' + './m*/**' ], exclude: [],