diff --git a/.gitignore b/.gitignore index 0f481fd..15a686e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ node_modules/ local.log errorShots -.DS_Store \ No newline at end of file +.DS_Store +*.log +performance-report*.json +package-lock.json \ No newline at end of file diff --git a/README.md b/README.md index ba30460..fa73400 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,12 @@ Getting Started with Appium tests using WebdriverIO on BrowserStack couldn't be **Note**: For other test frameworks supported by App-Automate refer our [Developer documentation](https://www.browserstack.com/docs/) +## Running your tests +- To run parallel tests, run `npm run test` +- To run local test, run `npm run local` + + Understand how many parallel sessions you need by using our [Parallel Test Calculator](https://www.browserstack.com/automate/parallel-calculator?ref=github) + ## Getting Help If you are running into any issues or have any queries, please check [Browserstack Support page](https://www.browserstack.com/support/app-automate) or [get in touch with us](https://www.browserstack.com/contact?ref=help). diff --git a/android/examples/LocalSample.apk b/android/examples/LocalSample.apk new file mode 100644 index 0000000..f31c574 Binary files /dev/null and b/android/examples/LocalSample.apk differ diff --git a/android/examples/WikipediaSample.apk b/android/examples/WikipediaSample.apk new file mode 100644 index 0000000..286e346 Binary files /dev/null and b/android/examples/WikipediaSample.apk differ diff --git a/android/examples/run-first-test/first.conf.js b/android/examples/run-first-test/first.conf.js deleted file mode 100644 index aa83cf6..0000000 --- a/android/examples/run-first-test/first.conf.js +++ /dev/null @@ -1,34 +0,0 @@ -exports.config = { - user: process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME', - key: process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY', - - updateJob: false, - specs: [ - './examples/run-first-test/specs/first_test.js' - ], - exclude: [], - - capabilities: [{ - project: "First Webdriverio Android Project", - build: 'Webdriverio Android', - name: 'first_test', - device: 'Google Pixel 3', - os_version: "9.0", - app: process.env.BROWSERSTACK_APP_ID || 'bs://', - 'browserstack.debug': true - }], - - logLevel: 'info', - coloredLogs: true, - screenshotPath: './errorShots/', - baseUrl: '', - waitforTimeout: 10000, - connectionRetryTimeout: 90000, - connectionRetryCount: 3, - - framework: 'mocha', - mochaOpts: { - ui: 'bdd', - timeout: 20000 - } -}; diff --git a/android/examples/run-first-test/specs/first_test.js b/android/examples/run-first-test/specs/first_test.js deleted file mode 100644 index 271e958..0000000 --- a/android/examples/run-first-test/specs/first_test.js +++ /dev/null @@ -1,18 +0,0 @@ -var assert = require('assert'); - -describe('Search Wikipedia Functionality', () => { - it('can find search results', async () => { - var searchSelector = await $(`~Search Wikipedia`); - await searchSelector.waitForDisplayed({ timeout: 30000 }); - await searchSelector.click(); - - var insertTextSelector = await $('android=new UiSelector().resourceId("org.wikipedia.alpha:id/search_src_text")'); - await insertTextSelector.waitForDisplayed({ timeout: 30000 }); - - await insertTextSelector.addValue("Browsertack"); - await browser.pause(5000); - - var allProductsName = await $$(`android.widget.TextView`); - assert(allProductsName.length > 0); - }); -}); diff --git a/android/examples/run-local-test/local.conf.js b/android/examples/run-local-test/local.conf.js index 966c7c9..2833a57 100644 --- a/android/examples/run-local-test/local.conf.js +++ b/android/examples/run-local-test/local.conf.js @@ -1,26 +1,38 @@ -var browserstack = require('browserstack-local'); - exports.config = { user: process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME', key: process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY', + services: [ + [ + 'browserstack', + { + buildIdentifier: '${BUILD_NUMBER}', + browserstackLocal: true, + opts: { forcelocal: false, localIdentifier: "webdriverio-appium-app-browserstack-android-repo" }, + app: process.env.BROWSERSTACK_APP_PATH || './examples/LocalSample.apk' + } + ] + ], + + capabilities: [{ + 'bstack:options': { + projectName: "BrowserStack Samples", + buildName: 'browserstack build', + sessionName: 'BStack local webdriverio-appium', + deviceName: 'Google Pixel 3', + osVersion: "9.0", + debug: true, + networkLogs: true, + source: 'webdriverio:appium-sample-sdk:v1.0' + } + }], + updateJob: false, specs: [ './examples/run-local-test/specs/local_test.js' ], exclude: [], - capabilities: [{ - project: "First Webdriverio Android Project", - build: 'Webdriverio Android Local', - name: 'local_test', - device: 'Google Pixel 3', - os_version: "9.0", - app: process.env.BROWSERSTACK_APP_ID || 'bs://', - 'browserstack.local': true, - 'browserstack.debug': true - }], - logLevel: 'info', coloredLogs: true, screenshotPath: './errorShots/', @@ -33,32 +45,5 @@ exports.config = { mochaOpts: { ui: 'bdd', timeout: 20000 - }, - - // Code to start browserstack local before start of test - onPrepare: (config, capabilities) => { - console.log("Connecting local"); - return new Promise( (resolve, reject) => { - exports.bs_local = new browserstack.Local(); - exports.bs_local.start({'key': exports.config.key }, (error) => { - if (error) return reject(error); - console.log('Connected. Now testing...'); - - resolve(); - }); - }); - }, - - // Code to stop browserstack local after end of test - onComplete: (capabilties, specs) => { - console.log("Closing local tunnel"); - return new Promise( (resolve, reject) => { - exports.bs_local.stop( (error) => { - if (error) return reject(error); - console.log("Stopped BrowserStackLocal"); - - resolve(); - }); - }); } }; diff --git a/android/examples/run-multiple-test/multiple.conf.js b/android/examples/run-multiple-test/multiple.conf.js deleted file mode 100644 index c377060..0000000 --- a/android/examples/run-multiple-test/multiple.conf.js +++ /dev/null @@ -1,34 +0,0 @@ -exports.config = { - user: process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME', - key: process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY', - - updateJob: false, - specs: [ - './examples/run-multiple-test/specs/multiple_test.js' - ], - exclude: [], - - capabilities: [{ - project: "First Webdriverio Android Project", - build: 'Webdriverio Android Multiple', - name: 'multiple_test', - device: 'Google Pixel 3', - os_version: "9.0", - app: process.env.BROWSERSTACK_APP_ID || 'bs://', - 'browserstack.debug': true - }], - - logLevel: 'info', - coloredLogs: true, - screenshotPath: './errorShots/', - baseUrl: '', - waitforTimeout: 10000, - connectionRetryTimeout: 90000, - connectionRetryCount: 3, - - framework: 'mocha', - mochaOpts: { - ui: 'bdd', - timeout: 30000 - } -}; diff --git a/android/examples/run-multiple-test/specs/multiple/test_01.js b/android/examples/run-multiple-test/specs/multiple/test_01.js deleted file mode 100644 index fc33c42..0000000 --- a/android/examples/run-multiple-test/specs/multiple/test_01.js +++ /dev/null @@ -1,18 +0,0 @@ -var assert = require('assert'); - -describe('Search Wikipedia Functionality', () => { - it('can find search results', async () => { - var searchSelector = await $(`~Search Wikipedia`); - await searchSelector.waitForDisplayed({ timeout: 30000 }); - await searchSelector.click(); - - var insertTextSelector = await $('android=new UiSelector().resourceId("org.wikipedia.alpha:id/search_src_text")'); - await insertTextSelector.waitForDisplayed({ timeout: 30000 }); - - await insertTextSelector.addValue("Browsertack01"); - await browser.pause(5000); - - var allProductsName = await $$(`android.widget.TextView`); - assert(allProductsName.length > 0); - }); -}); diff --git a/android/examples/run-multiple-test/specs/multiple/test_02.js b/android/examples/run-multiple-test/specs/multiple/test_02.js deleted file mode 100644 index ac5f06a..0000000 --- a/android/examples/run-multiple-test/specs/multiple/test_02.js +++ /dev/null @@ -1,18 +0,0 @@ -var assert = require('assert'); - -describe('Search Wikipedia Functionality', () => { - it('can find search results', async () => { - var searchSelector = await $(`~Search Wikipedia`); - await searchSelector.waitForDisplayed({ timeout: 30000 }); - await searchSelector.click(); - - var insertTextSelector = await $('android=new UiSelector().resourceId("org.wikipedia.alpha:id/search_src_text")'); - await insertTextSelector.waitForDisplayed({ timeout: 30000 }); - - await insertTextSelector.addValue("Browsertack02"); - await browser.pause(5000); - - var allProductsName = await $$(`android.widget.TextView`); - assert(allProductsName.length > 0); - }); -}); diff --git a/android/examples/run-multiple-test/specs/multiple/test_03.js b/android/examples/run-multiple-test/specs/multiple/test_03.js deleted file mode 100644 index bbe3bdf..0000000 --- a/android/examples/run-multiple-test/specs/multiple/test_03.js +++ /dev/null @@ -1,18 +0,0 @@ -var assert = require('assert'); - -describe('Search Wikipedia Functionality', () => { - it('can find search results', async () => { - var searchSelector = await $(`~Search Wikipedia`); - await searchSelector.waitForDisplayed({ timeout: 30000 }); - await searchSelector.click(); - - var insertTextSelector = await $('android=new UiSelector().resourceId("org.wikipedia.alpha:id/search_src_text")'); - await insertTextSelector.waitForDisplayed({ timeout: 30000 }); - - await insertTextSelector.addValue("Browsertack03"); - await browser.pause(5000); - - var allProductsName = await $$(`android.widget.TextView`); - assert(allProductsName.length > 0); - }); -}); diff --git a/android/examples/run-multiple-test/specs/multiple_test.js b/android/examples/run-multiple-test/specs/multiple_test.js deleted file mode 100644 index af52ba7..0000000 --- a/android/examples/run-multiple-test/specs/multiple_test.js +++ /dev/null @@ -1,9 +0,0 @@ -var specs = [ - './multiple/test_01.js', - './multiple/test_02.js', - './multiple/test_03.js' -]; - -for (var i = specs.length - 1; i >= 0; i--) { - require(specs[i]); -} diff --git a/android/examples/run-parallel-test/moduleA/specs/test.js b/android/examples/run-parallel-test/moduleA/specs/test.js new file mode 100644 index 0000000..fd0be5a --- /dev/null +++ b/android/examples/run-parallel-test/moduleA/specs/test.js @@ -0,0 +1,81 @@ +const assert = require('assert'); + +describe("BStackDemo Tests Module A", () => { + before(async () => { + var skipButton = await $('android=new UiSelector().resourceId("org.wikipedia.alpha:id/fragment_onboarding_skip_button")'); + await skipButton.waitForDisplayed({ timeout: 30000 }); + await skipButton.click(); + + var searchSelector = await $(`~Search Wikipedia`); + await searchSelector.waitForDisplayed({ timeout: 30000 }); + await searchSelector.click(); + }); + + it("flaky test - random product selection", async () => { + // this.tags = ['regression', 'p1']; + const selector = Math.random() > 0.5 ? 'android=new UiSelector().resourceId("org.wikipedia.alpha:id/search_src_text")' : 'falseSelector'; + const insertTextSelector = await $(selector); + await insertTextSelector.waitForDisplayed({ timeout: 30000 }); + + // Randomly choose a term that will always have results or never have results + const randomTerm = Math.random() > 0.5 ? "BrowserStack" : "akjsdhfakjsdhf"; // nonsense string + await insertTextSelector.addValue(randomTerm); + await browser.pause(5000); + + const allResults = await $$(`android.widget.TextView`); + // This assertion will sometimes fail if randomTerm is the nonsense string + assert(allResults.length > 0); + }); + + it("always failing test - missing element 1", async () => { + // Try to click a non-existent element, which should fail + const nonExistent = await $(`~non-existent-1`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example C", async () => { + assert.equal(true, true); + }); + + it("always failing test - same stacktrace 1", async () => { + // Try to click a non-existent element, which should fail (same selector as below) + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always failing test - same stacktrace 2", async () => { + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example D", async () => { + assert.equal(true, true); + }); + + it("always passing test - example A", async () => { + assert.strictEqual(1 + 1, 2, 'This test should always pass'); + }); + + it("Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("Another Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("always passing test - example B", async () => { + assert.strictEqual("Browser" + "Stack", "BrowserStack", 'This test should always pass'); + }); +}); \ No newline at end of file diff --git a/android/examples/run-parallel-test/moduleB/specs/test.js b/android/examples/run-parallel-test/moduleB/specs/test.js new file mode 100644 index 0000000..9edd5fd --- /dev/null +++ b/android/examples/run-parallel-test/moduleB/specs/test.js @@ -0,0 +1,89 @@ +const assert = require('assert'); + +describe("BStackDemo Tests Module B", () => { + before(async () => { + var skipButton = await $('android=new UiSelector().resourceId("org.wikipedia.alpha:id/fragment_onboarding_skip_button")'); + await skipButton.waitForDisplayed({ timeout: 30000 }); + await skipButton.click(); + + var searchSelector = await $(`~Search Wikipedia`); + await searchSelector.waitForDisplayed({ timeout: 30000 }); + await searchSelector.click(); + + // var insertTextSelector = await $('android=new UiSelector().resourceId("org.wikipedia.alpha:id/search_src_text")'); + // await insertTextSelector.waitForDisplayed({ timeout: 30000 }); + + // await insertTextSelector.addValue("BrowserStack"); + // await browser.pause(5000); + + // var allProductsName = await $$(`android.widget.TextView`); + // assert(allProductsName.length > 0, 'Search results should be present'); + }); + + it("flaky test - random product selection", async () => { + const selector = Math.random() > 0.5 ? 'android=new UiSelector().resourceId("org.wikipedia.alpha:id/search_src_text")' : 'falseSelector'; + const insertTextSelector = await $(selector); + await insertTextSelector.waitForDisplayed({ timeout: 30000 }); + + // Randomly choose a term that will always have results or never have results + const randomTerm = Math.random() > 0.5 ? "BrowserStack" : "akjsdhfakjsdhf"; // nonsense string + await insertTextSelector.addValue(randomTerm); + await browser.pause(5000); + + const allResults = await $$(`android.widget.TextView`); + // This assertion will sometimes fail if randomTerm is the nonsense string + assert(allResults.length > 0); + }); + + it("always failing test - missing element 1", async () => { + // Try to click a non-existent element, which should fail + const nonExistent = await $(`~non-existent-1`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example C", async () => { + assert.equal(true, true); + }); + + it("always failing test - same stacktrace 1", async () => { + // Try to click a non-existent element, which should fail (same selector as below) + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always failing test - same stacktrace 2", async () => { + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example D", async () => { + assert.equal(true, true); + }); + + it("always passing test - example A", async () => { + assert.strictEqual(1 + 1, 2, 'This test should always pass'); + }); + + it("Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("Another Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("always passing test - example B", async () => { + assert.strictEqual("Browser" + "Stack", "BrowserStack", 'This test should always pass'); + }); +}); \ No newline at end of file diff --git a/android/examples/run-parallel-test/moduleC/specs/test.js b/android/examples/run-parallel-test/moduleC/specs/test.js new file mode 100644 index 0000000..6fbfb4d --- /dev/null +++ b/android/examples/run-parallel-test/moduleC/specs/test.js @@ -0,0 +1,89 @@ +const assert = require('assert'); + +describe("BStackDemo Tests Module C", () => { + before(async () => { + var skipButton = await $('android=new UiSelector().resourceId("org.wikipedia.alpha:id/fragment_onboarding_skip_button")'); + await skipButton.waitForDisplayed({ timeout: 30000 }); + await skipButton.click(); + + var searchSelector = await $(`~Search Wikipedia`); + await searchSelector.waitForDisplayed({ timeout: 30000 }); + await searchSelector.click(); + + // var insertTextSelector = await $('android=new UiSelector().resourceId("org.wikipedia.alpha:id/search_src_text")'); + // await insertTextSelector.waitForDisplayed({ timeout: 30000 }); + + // await insertTextSelector.addValue("BrowserStack"); + // await browser.pause(5000); + + // var allProductsName = await $$(`android.widget.TextView`); + // assert(allProductsName.length > 0, 'Search results should be present'); + }); + + it("flaky test - random product selection", async () => { + const selector = Math.random() > 0.5 ? 'android=new UiSelector().resourceId("org.wikipedia.alpha:id/search_src_text")' : 'falseSelector'; + const insertTextSelector = await $(selector); + await insertTextSelector.waitForDisplayed({ timeout: 30000 }); + + // Randomly choose a term that will always have results or never have results + const randomTerm = Math.random() > 0.5 ? "BrowserStack" : "akjsdhfakjsdhf"; // nonsense string + await insertTextSelector.addValue(randomTerm); + await browser.pause(5000); + + const allResults = await $$(`android.widget.TextView`); + // This assertion will sometimes fail if randomTerm is the nonsense string + assert(allResults.length > 0); + }); + + it("always failing test - missing element 1", async () => { + // Try to click a non-existent element, which should fail + const nonExistent = await $(`~non-existent-1`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example C", async () => { + assert.equal(true, true); + }); + + it("always passing test - example D", async () => { + assert.equal(true, true); + }); + + it("always failing test - same stacktrace 1", async () => { + // Try to click a non-existent element, which should fail (same selector as below) + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always failing test - same stacktrace 2", async () => { + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example A", async () => { + assert.strictEqual(1 + 1, 2, 'This test should always pass'); + }); + + it("Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("Another Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("always passing test - example B", async () => { + assert.strictEqual("Browser" + "Stack", "BrowserStack", 'This test should always pass'); + }); +}); \ No newline at end of file diff --git a/android/examples/run-parallel-test/parallel.conf.js b/android/examples/run-parallel-test/parallel.conf.js index 930b16b..7941dad 100644 --- a/android/examples/run-parallel-test/parallel.conf.js +++ b/android/examples/run-parallel-test/parallel.conf.js @@ -2,29 +2,56 @@ exports.config = { user: process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME', key: process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY', - updateJob: false, - specs: [ - './examples/run-parallel-test/specs/single_test.js' - ], - exclude: [], + hostname: 'hub.browserstack.com', - maxInstances: 10, - commonCapabilities: { - project: "First Webdriverio Android Project", - build: 'Webdriverio Android Parallel', - name: 'parallel_test', - app: process.env.BROWSERSTACK_APP_ID || 'bs://', - 'browserstack.debug': true - }, + services: [ + [ + 'browserstack', + { + accessibility: false, + buildIdentifier: '${BUILD_NUMBER}', + browserstackLocal: true, + opts: { forcelocal: false, localIdentifier: "webdriverio-appium-app-browserstack-repo" }, + app: process.env.BROWSERSTACK_APP_PATH || './examples/WikipediaSample.apk', + testObservability: true, + testObservabilityOptions: { + buildTag: ['bstack_sample'], + } + } + ] + ], capabilities: [{ - device: 'Google Pixel 3', - os_version: "9.0" + 'bstack:options': { + deviceName: 'Google Pixel 8', + osVersion: "14.0" + } }, { - device: 'Samsung Galaxy S10e', - os_version: "9.0" + 'bstack:options': { + deviceName: 'Samsung Galaxy S21', + osVersion: "11.0" + } }], + commonCapabilities: { + 'bstack:options': { + projectName: "BrowserStack Samples", + buildName: 'browserstack build', + sessionName: 'BStack parallel webdriverio-appium', + debug: true, + networkLogs: true, + source: 'webdriverio:appium-sample-sdk:v1.0' + } + }, + + maxInstances: 10, + + updateJob: false, + specs: [ + './m*/**' + ], + exclude: [], + logLevel: 'info', coloredLogs: true, screenshotPath: './errorShots/', @@ -36,11 +63,12 @@ exports.config = { framework: 'mocha', mochaOpts: { ui: 'bdd', - timeout: 20000 + timeout: 40000 } }; // Code to support common capabilities exports.config.capabilities.forEach(function(caps){ - for(var i in exports.config.commonCapabilities) caps[i] = caps[i] || exports.config.commonCapabilities[i]; + for(let key in exports.config.commonCapabilities) + caps[key] = { ...caps[key], ...exports.config.commonCapabilities[key]}; }); diff --git a/android/examples/run-parallel-test/specs/single_test.js b/android/examples/run-parallel-test/specs/single_test.js index 271e958..4507b2a 100644 --- a/android/examples/run-parallel-test/specs/single_test.js +++ b/android/examples/run-parallel-test/specs/single_test.js @@ -2,6 +2,11 @@ var assert = require('assert'); describe('Search Wikipedia Functionality', () => { it('can find search results', async () => { + + var skipButton = await $('android=new UiSelector().resourceId("org.wikipedia.alpha:id/fragment_onboarding_skip_button")'); + await skipButton.waitForDisplayed({ timeout: 30000 }); + await skipButton.click(); + var searchSelector = await $(`~Search Wikipedia`); await searchSelector.waitForDisplayed({ timeout: 30000 }); await searchSelector.click(); @@ -9,7 +14,7 @@ describe('Search Wikipedia Functionality', () => { var insertTextSelector = await $('android=new UiSelector().resourceId("org.wikipedia.alpha:id/search_src_text")'); await insertTextSelector.waitForDisplayed({ timeout: 30000 }); - await insertTextSelector.addValue("Browsertack"); + await insertTextSelector.addValue("BrowserStack"); await browser.pause(5000); var allProductsName = await $$(`android.widget.TextView`); diff --git a/android/package.json b/android/package.json index 2cfca95..5778b11 100644 --- a/android/package.json +++ b/android/package.json @@ -4,11 +4,8 @@ "readme": "WendriverIO Integration with [BrowserStack](https://www.browserstack.com)", "description": "Selenium examples for WebdriverIO and BrowserStack App Automate", "scripts": { - "test": "npm run first && npm run local && npm run parallel", - "first": "./node_modules/.bin/wdio examples/run-first-test/first.conf.js", - "parallel": "./node_modules/.bin/wdio examples/run-parallel-test/parallel.conf.js", - "local": "./node_modules/.bin/wdio examples/run-local-test/local.conf.js", - "multiple": "./node_modules/.bin/wdio examples/run-multiple-test/multiple.conf.js" + "test": "npx wdio examples/run-parallel-test/parallel.conf.js", + "local": "npx wdio examples/run-local-test/local.conf.js" }, "repository": { "type": "git", @@ -25,9 +22,13 @@ }, "homepage": "https://github.com/browserstack/webdriverio-appium-app-browserstack#readme", "dependencies": { - "@wdio/cli": "^5.20.1", - "@wdio/local-runner": "^5.20.1", - "@wdio/mocha-framework": "^5.18.7", - "browserstack-local": "^1.4.5" + "@wdio/cli": "^9", + "chai": "^4.3.6", + "webdriverio": "^9" + }, + "devDependencies": { + "@wdio/browserstack-service": "^9", + "@wdio/local-runner": "^9", + "@wdio/mocha-framework": "^9" } } diff --git a/ios/examples/BStackSampleApp.ipa b/ios/examples/BStackSampleApp.ipa new file mode 100644 index 0000000..c1891b8 Binary files /dev/null and b/ios/examples/BStackSampleApp.ipa differ diff --git a/ios/examples/LocalSample.ipa b/ios/examples/LocalSample.ipa new file mode 100644 index 0000000..a937349 Binary files /dev/null and b/ios/examples/LocalSample.ipa differ diff --git a/ios/examples/run-first-test/first.conf.js b/ios/examples/run-first-test/first.conf.js deleted file mode 100644 index 2462928..0000000 --- a/ios/examples/run-first-test/first.conf.js +++ /dev/null @@ -1,34 +0,0 @@ -exports.config = { - user: process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME', - key: process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY', - - updateJob: false, - specs: [ - './examples/run-first-test/specs/first_test.js' - ], - exclude: [], - - capabilities: [{ - project: "First Webdriverio iOS Project", - build: 'Webdriverio iOS', - name: 'single_test', - device: 'iPhone 11 Pro', - os_version: "13", - app: process.env.BROWSERSTACK_APP_ID || 'bs://', - 'browserstack.debug': true - }], - - logLevel: 'info', - coloredLogs: true, - screenshotPath: './errorShots/', - baseUrl: '', - waitforTimeout: 10000, - connectionRetryTimeout: 90000, - connectionRetryCount: 3, - - framework: 'mocha', - mochaOpts: { - ui: 'bdd', - timeout: 40000 - } -}; diff --git a/ios/examples/run-first-test/specs/first_test.js b/ios/examples/run-first-test/specs/first_test.js deleted file mode 100644 index ef4e4ac..0000000 --- a/ios/examples/run-first-test/specs/first_test.js +++ /dev/null @@ -1,23 +0,0 @@ -var assert = require('assert'); - -describe('Text Verification', () => { - it('should match displayed text with input text', async () => { - var textButton = await $(`~Text Button`); - await textButton.waitForDisplayed({ timeout: 30000 }); - await textButton.click(); - - var textInput = await $(`~Text Input`); - await textInput.waitForDisplayed({ timeout: 30000 }); - await textInput.click() - await textInput.addValue("hello@browserstack.com"+"\n"); - - var textOutput = await $(`~Text Output`); - await textOutput.waitForDisplayed({ timeout: 30000 }); - var value = await textOutput.getText(); - - if (value === "hello@browserstack.com") - assert(true) - else - assert(false) - }); -}); diff --git a/ios/examples/run-local-test/local.conf.js b/ios/examples/run-local-test/local.conf.js index 8fd8bd3..5194597 100644 --- a/ios/examples/run-local-test/local.conf.js +++ b/ios/examples/run-local-test/local.conf.js @@ -1,26 +1,38 @@ -var browserstack = require('browserstack-local'); - exports.config = { user: process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME', key: process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY', + services: [ + [ + 'browserstack', + { + buildIdentifier: '${BUILD_NUMBER}', + browserstackLocal: true, + opts: { forcelocal: false, localIdentifier: "webdriverio-appium-app-browserstack-ios-repo" }, + app: process.env.BROWSERSTACK_APP_PATH || './examples/LocalSample.ipa' + } + ] + ], + + capabilities: [{ + 'bstack:options': { + projectName: "BrowserStack Samples", + buildName: 'browserstack build', + sessionName: 'BStack local webdriverio-appium', + deviceName: 'iPhone 14', + osVersion: "16", + debug: true, + networkLogs: true, + source: 'webdriverio:appium-sample-sdk:v1.0' + } + }], + updateJob: false, specs: [ './examples/run-local-test/specs/local_test.js' ], exclude: [], - capabilities: [{ - project: "First Webdriverio iOS Project", - build: 'Webdriverio iOS Local', - name: 'local_test', - device: 'iPhone 11 Pro', - os_version: "13", - app: process.env.BROWSERSTACK_APP_ID || 'bs://', - 'browserstack.local': true, - 'browserstack.debug': true - }], - logLevel: 'info', coloredLogs: true, screenshotPath: './errorShots/', @@ -33,32 +45,5 @@ exports.config = { mochaOpts: { ui: 'bdd', timeout: 30000 - }, - - // Code to start browserstack local before start of test - onPrepare: (config, capabilities) => { - console.log("Connecting local"); - return new Promise( (resolve, reject) => { - exports.bs_local = new browserstack.Local(); - exports.bs_local.start({'key': exports.config.key }, (error) => { - if (error) return reject(error); - console.log('Connected. Now testing...'); - - resolve(); - }); - }); - }, - - // Code to stop browserstack local after end of test - onComplete: (capabilties, specs) => { - console.log("Closing local tunnel"); - return new Promise( (resolve, reject) => { - exports.bs_local.stop( (error) => { - if (error) return reject(error); - console.log("Stopped BrowserStackLocal"); - - resolve(); - }); - }); } }; diff --git a/ios/examples/run-local-test/specs/local_test.js b/ios/examples/run-local-test/specs/local_test.js index 94fdf1f..86a3a01 100644 --- a/ios/examples/run-local-test/specs/local_test.js +++ b/ios/examples/run-local-test/specs/local_test.js @@ -25,6 +25,6 @@ describe('BrowserStack Local Testing', () => { } var matchedString = await testElement.getText(); - assert(matchedString == 'Response is: Up and running'); + assert(matchedString == 'Up and running'); }); }); diff --git a/ios/examples/run-multiple-test/multiple.conf.js b/ios/examples/run-multiple-test/multiple.conf.js deleted file mode 100644 index b67b1eb..0000000 --- a/ios/examples/run-multiple-test/multiple.conf.js +++ /dev/null @@ -1,34 +0,0 @@ -exports.config = { - user: process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME', - key: process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY', - - updateJob: false, - specs: [ - './examples/run-multiple-test/specs/multiple_test.js' - ], - exclude: [], - - capabilities: [{ - project: "First Webdriverio iOS Project", - build: 'Webdriverio iOS Multiple', - name: 'multiple_test', - device: 'iPhone 11 Pro', - os_version: "13", - app: process.env.BROWSERSTACK_APP_ID || 'bs://', - 'browserstack.debug': true - }], - - logLevel: 'info', - coloredLogs: true, - screenshotPath: './errorShots/', - baseUrl: '', - waitforTimeout: 10000, - connectionRetryTimeout: 90000, - connectionRetryCount: 3, - - framework: 'mocha', - mochaOpts: { - ui: 'bdd', - timeout: 30000 - } -}; diff --git a/ios/examples/run-multiple-test/specs/multiple/test_01.js b/ios/examples/run-multiple-test/specs/multiple/test_01.js deleted file mode 100644 index fe0a1b9..0000000 --- a/ios/examples/run-multiple-test/specs/multiple/test_01.js +++ /dev/null @@ -1,26 +0,0 @@ -var assert = require('assert'); - -describe('Text Verification', () => { - it('should match displayed text with input text', async () => { - var textButton = await $(`~Text Button`); - await textButton.waitForDisplayed({ timeout: 30000 }); - await textButton.click(); - - var textInput = await $(`~Text Input`); - await textInput.waitForDisplayed({ timeout: 30000 }); - await textInput.click() - await textInput.addValue("hello01@browserstack.com"+"\n"); - - var textOutput = await $(`~Text Output`); - await textOutput.waitForDisplayed({ timeout: 30000 }); - var value = await textOutput.getText(); - - if (value === "hello01@browserstack.com") - assert(true) - else - assert(false) - - var back = await $('~UI Elements'); - await back.click(); - }); -}); diff --git a/ios/examples/run-multiple-test/specs/multiple/test_02.js b/ios/examples/run-multiple-test/specs/multiple/test_02.js deleted file mode 100644 index 8b1d9dc..0000000 --- a/ios/examples/run-multiple-test/specs/multiple/test_02.js +++ /dev/null @@ -1,26 +0,0 @@ -var assert = require('assert'); - -describe('Text Verification', () => { - it('should match displayed text with input text', async () => { - var textButton = await $(`~Text Button`); - await textButton.waitForDisplayed({ timeout: 30000 }); - await textButton.click(); - - var textInput = await $(`~Text Input`); - await textInput.waitForDisplayed({ timeout: 30000 }); - await textInput.click() - await textInput.addValue("hello02@browserstack.com"+"\n"); - - var textOutput = await $(`~Text Output`); - await textOutput.waitForDisplayed({ timeout: 30000 }); - var value = await textOutput.getText(); - - if (value === "hello02@browserstack.com") - assert(true) - else - assert(false) - - var back = await $('~UI Elements'); - await back.click(); - }); -}); diff --git a/ios/examples/run-multiple-test/specs/multiple/test_03.js b/ios/examples/run-multiple-test/specs/multiple/test_03.js deleted file mode 100644 index e9862ab..0000000 --- a/ios/examples/run-multiple-test/specs/multiple/test_03.js +++ /dev/null @@ -1,26 +0,0 @@ -var assert = require('assert'); - -describe('Text Verification', () => { - it('should match displayed text with input text', async () => { - var textButton = await $(`~Text Button`); - await textButton.waitForDisplayed({ timeout: 30000 }); - await textButton.click(); - - var textInput = await $(`~Text Input`); - await textInput.waitForDisplayed({ timeout: 30000 }); - await textInput.click() - await textInput.addValue("hello03@browserstack.com"+"\n"); - - var textOutput = await $(`~Text Output`); - await textOutput.waitForDisplayed({ timeout: 30000 }); - var value = await textOutput.getText(); - - if (value === "hello03@browserstack.com") - assert(true) - else - assert(false) - - var back = await $('~UI Elements'); - await back.click(); - }); -}); diff --git a/ios/examples/run-multiple-test/specs/multiple_test.js b/ios/examples/run-multiple-test/specs/multiple_test.js deleted file mode 100644 index af52ba7..0000000 --- a/ios/examples/run-multiple-test/specs/multiple_test.js +++ /dev/null @@ -1,9 +0,0 @@ -var specs = [ - './multiple/test_01.js', - './multiple/test_02.js', - './multiple/test_03.js' -]; - -for (var i = specs.length - 1; i >= 0; i--) { - require(specs[i]); -} diff --git a/ios/examples/run-parallel-test/moduleA/specs/test.js b/ios/examples/run-parallel-test/moduleA/specs/test.js new file mode 100644 index 0000000..ae1cae1 --- /dev/null +++ b/ios/examples/run-parallel-test/moduleA/specs/test.js @@ -0,0 +1,74 @@ +const assert = require('assert'); + +describe("BStackDemo Tests Module A", () => { + before(async () => { + var textButton = await $(`~Text Button`); + await textButton.waitForDisplayed({ timeout: 30000 }); + await textButton.click(); + + var textInput = await $(`~Text Input`); + await textInput.waitForDisplayed({ timeout: 30000 }); + await textInput.click() + await textInput.addValue("hello@browserstack.com"+"\n"); + + var textOutput = await $(`~Text Output`); + await textOutput.waitForDisplayed({ timeout: 30000 }); + var value = await textOutput.getText(); + }); + + it("flaky test - passes and fails intermittently", async () => { + assert(Math.random() > 0.3, 'Flaky test failed'); + }); + + it("always failing test - missing element 1", async () => { + // Try to click a non-existent element, which should fail + const nonExistent = await $(`~non-existent-1`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example C", async () => { + assert.equal(true, true); + }); + + it("always failing test - same stacktrace 1", async () => { + // Try to click a non-existent element, which should fail (same selector as below) + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always failing test - same stacktrace 2", async () => { + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example D", async () => { + assert.equal(true, true); + }); + + it("always passing test - example A", async () => { + assert.strictEqual(1 + 1, 2, 'This test should always pass'); + }); + + it("Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("Another Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("always passing test - example B", async () => { + assert.strictEqual("Browser" + "Stack", "BrowserStack", 'This test should always pass'); + }); +}); diff --git a/ios/examples/run-parallel-test/moduleB/specs/test.js b/ios/examples/run-parallel-test/moduleB/specs/test.js new file mode 100644 index 0000000..2c7cf0c --- /dev/null +++ b/ios/examples/run-parallel-test/moduleB/specs/test.js @@ -0,0 +1,74 @@ +const assert = require('assert'); + +describe("BStackDemo Tests Module B", () => { + before(async () => { + var textButton = await $(`~Text Button`); + await textButton.waitForDisplayed({ timeout: 30000 }); + await textButton.click(); + + var textInput = await $(`~Text Input`); + await textInput.waitForDisplayed({ timeout: 30000 }); + await textInput.click() + await textInput.addValue("hello@browserstack.com"+"\n"); + + var textOutput = await $(`~Text Output`); + await textOutput.waitForDisplayed({ timeout: 30000 }); + var value = await textOutput.getText(); + }); + + it("flaky test - passes and fails intermittently", async () => { + assert(Math.random() > 0.3, 'Flaky test failed'); + }); + + it("always failing test - missing element 1", async () => { + // Try to click a non-existent element, which should fail + const nonExistent = await $(`~non-existent-1`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example C", async () => { + assert.equal(true, true); + }); + + it("always failing test - same stacktrace 1", async () => { + // Try to click a non-existent element, which should fail (same selector as below) + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always failing test - same stacktrace 2", async () => { + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example D", async () => { + assert.equal(true, true); + }); + + it("always passing test - example A", async () => { + assert.strictEqual(1 + 1, 2, 'This test should always pass'); + }); + + it("Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("Another Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("always passing test - example B", async () => { + assert.strictEqual("Browser" + "Stack", "BrowserStack", 'This test should always pass'); + }); +}); diff --git a/ios/examples/run-parallel-test/moduleC/specs/test.js b/ios/examples/run-parallel-test/moduleC/specs/test.js new file mode 100644 index 0000000..5b0f46a --- /dev/null +++ b/ios/examples/run-parallel-test/moduleC/specs/test.js @@ -0,0 +1,74 @@ +const assert = require('assert'); + +describe("BStackDemo Tests Module C", () => { + before(async () => { + var textButton = await $(`~Text Button`); + await textButton.waitForDisplayed({ timeout: 30000 }); + await textButton.click(); + + var textInput = await $(`~Text Input`); + await textInput.waitForDisplayed({ timeout: 30000 }); + await textInput.click() + await textInput.addValue("hello@browserstack.com"+"\n"); + + var textOutput = await $(`~Text Output`); + await textOutput.waitForDisplayed({ timeout: 30000 }); + var value = await textOutput.getText(); + }); + + it("flaky test - passes and fails intermittently", async () => { + assert(Math.random() > 0.3, 'Flaky test failed'); + }); + + it("always failing test - missing element 1", async () => { + // Try to click a non-existent element, which should fail + const nonExistent = await $(`~non-existent-1`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example C", async () => { + assert.equal(true, true); + }); + + it("always passing test - example D", async () => { + assert.equal(true, true); + }); + + it("always failing test - same stacktrace 1", async () => { + // Try to click a non-existent element, which should fail (same selector as below) + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always failing test - same stacktrace 2", async () => { + const nonExistent = await $(`~common-error`); + await nonExistent.waitForDisplayed({ timeout: 3000 }); + await nonExistent.click(); // Will fail + }); + + it("always passing test - example A", async () => { + assert.strictEqual(1 + 1, 2, 'This test should always pass'); + }); + + it("Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("Another Test with framework-level retry - 2 retries configured", function () { + this.retries(2); // Framework-level retry + const randomOutcome = Math.random() > 0.7; + if (!randomOutcome) { + throw new Error("Test failed, retrying..."); + } + }); + + it("always passing test - example B", async () => { + assert.strictEqual("Browser" + "Stack", "BrowserStack", 'This test should always pass'); + }); +}); diff --git a/ios/examples/run-parallel-test/parallel.conf.js b/ios/examples/run-parallel-test/parallel.conf.js index 8012251..e5441f5 100644 --- a/ios/examples/run-parallel-test/parallel.conf.js +++ b/ios/examples/run-parallel-test/parallel.conf.js @@ -2,29 +2,55 @@ exports.config = { user: process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME', key: process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY', - updateJob: false, - specs: [ - './examples/run-parallel-test/specs/single_test.js' + services: [ + [ + 'browserstack', + { + accessibility: false, + buildIdentifier: '${BUILD_NUMBER}', + browserstackLocal: true, + opts: { forcelocal: false, localIdentifier: "webdriverio-appium-app-browserstack-repo" }, + app: process.env.BROWSERSTACK_APP_PATH || './examples/BStackSampleApp.ipa', + testObservability: true, + testObservabilityOptions: { + buildTag: ['bstack_sample'], + } + } + ] ], - exclude: [], - - maxInstances: 10, - commonCapabilities: { - project: "First Webdriverio iOS Project", - build: 'Webdriverio iOS Parallel', - name: 'parallel_test', - app: process.env.BROWSERSTACK_APP_ID || 'bs://', - 'browserstack.debug': true - }, capabilities: [{ - device: "iPhone 11 Pro", - os_version: "13" + 'bstack:options': { + deviceName: "iPhone 14", + osVersion: "16" + } }, { - device: "iPhone 11 Pro Max", - os_version: "13" + 'bstack:options': { + deviceName: "iPhone 13 Pro Max", + osVersion: "15" + } }], + commonCapabilities: { + 'bstack:options': { + projectName: "BrowserStack Samples", + buildName: 'browserstack build', + sessionName: 'BStack parallel webdriverio-appium', + debug: true, + networkLogs: true, + source: 'webdriverio:appium-sample-sdk:v1.0' + } + }, + + maxInstances: 10, + + updateJob: false, + specs: [ + './m*/**' + ], + exclude: [], + + logLevel: 'info', coloredLogs: true, screenshotPath: './errorShots/', @@ -42,5 +68,6 @@ exports.config = { // Code to support common capabilities exports.config.capabilities.forEach(function(caps){ - for(var i in exports.config.commonCapabilities) caps[i] = caps[i] || exports.config.commonCapabilities[i]; + for(let key in exports.config.commonCapabilities) + caps[key] = { ...caps[key], ...exports.config.commonCapabilities[key]}; }); diff --git a/ios/package.json b/ios/package.json index 9b5953d..5778b11 100644 --- a/ios/package.json +++ b/ios/package.json @@ -4,11 +4,8 @@ "readme": "WendriverIO Integration with [BrowserStack](https://www.browserstack.com)", "description": "Selenium examples for WebdriverIO and BrowserStack App Automate", "scripts": { - "test": "npm run first && npm run local && npm run parallel", - "first": "./node_modules/.bin/wdio examples/run-first-test/first.conf.js", - "parallel": "./node_modules/.bin/wdio examples/run-parallel-test/parallel.conf.js", - "local": "./node_modules/.bin/wdio examples/run-local-test/local.conf.js", - "multiple": "./node_modules/.bin/wdio examples/run-multiple-test/multiple.conf.js" + "test": "npx wdio examples/run-parallel-test/parallel.conf.js", + "local": "npx wdio examples/run-local-test/local.conf.js" }, "repository": { "type": "git", @@ -25,9 +22,13 @@ }, "homepage": "https://github.com/browserstack/webdriverio-appium-app-browserstack#readme", "dependencies": { - "browserstack-local": "^1.4.5", - "@wdio/cli": "^5.20.1", - "@wdio/local-runner": "^5.20.1", - "@wdio/mocha-framework": "^5.18.7" + "@wdio/cli": "^9", + "chai": "^4.3.6", + "webdriverio": "^9" + }, + "devDependencies": { + "@wdio/browserstack-service": "^9", + "@wdio/local-runner": "^9", + "@wdio/mocha-framework": "^9" } }