diff --git a/build-test-publish.pipeline.yml b/build-test-publish.pipeline.yml index a33ff971..f6cb0c23 100644 --- a/build-test-publish.pipeline.yml +++ b/build-test-publish.pipeline.yml @@ -1,4 +1,4 @@ -name: 1.13$(Rev:.r) +name: 1.14$(Rev:.r) trigger: - master diff --git a/integ-test-promote-template.yml b/integ-test-promote-template.yml index cf50b902..474a8501 100644 --- a/integ-test-promote-template.yml +++ b/integ-test-promote-template.yml @@ -209,7 +209,9 @@ jobs: displayName: Run MATLAB tests producing artifacts inputs: testResultsJUnit: test-results/matlab/results.xml + testResultsHTML: test-results/matlab/resultsHTML codeCoverageCobertura: code-coverage/coverage.xml + codeCoverageHTML: code-coverage/coverageHTML sourceFolder: src - task: MathWorks.matlab-azure-devops-extension-dev.RunMATLABTests.RunMATLABTests@${{ version }} displayName: Run MATLAB tests filter by folder @@ -225,10 +227,22 @@ jobs: set -e grep -q FirstTest test-results/matlab/results.xml displayName: Verify test results file was created + - bash: | + set -e + test -d test-results/matlab/resultsHTML + test -f test-results/matlab/resultsHTML/*.html + displayName: Verify HTML test results folder and HTML file was created + condition: not(eq(${{ version }}, '0')) - bash: | set -e grep -q add code-coverage/coverage.xml displayName: Verify code coverage file was created + - bash: | + set -e + test -d code-coverage/coverageHTML + test -f code-coverage/coverageHTML/*.html + displayName: Verify HTML code coverage folder and HTML file was created + condition: not(eq(${{ version }}, '0')) - bash: | set -e grep -q simpleTest test-results/matlab/selectbyfolder.xml @@ -273,7 +287,7 @@ jobs: sourceFolder: src - bash: | set -e - grep -q "TestRunner.withTextOutput('OutputDetail', 3)" console.log + grep -q "('OutputDetail', 3)" console.log rm console.log displayName: Verify tests ran with detailed display level for event details - task: MathWorks.matlab-azure-devops-extension-dev.RunMATLABTests.RunMATLABTests@${{ version }} @@ -283,7 +297,7 @@ jobs: sourceFolder: src - bash: | set -e - grep -q "TestRunner.withTextOutput('LoggingLevel', 3)" console.log + grep -q "('LoggingLevel', 3)" console.log rm console.log displayName: Verify tests ran with detailed verbosity level for logged diagnostics - bash: | @@ -346,6 +360,7 @@ jobs: inputs: selectByFolder: "simtests" modelCoverageCobertura: test-results/matlab/modelcoverage.xml + modelCoverageHTML: test-results/matlab/modelcoverageHTML testResultsSimulinkTest: test-results/matlab/stmResult.mldatx testResultsPDF: test-results/matlab/results.pdf condition: eq(variables['Agent.OS'], 'Linux') @@ -354,6 +369,12 @@ jobs: grep -q new_temp_model test-results/matlab/modelcoverage.xml displayName: Verify Model coverage was created condition: eq(variables['Agent.OS'], 'Linux') + - bash: | + set -e + test -d test-results/matlab/modelcoverageHTML + test -f test-results/matlab/modelcoverageHTML/*.html + displayName: Verify HTML Model coverage folder and HTML file was created + condition: not(eq(${{ version }}, '0')) - bash: | set -e test -f test-results/matlab/stmResult.mldatx diff --git a/overview.md b/overview.md index 44ea4f32..7ddeb020 100644 --- a/overview.md +++ b/overview.md @@ -233,11 +233,14 @@ Input | Description `useParallel` |

(Optional) Option to run tests in parallel, specified as `false` or `true`. By default, the value is `false` and tests run in serial. If the test runner configuration is suited for parallelization, you can specify a value of `true` to run tests in parallel. This input requires a Parallel Computing Toolbox license.

**Example:** `useParallel: true`

`outputDetail` |

(Optional) Amount of event detail displayed for the test run, specified as `none`, `terse`, `concise`, `detailed`, or `verbose`. By default, the task displays failing and logged events at the `detailed` level and test run progress at the `concise` level.

**Example:** `outputDetail: verbose`

`loggingLevel` |

(Optional) Maximum verbosity level for logged diagnostics included for the test run, specified as `none`, `terse`, `concise`, `detailed`, or `verbose`. By default, the task includes diagnostics logged at the `terse` level.

**Example:** `loggingLevel: detailed`

-`testResultsPDF` |

(Optional) Location to write the test results in PDF format, specified as a path relative to the project root folder. On macOS platforms, this input is supported in MATLAB R2020b and later.

**Example:** `testResultsPDF: test-results/results.pdf`

-`testResultsJUnit` |

(Optional) Location to write the test results in JUnit-style XML format, specified as a path relative to the project root folder.

**Example:** `testResultsJUnit: test-results/results.xml`

-`testResultsSimulinkTest` |

(Optional) Location to export Simulink Test Manager results in MLDATX format, specified as a path relative to the project root folder. This input requires a Simulink Test license and is supported in MATLAB R2019a and later.

**Example:** `testResultsSimulinkTest: test-results/results.mldatx`

-`codeCoverageCobertura` |

(Optional) Location to write the code coverage results in Cobertura XML format, specified as a path relative to the project root folder.

**Example:** `codeCoverageCobertura: code-coverage/coverage.xml`

-`modelCoverageCobertura` |

(Optional) Location to write the model coverage results in Cobertura XML format, specified as a path relative to the project root folder. This input requires a Simulink Coverage™ license and is supported in MATLAB R2018b and later.

**Example:** `modelCoverageCobertura: model-coverage/coverage.xml`

+`testResultsHTML` |

(Optional) Location to write the test results in HTML format, specified as a folder path relative to the project root folder.

**Example:** `testResultsHTML: test-results`

+`testResultsPDF` |

(Optional) Location to write the test results in PDF format, specified as a file path relative to the project root folder. On macOS platforms, this input is supported in MATLAB R2020b and later.

**Example:** `testResultsPDF: test-results/results.pdf`

+`testResultsJUnit` |

(Optional) Location to write the test results in JUnit-style XML format, specified as a file path relative to the project root folder.

**Example:** `testResultsJUnit: test-results/results.xml`

+`testResultsSimulinkTest` |

(Optional) Location to export Simulink Test Manager results in MLDATX format, specified as a file path relative to the project root folder. This input requires a Simulink Test license and is supported in MATLAB R2019a and later.

**Example:** `testResultsSimulinkTest: test-results/results.mldatx`

+`codeCoverageHTML` |

(Optional) Location to write the code coverage results in HTML format, specified as a folder path relative to the project root folder.

**Example:** `codeCoverageHTML: code-coverage`

+`codeCoverageCobertura` |

(Optional) Location to write the code coverage results in Cobertura XML format, specified as a file path relative to the project root folder.

**Example:** `codeCoverageCobertura: code-coverage/coverage.xml`

+`modelCoverageHTML` |

(Optional) Location to write the model coverage results in HTML format, specified as a folder path relative to the project root folder. This input requires a Simulink Coverage™ license and is supported in MATLAB R2018b and later.

**Example:** `modelCoverageHTML: model-coverage`

+`modelCoverageCobertura` |

(Optional) Location to write the model coverage results in Cobertura XML format, specified as a file path relative to the project root folder. This input requires a Simulink Coverage license and is supported in MATLAB R2018b and later.

**Example:** `modelCoverageCobertura: model-coverage/coverage.xml`

`startupOptions` |

(Optional) MATLAB startup options, specified as a list of options separated by spaces. For more information about startup options, see [Commonly Used Startup Options](https://www.mathworks.com/help/matlab/matlab_env/commonly-used-startup-options.html).

Using this input to specify the `-batch` or `-r` option is not supported.

**Example:** `startupOptions: -nojvm`
**Example:** `startupOptions: -nojvm -logfile output.log`

>**Note:** To customize the pretest state of the system, you can specify startup code that automatically executes before your tests run. For information on how to specify startup or shutdown files in a MATLAB project, see [Automate Startup and Shutdown Tasks](https://www.mathworks.com/help/matlab/matlab_prog/automate-startup-and-shutdown-tasks.html). If your pipeline does not use a MATLAB project, specify the commands you want executed at startup in a `startup.m` file instead, and save the file to the root of your repository. See [`startup`](https://www.mathworks.com/help/matlab/ref/startup.html) for more information. diff --git a/tasks/run-matlab-tests/v1/main.ts b/tasks/run-matlab-tests/v1/main.ts index ed0bdb0f..6e94c0eb 100644 --- a/tasks/run-matlab-tests/v1/main.ts +++ b/tasks/run-matlab-tests/v1/main.ts @@ -11,12 +11,15 @@ async function run() { const options: scriptgen.IRunTestsOptions = { JUnitTestResults: taskLib.getInput("testResultsJUnit"), CoberturaCodeCoverage: taskLib.getInput("codeCoverageCobertura"), + HTMLCodeCoverage: taskLib.getInput("codeCoverageHTML"), SourceFolder: taskLib.getInput("sourceFolder"), SelectByFolder: taskLib.getInput("selectByFolder"), SelectByTag: taskLib.getInput("selectByTag"), CoberturaModelCoverage: taskLib.getInput("modelCoverageCobertura"), + HTMLModelCoverage: taskLib.getInput("modelCoverageHTML"), SimulinkTestResults: taskLib.getInput("testResultsSimulinkTest"), PDFTestReport: taskLib.getInput("testResultsPDF"), + HTMLTestReport: taskLib.getInput("testResultsHTML"), Strict: taskLib.getBoolInput("strict"), UseParallel: taskLib.getBoolInput("useParallel"), OutputDetail: taskLib.getInput("outputDetail"), diff --git a/tasks/run-matlab-tests/v1/scriptgen.ts b/tasks/run-matlab-tests/v1/scriptgen.ts index 68d6e3c8..76249d94 100644 --- a/tasks/run-matlab-tests/v1/scriptgen.ts +++ b/tasks/run-matlab-tests/v1/scriptgen.ts @@ -5,10 +5,13 @@ import * as path from "path"; export interface IRunTestsOptions { JUnitTestResults?: string; CoberturaCodeCoverage?: string; + HTMLCodeCoverage?: string; SourceFolder?: string; PDFTestReport?: string; + HTMLTestReport?: string; SimulinkTestResults?: string; CoberturaModelCoverage?: string; + HTMLModelCoverage?: string; SelectByTag?: string; SelectByFolder?: string; Strict?: boolean; @@ -22,10 +25,13 @@ export function generateCommand(options: IRunTestsOptions): string { `testScript = genscript('Test',` + `'JUnitTestResults','${options.JUnitTestResults || ""}',` + `'CoberturaCodeCoverage','${options.CoberturaCodeCoverage || ""}',` + + `'HTMLCodeCoverage','${options.HTMLCodeCoverage || ""}',` + `'SourceFolder','${options.SourceFolder || ""}',` + `'PDFTestReport','${options.PDFTestReport || ""}',` + + `'HTMLTestReport','${options.HTMLTestReport || ""}',` + `'SimulinkTestResults','${options.SimulinkTestResults || ""}',` + `'CoberturaModelCoverage','${options.CoberturaModelCoverage || ""}',` + + `'HTMLModelCoverage','${options.HTMLModelCoverage || ""}',` + `'SelectByTag','${options.SelectByTag || ""}',` + `'SelectByFolder','${options.SelectByFolder || ""}',` + `'Strict',${options.Strict || false},` + diff --git a/tasks/run-matlab-tests/v1/task.json b/tasks/run-matlab-tests/v1/task.json index ef9cc098..f6316ca3 100644 --- a/tasks/run-matlab-tests/v1/task.json +++ b/tasks/run-matlab-tests/v1/task.json @@ -100,7 +100,16 @@ "defaultValue": "", "groupName": "testArtifacts", "required": false, - "helpMarkDown": "Path to write the test results in PDF format (requires MATLAB R2020b or later on macOS platforms)." + "helpMarkDown": "File path to write the test results in PDF format (requires MATLAB R2020b or later on macOS platforms)." + }, + { + "name": "testResultsHTML", + "type": "string", + "label": "HTML test report", + "defaultValue": "", + "groupName": "testArtifacts", + "required": false, + "helpMarkDown": "Folder path to write the test results in HTML format." }, { "name": "testResultsJUnit", @@ -109,7 +118,7 @@ "defaultValue": "", "groupName": "testArtifacts", "required": false, - "helpMarkDown": "Path to write the test results in JUnit-style XML format." + "helpMarkDown": "File path to write the test results in JUnit-style XML format." }, { "name": "testResultsSimulinkTest", @@ -118,7 +127,7 @@ "defaultValue": "", "groupName": "testArtifacts", "required": false, - "helpMarkDown": "Path to export Simulink Test Manager results in MLDATX format (requires Simulink Test and is supported in MATLAB R2019a or later)." + "helpMarkDown": "File path to export Simulink Test Manager results in MLDATX format (requires Simulink Test and is supported in MATLAB R2019a or later)." }, { "name": "codeCoverageCobertura", @@ -127,7 +136,16 @@ "defaultValue": "", "required": false, "groupName": "coverageArtifacts", - "helpMarkDown": "Path to write the code coverage results in Cobertura XML format." + "helpMarkDown": "File path to write the code coverage results in Cobertura XML format." + }, + { + "name": "codeCoverageHTML", + "type": "string", + "label": "HTML code coverage", + "defaultValue": "", + "required": false, + "groupName": "coverageArtifacts", + "helpMarkDown": "Folder path to write the code coverage results in HTML format." }, { "name": "modelCoverageCobertura", @@ -136,7 +154,16 @@ "defaultValue": "", "groupName": "coverageArtifacts", "required": false, - "helpMarkDown": "Path to write the model coverage report in Cobertura XML format (requires Simulink Coverage and is supported in MATLAB R2018b or later)." + "helpMarkDown": "File path to write the model coverage report in Cobertura XML format (requires Simulink Coverage and is supported in MATLAB R2018b or later)." + }, + { + "name": "modelCoverageHTML", + "type": "string", + "label": "HTML model coverage", + "defaultValue": "", + "groupName": "coverageArtifacts", + "required": false, + "helpMarkDown": "Folder path to write the model coverage results in HTML format (requires Simulink Coverage and is supported in MATLAB R2018b or later)." }, { "name": "startupOptions", diff --git a/tasks/run-matlab-tests/v1/test/scriptgen.test.ts b/tasks/run-matlab-tests/v1/test/scriptgen.test.ts index d89e4921..1323108b 100644 --- a/tasks/run-matlab-tests/v1/test/scriptgen.test.ts +++ b/tasks/run-matlab-tests/v1/test/scriptgen.test.ts @@ -9,10 +9,13 @@ export default function suite() { const options: scriptgen.IRunTestsOptions = { JUnitTestResults: "", CoberturaCodeCoverage: "", + HTMLCodeCoverage: "", SourceFolder: "", PDFTestReport: "", + HTMLTestReport: "", SimulinkTestResults: "", CoberturaModelCoverage: "", + HTMLModelCoverage: "", SelectByTag: "", SelectByFolder: "", Strict: false, @@ -26,10 +29,13 @@ export default function suite() { assert(actual.includes("genscript('Test'")); assert(actual.includes("'JUnitTestResults',''")); assert(actual.includes("'CoberturaCodeCoverage',''")); + assert(actual.includes("'HTMLCodeCoverage',''")); assert(actual.includes("'SourceFolder',''")); assert(actual.includes("'PDFTestReport',''")); + assert(actual.includes("'HTMLTestReport',''")); assert(actual.includes("'SimulinkTestResults',''")); assert(actual.includes("'CoberturaModelCoverage',''")); + assert(actual.includes("'HTMLModelCoverage',''")); assert(actual.includes("'SelectByTag',''")); assert(actual.includes("'SelectByFolder',''")); assert(actual.includes("'Strict',false")); @@ -37,9 +43,9 @@ export default function suite() { assert(actual.includes("'OutputDetail',''")); assert(actual.includes("'LoggingLevel',''")); - const expected = `genscript('Test', 'JUnitTestResults','', 'CoberturaCodeCoverage','', - 'SourceFolder','', 'PDFTestReport','', 'SimulinkTestResults','', - 'CoberturaModelCoverage','', 'SelectByTag','', 'SelectByFolder','', + const expected = `genscript('Test', 'JUnitTestResults','', 'CoberturaCodeCoverage','','HTMLCodeCoverage','', + 'SourceFolder','', 'PDFTestReport','', 'HTMLTestReport','', 'SimulinkTestResults','', + 'CoberturaModelCoverage','', 'HTMLModelCoverage','', 'SelectByTag','', 'SelectByFolder','', 'Strict',false, 'UseParallel',false, 'OutputDetail','', 'LoggingLevel','')` .replace(/\s+/g, ""); assert(actual.replace(/\s+/g, "").includes(expected)); @@ -49,10 +55,13 @@ export default function suite() { const options: scriptgen.IRunTestsOptions = { JUnitTestResults: "test-results/results.xml", CoberturaCodeCoverage: "code-coverage/coverage.xml", + HTMLCodeCoverage: "code-coverage/coverageHTML", SourceFolder: "source", PDFTestReport: "test-results/pdf-results.pdf", + HTMLTestReport: "test-results/html-resultsHTML", SimulinkTestResults: "test-results/simulinkTest.mldatx", CoberturaModelCoverage: "test-results/modelcoverage.xml", + HTMLModelCoverage: "test-results/modelcoverageHTML", SelectByTag: "FeatureA", SelectByFolder: "test/tools;test/toolbox", Strict: true, @@ -66,10 +75,13 @@ export default function suite() { assert(actual.includes("genscript('Test'")); assert(actual.includes("'JUnitTestResults','test-results/results.xml'")); assert(actual.includes("'CoberturaCodeCoverage','code-coverage/coverage.xml'")); + assert(actual.includes("'HTMLCodeCoverage','code-coverage/coverageHTML'")); assert(actual.includes("'SourceFolder','source'")); assert(actual.includes("'PDFTestReport','test-results/pdf-results.pdf'")); + assert(actual.includes("'HTMLTestReport','test-results/html-resultsHTML'")); assert(actual.includes("'SimulinkTestResults','test-results/simulinkTest.mldatx'")); assert(actual.includes("'CoberturaModelCoverage','test-results/modelcoverage.xml'")); + assert(actual.includes("'HTMLModelCoverage','test-results/modelcoverageHTML'")); assert(actual.includes("'SelectByTag','FeatureA'")); assert(actual.includes("'SelectByFolder','test/tools;test/toolbox'")); assert(actual.includes("'Strict',true")); @@ -78,9 +90,9 @@ export default function suite() { assert(actual.includes("'LoggingLevel','Detailed'")); const expected = `genscript('Test', 'JUnitTestResults','test-results/results.xml', - 'CoberturaCodeCoverage','code-coverage/coverage.xml', 'SourceFolder','source', - 'PDFTestReport','test-results/pdf-results.pdf', 'SimulinkTestResults','test-results/simulinkTest.mldatx', - 'CoberturaModelCoverage','test-results/modelcoverage.xml', 'SelectByTag','FeatureA', + 'CoberturaCodeCoverage','code-coverage/coverage.xml','HTMLCodeCoverage','code-coverage/coverageHTML', 'SourceFolder','source', + 'PDFTestReport','test-results/pdf-results.pdf', 'HTMLTestReport','test-results/html-resultsHTML', 'SimulinkTestResults','test-results/simulinkTest.mldatx', + 'CoberturaModelCoverage','test-results/modelcoverage.xml', 'HTMLModelCoverage','test-results/modelcoverageHTML', 'SelectByTag','FeatureA', 'SelectByFolder','test/tools;test/toolbox', 'Strict',true, 'UseParallel',true, 'OutputDetail','Detailed', 'LoggingLevel','Detailed' )` .replace(/\s+/g, "");