Test Result Reporting Tool (#15)
Adding a new feature to report on CircleCI test runs.
From the root folder of the REPO, run the following command...
npm run viewTestResults [build-number]
This commit is contained in:
Michael Skowronski
GitHub
Родитель
e9fb9ea824
Коммит
12a305f2dd
|
|
@ -7,6 +7,7 @@
|
|||
"typescript": "3.7.2"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "lerna run build"
|
||||
"build": "lerna run build",
|
||||
"viewTestResults": "node packages\\ui\\src\\testResultReporting\\TestResultAnalyzer.js"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -7737,11 +7737,13 @@
|
|||
},
|
||||
"balanced-match": {
|
||||
"version": "1.0.0",
|
||||
"bundled": true
|
||||
"bundled": true,
|
||||
"optional": true
|
||||
},
|
||||
"brace-expansion": {
|
||||
"version": "1.1.11",
|
||||
"bundled": true,
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"balanced-match": "^1.0.0",
|
||||
"concat-map": "0.0.1"
|
||||
|
|
@ -7754,15 +7756,18 @@
|
|||
},
|
||||
"code-point-at": {
|
||||
"version": "1.1.0",
|
||||
"bundled": true
|
||||
"bundled": true,
|
||||
"optional": true
|
||||
},
|
||||
"concat-map": {
|
||||
"version": "0.0.1",
|
||||
"bundled": true
|
||||
"bundled": true,
|
||||
"optional": true
|
||||
},
|
||||
"console-control-strings": {
|
||||
"version": "1.1.0",
|
||||
"bundled": true
|
||||
"bundled": true,
|
||||
"optional": true
|
||||
},
|
||||
"core-util-is": {
|
||||
"version": "1.0.2",
|
||||
|
|
@ -7865,7 +7870,8 @@
|
|||
},
|
||||
"inherits": {
|
||||
"version": "2.0.4",
|
||||
"bundled": true
|
||||
"bundled": true,
|
||||
"optional": true
|
||||
},
|
||||
"ini": {
|
||||
"version": "1.3.5",
|
||||
|
|
@ -7875,6 +7881,7 @@
|
|||
"is-fullwidth-code-point": {
|
||||
"version": "1.0.0",
|
||||
"bundled": true,
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"number-is-nan": "^1.0.0"
|
||||
}
|
||||
|
|
@ -7887,17 +7894,20 @@
|
|||
"minimatch": {
|
||||
"version": "3.0.4",
|
||||
"bundled": true,
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
}
|
||||
},
|
||||
"minimist": {
|
||||
"version": "0.0.8",
|
||||
"bundled": true
|
||||
"bundled": true,
|
||||
"optional": true
|
||||
},
|
||||
"minipass": {
|
||||
"version": "2.9.0",
|
||||
"bundled": true,
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"safe-buffer": "^5.1.2",
|
||||
"yallist": "^3.0.0"
|
||||
|
|
@ -7914,6 +7924,7 @@
|
|||
"mkdirp": {
|
||||
"version": "0.5.1",
|
||||
"bundled": true,
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"minimist": "0.0.8"
|
||||
}
|
||||
|
|
@ -7994,7 +8005,8 @@
|
|||
},
|
||||
"number-is-nan": {
|
||||
"version": "1.0.1",
|
||||
"bundled": true
|
||||
"bundled": true,
|
||||
"optional": true
|
||||
},
|
||||
"object-assign": {
|
||||
"version": "4.1.1",
|
||||
|
|
@ -8004,6 +8016,7 @@
|
|||
"once": {
|
||||
"version": "1.4.0",
|
||||
"bundled": true,
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"wrappy": "1"
|
||||
}
|
||||
|
|
@ -8109,6 +8122,7 @@
|
|||
"string-width": {
|
||||
"version": "1.0.2",
|
||||
"bundled": true,
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"code-point-at": "^1.0.0",
|
||||
"is-fullwidth-code-point": "^1.0.0",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,26 @@
|
|||
const axios = require('axios')
|
||||
|
||||
exports.Get = Get
|
||||
|
||||
async function Get(url) {
|
||||
return await axios.get(url).then(response => {
|
||||
return response.data
|
||||
})
|
||||
.catch(err => {
|
||||
console.log(`GetApiData - Error accessing: ${url}`)
|
||||
console.log(err.message)
|
||||
})
|
||||
}
|
||||
|
||||
// UNIT TESTS - These are triggered ONLY when running this as a standalone module.
|
||||
if (require.main === module) {
|
||||
(async function() {
|
||||
let buildNumber = process.argv[2]
|
||||
if (!buildNumber) {
|
||||
buildNumber = 5501
|
||||
}
|
||||
|
||||
const artifacts = await Get(`https://circleci.com/api/v1.1/project/github/microsoft/ConversationLearner-UI/${buildNumber}/artifacts?circle-token=2ad1e457047948114cb3bbb1957d6f90c1e2ee25`)
|
||||
console.log(artifacts)
|
||||
}())
|
||||
}
|
||||
|
|
@ -0,0 +1,373 @@
|
|||
const ttf = require('./TriageTestFailure')
|
||||
const apiData = require('./ApiData')
|
||||
const fs = require('fs')
|
||||
const child_process = require('child_process')
|
||||
|
||||
const triageData = require('./TriageData').triageData
|
||||
ttf.SetTriageData(triageData);
|
||||
|
||||
(async function () {
|
||||
|
||||
let buildNumber
|
||||
let logs = []
|
||||
let mp4s = []
|
||||
let pngs = []
|
||||
let unknownTestFailures = []
|
||||
let knownTestFailures = []
|
||||
let passingTests = []
|
||||
let errors = []
|
||||
let htmlContent = ''
|
||||
|
||||
let iStart
|
||||
let iEnd
|
||||
let iSpec
|
||||
let testName
|
||||
let key
|
||||
|
||||
buildNumber = process.argv[2]
|
||||
|
||||
const artifacts = await apiData.Get(`https://circleci.com/api/v1.1/project/github/microsoft/conversationLearner/${buildNumber}/artifacts?circle-token=2ad1e457047948114cb3bbb1957d6f90c1e2ee25`)
|
||||
MoveArtifactJsonIntoArrays()
|
||||
console.log('Processing the Failed Test Results ----------------------------------')
|
||||
await ProcessFailingTestArtifacts()
|
||||
console.log('Processing the Passed Test Results ----------------------------------')
|
||||
ProcessPassingTestArtifacts()
|
||||
|
||||
console.log('Rendering all Results to HTML ----------------------------------')
|
||||
RenderResults()
|
||||
|
||||
const outputPath = `${process.env.TEMP}\\TestResults.${buildNumber}.html`
|
||||
console.log(`Writing HTML to File ${outputPath} ----------------------------------`)
|
||||
fs.writeFileSync(outputPath, htmlContent)
|
||||
|
||||
child_process.exec(outputPath)
|
||||
|
||||
function MoveArtifactJsonIntoArrays() {
|
||||
artifacts.forEach(artifact => {
|
||||
const suffix = artifact.url.substring(artifact.url.length - 3)
|
||||
switch (suffix) {
|
||||
case 'log':
|
||||
iStart = artifact.url.indexOf('/cypress/') + '/cypress/'.length
|
||||
iEnd = artifact.url.length - '.19.12.05.02.42.14..456.log'.length
|
||||
key = artifact.url.substring(iStart, iEnd)
|
||||
console.log('key:', key)
|
||||
console.log('url:', artifact.url)
|
||||
logs.push({ key: key, url: artifact.url })
|
||||
break
|
||||
|
||||
case 'mp4':
|
||||
iStart = artifact.url.indexOf('/videos/') + '/videos/'.length
|
||||
iEnd = artifact.url.length - 4
|
||||
testName = artifact.url.substring(iStart, iEnd)
|
||||
key = testName.replace(/\/|\\/g, '-')
|
||||
console.log('testName:', testName)
|
||||
console.log('key:', key)
|
||||
console.log('url:', artifact.url)
|
||||
mp4s.push({ testName: testName, key: key, url: artifact.url })
|
||||
break
|
||||
|
||||
case 'png':
|
||||
iStart = artifact.url.indexOf('/screenshots/') + '/screenshots/'.length
|
||||
iSpec = artifact.url.indexOf('.spec.', iStart)
|
||||
iEnd = artifact.url.indexOf('/', iSpec)
|
||||
testName = artifact.url.substring(iStart, iEnd)
|
||||
key = testName.replace(/\/|\\/g, '-')
|
||||
console.log('testName:', testName)
|
||||
console.log('key:', key)
|
||||
console.log('url:', artifact.url)
|
||||
pngs.push({ testName: testName, key: key, url: artifact.url })
|
||||
break
|
||||
|
||||
default:
|
||||
console.log('!!!*** What file is this? ***!!!', artifact.url)
|
||||
break
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async function ProcessFailingTestArtifacts() {
|
||||
for (let i = 0; i < pngs.length; i++) {
|
||||
const png = pngs[i]
|
||||
let error
|
||||
let failureDetails
|
||||
|
||||
//if (png.testName.endsWith('.ts')) continue
|
||||
|
||||
console.log(`*** PNG: ${png.testName} *****************************`)
|
||||
|
||||
const log = logs.find(log => log.key === png.key)
|
||||
if (!log) {
|
||||
console.log(`ProcessFailingTestArtifacts - going to return since log is undefined`)
|
||||
errors.push({ testName: png.testName, key: png.key, url: 'page error: Log file not found' })
|
||||
continue
|
||||
}
|
||||
|
||||
console.log(`ProcessFailingTestArtifacts - going to await GetTriageDetailsAboutTestFailure`)
|
||||
failureDetails = await ttf.GetTriageDetailsAboutTestFailure(log)
|
||||
if (typeof failureDetails == 'string') {
|
||||
console.log(`ProcessFailingTestArtifacts got failureDetails: ${failureDetails}`)
|
||||
} else {
|
||||
console.log(`ProcessFailingTestArtifacts got failureDetails: { message: ${failureDetails.message}, bugs: ${failureDetails.bugs}, comment: ${failureDetails.comment} }`)
|
||||
}
|
||||
|
||||
const mp4 = mp4s.find(mp4 => mp4.key === png.key)
|
||||
if (!mp4) {
|
||||
console.log('ProcessFailingTestArtifacts - ERROR: Did not find matching mp4')
|
||||
errors.push({ testName: png.testName, key: png.key, url: 'page error: mp4 file not found' })
|
||||
continue
|
||||
}
|
||||
|
||||
let testFailure = {
|
||||
testName: png.testName,
|
||||
key: png.key,
|
||||
snapshotUrl: png.url,
|
||||
videoUrl: mp4.url,
|
||||
logUrl: log.url,
|
||||
knownIssue: failureDetails.knownIssue,
|
||||
failureMessage: failureDetails.message,
|
||||
bugs: failureDetails.bugs,
|
||||
comment: failureDetails.comment,
|
||||
errorPanelText: failureDetails.errorPanelText,
|
||||
}
|
||||
|
||||
if (testFailure.knownIssue) {
|
||||
knownTestFailures.push(testFailure)
|
||||
} else {
|
||||
unknownTestFailures.push(testFailure)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function ProcessPassingTestArtifacts() {
|
||||
logs.forEach(log => {
|
||||
if (unknownTestFailures.findIndex(failure => failure.key === log.key) >= 0) return
|
||||
if (knownTestFailures.findIndex(failure => failure.key === log.key) >= 0) return
|
||||
|
||||
const mp4 = mp4s.find(mp4 => mp4.key === log.key)
|
||||
if (!mp4) {
|
||||
errors.push({ testName: log.key, key: log.key, url: 'page error: mp4 file not found' })
|
||||
return
|
||||
}
|
||||
|
||||
passingTests.push({
|
||||
testName: mp4.testName,
|
||||
videoUrl: mp4.url,
|
||||
logUrl: log.url,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function OLD_RenderResults() {
|
||||
console.log(`${unknownTestFailures.length} UNKNOWN TEST FAILURES -------------------------------------------------------`)
|
||||
unknownTestFailures.forEach(unknownTestFailure => {
|
||||
console.log(`unknownTestFailures.push({
|
||||
testName: '${unknownTestFailure.testName}',
|
||||
key: ${unknownTestFailure.key},
|
||||
snapshotUrl: '${unknownTestFailure.snapshotUrl}',
|
||||
videoUrl: '${unknownTestFailure.videoUrl}',
|
||||
logUrl: '${unknownTestFailure.logUrl}',
|
||||
failureMessage: '${unknownTestFailure.failureMessage}',
|
||||
})`)
|
||||
})
|
||||
|
||||
console.log(`${knownTestFailures.length} KNOWN TEST FAILURES ---------------------------------------------------------`)
|
||||
knownTestFailures.forEach(knownTestFailure => {
|
||||
console.log(`knownTestFailures.push({
|
||||
testName: '${knownTestFailure.testName}',
|
||||
key: ${knownTestFailure.key},
|
||||
snapshotUrl: '${knownTestFailure.snapshotUrl}',
|
||||
videoUrl: '${knownTestFailure.videoUrl}',
|
||||
logUrl: '${knownTestFailure.logUrl}',
|
||||
failureMessage: '${knownTestFailure.failureMessage}',
|
||||
bugs: '${knownTestFailure.bugs}',
|
||||
})`)
|
||||
})
|
||||
|
||||
console.log(`${passingTests.length} PASSING TESTS ---------------------------------------------------------------`)
|
||||
passingTests.forEach(passingTest => {
|
||||
console.log(`passingTests.push({
|
||||
testName: '${passingTest.testName}',
|
||||
videoUrl: '${passingTest.videoUrl}',
|
||||
logUrl: '${passingTest.logUrl}',
|
||||
})`)
|
||||
})
|
||||
|
||||
console.log(`${errors.length} ERRORS ---------------------------------------------------------------`)
|
||||
errors.forEach(error => {
|
||||
console.log(`errors.push({
|
||||
testName: '${error.testName}',
|
||||
key: '${error.key}',
|
||||
url: '${error.url}',
|
||||
})`)
|
||||
})
|
||||
}
|
||||
|
||||
function ReplaceSpecialHtmlCharacters(text) {
|
||||
return text.replace(/&/g, "&").replace(/>/g, ">").replace(/</g, "<")
|
||||
}
|
||||
|
||||
function RenderResults() {
|
||||
htmlContent = `
|
||||
<html>
|
||||
<head>
|
||||
<style>
|
||||
.grid-container1 {
|
||||
display: grid;
|
||||
grid-template-columns: auto;
|
||||
background-color: #2196F3;
|
||||
padding: 5px;
|
||||
}
|
||||
|
||||
.grid-container2 {
|
||||
display: grid;
|
||||
grid-template-columns: auto auto;
|
||||
background-color: #2196F3;
|
||||
padding: 5px;
|
||||
}
|
||||
|
||||
.grid-item {
|
||||
background-color: rgba(255, 255, 255, 0.8);
|
||||
border: 1px solid rgba(0, 0, 0, 0.8);
|
||||
padding: 5px;
|
||||
font-size: 15px;
|
||||
text-align: left;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<h1>Test Results for Build Number <a href='https://circleci.com/gh/microsoft/ConversationLearner-UI/${buildNumber}#artifacts/containers/0' target='_blank'>${buildNumber}</a></h1>
|
||||
Triage Data contains signatures for ${triageData.length} known issues.
|
||||
<div>
|
||||
`
|
||||
RenderUnknownFailures()
|
||||
htmlContent += `</div>
|
||||
<div>
|
||||
`
|
||||
RenderKnownFailures()
|
||||
htmlContent += `</div>
|
||||
<div>
|
||||
`
|
||||
|
||||
RenderPassingTests()
|
||||
htmlContent += `</div>
|
||||
</body>
|
||||
</html>`
|
||||
}
|
||||
|
||||
function RenderUnknownFailures() {
|
||||
console.log(`${unknownTestFailures.length} UNKNOWN TEST FAILURES -------------------------------------------------------`)
|
||||
|
||||
if (!unknownTestFailures || unknownTestFailures.length == 0) {
|
||||
htmlContent += `<h2>No Unknown Failures</h2>`
|
||||
} else {
|
||||
htmlContent += `
|
||||
<h2>${unknownTestFailures.length} Failures with an Unknown Cause</h2>
|
||||
<div class="grid-container2">
|
||||
`
|
||||
unknownTestFailures.forEach(unknownTestFailure => {
|
||||
htmlContent += `
|
||||
<div class="grid-item">
|
||||
<b>${unknownTestFailure.testName}</b><br>
|
||||
<a href='${unknownTestFailure.snapshotUrl}' target='_blank'>
|
||||
Snapshot
|
||||
</a> --
|
||||
<a href='${unknownTestFailure.videoUrl}' target='_blank'>Video</a>
|
||||
</div>
|
||||
|
||||
<div class="grid-item">
|
||||
`
|
||||
|
||||
if (unknownTestFailure.errorPanelText) {
|
||||
htmlContent += `<b>UI Error Panel:</b> ${unknownTestFailure.errorPanelText}<br>`
|
||||
}
|
||||
|
||||
htmlContent += `
|
||||
<b>Failure Message:</b> ${ReplaceSpecialHtmlCharacters(unknownTestFailure.failureMessage)}<br>
|
||||
<a href='${unknownTestFailure.logUrl}' target='_blank'>
|
||||
Log File
|
||||
</a><br>
|
||||
</div>
|
||||
`
|
||||
})
|
||||
htmlContent + '</div>'
|
||||
}
|
||||
}
|
||||
|
||||
function RenderKnownFailures() {
|
||||
console.log(`${knownTestFailures.length} KNOWN TEST FAILURES ---------------------------------------------------------`)
|
||||
|
||||
if (!knownTestFailures || knownTestFailures.length == 0) {
|
||||
htmlContent += `<h2>No Known Failures</h2>`
|
||||
} else {
|
||||
htmlContent += `
|
||||
<h2>${knownTestFailures.length} Failures with a Known Cause</h2>
|
||||
<div class="grid-container2">
|
||||
`
|
||||
knownTestFailures.forEach(knownTestFailure => {
|
||||
htmlContent += `
|
||||
<div class="grid-item">
|
||||
<b>${knownTestFailure.testName}</b><br>
|
||||
<a href='${knownTestFailure.snapshotUrl}' target='_blank'>
|
||||
Snapshot
|
||||
</a> --
|
||||
<a href='${knownTestFailure.videoUrl}' target='_blank'>Video</a>
|
||||
</div>
|
||||
|
||||
<div class="grid-item">
|
||||
`
|
||||
if (knownTestFailure.comment) {
|
||||
htmlContent += `<b>${knownTestFailure.comment}</b><br>`
|
||||
}
|
||||
|
||||
if (knownTestFailure.errorPanelText) {
|
||||
htmlContent += `<b>UI Error Panel:</b> ${knownTestFailure.errorPanelText}<br>`
|
||||
}
|
||||
|
||||
htmlContent += `
|
||||
<b>Failure Message:</b> ${ReplaceSpecialHtmlCharacters(knownTestFailure.failureMessage)}<br>
|
||||
<a href='${knownTestFailure.logUrl}' target='_blank'>
|
||||
Log File
|
||||
</a><br>
|
||||
`
|
||||
|
||||
if (knownTestFailure.bugs) {
|
||||
knownTestFailure.bugs.forEach(bugNumber => {
|
||||
htmlContent += `
|
||||
<a href='https://dialearn.visualstudio.com/BLIS/_workitems/edit/${bugNumber}' target='_blank'>Bug ${bugNumber}</a> TODO: Bug Title to go here.<br>
|
||||
`
|
||||
})
|
||||
}
|
||||
htmlContent += `</div>`
|
||||
})
|
||||
htmlContent += '</div>'
|
||||
}
|
||||
}
|
||||
|
||||
function RenderPassingTests() {
|
||||
console.log(`${passingTests.length} PASSING TESTS ---------------------------------------------------------------`)
|
||||
|
||||
if (!passingTests || passingTests.length == 0) {
|
||||
htmlContent += `<h2>No Passing Tests</h2>`
|
||||
} else {
|
||||
htmlContent += `
|
||||
<h2>${passingTests.length} Passing Tests</h2>
|
||||
<div class="grid-container1">
|
||||
`
|
||||
passingTests.forEach(passingTest => {
|
||||
htmlContent += `
|
||||
<div class="grid-item">
|
||||
<b>${passingTest.testName}</b><br>
|
||||
<a href='${passingTest.videoUrl}' target='_blank'>
|
||||
Video
|
||||
</a> --
|
||||
<a href='${passingTest.logUrl}' target='_blank'>
|
||||
Log File
|
||||
</a><br>
|
||||
</div>
|
||||
`
|
||||
})
|
||||
htmlContent += `</div>`
|
||||
}
|
||||
}
|
||||
}())
|
||||
|
|
@ -0,0 +1,151 @@
|
|||
const FAILURE_MESSAGE = 'fm'
|
||||
const ERROR_PANEL = 'ep'
|
||||
const FULL_LOG = 'fl'
|
||||
|
||||
exports.triageData = [
|
||||
{
|
||||
or: [
|
||||
`Timed out retrying: Stauts is queued - Still Waiting for Status == Running or Completed - Queued Wait Time:`,
|
||||
`Timed out retrying: Status is queued - Still Waiting for Status == Running or Completed - Queued Wait Time:`,
|
||||
],
|
||||
bugs: [2415]
|
||||
},
|
||||
{
|
||||
and: [`Timed out retrying: Expected to find element: 'button.ms-Dropdown-item[title="Enum"]', but never found it.`],
|
||||
bugs: [2409],
|
||||
},
|
||||
{
|
||||
and: [`Bugs 2389 & 2400 - Entity Detection panel shows a phrase that is different than the user's utterance.`],
|
||||
bugs: [2389, 2400]
|
||||
},
|
||||
{
|
||||
and: [
|
||||
"Timed out retrying: Expected to find content: 'z-rename-",
|
||||
"within the element: <div.css-69> but never did.",
|
||||
],
|
||||
bugs: [2407],
|
||||
},
|
||||
{
|
||||
or: [`Expected to find content: 'world peace' within the element: [ <span.`],
|
||||
bugs: [2416],
|
||||
},
|
||||
{
|
||||
or: [
|
||||
`Expected to find element: '[data-testid="fuse-match-option"]', but never found it. Queried from element: <div.editor-container.`,
|
||||
`Response: What's your name? - Expected to find data-testid="action-scorer-button-clickable" instead we found "action-scorer-button-no-click"`
|
||||
],
|
||||
bugs: [2396],
|
||||
},
|
||||
{
|
||||
testName: 'Settings.spec.js',
|
||||
and: [`RETRY - Loaded Model List Row Count does not yet match the aria-rowcount.`],
|
||||
bugs: [2418],
|
||||
},
|
||||
{
|
||||
testName: 'MissingAction.spec.js',
|
||||
and: [`Timed out retrying: Expected to find element: '[data-testid="action-scorer-button-clickable"]', but never found it. Queried from element: <div.ms-DetailsRow-fields.`],
|
||||
bugs: [2419],
|
||||
},
|
||||
{
|
||||
testName: 'Bug2379Repro.spec.js',
|
||||
and: [`Timed out retrying: Expected to find element: 'span[role="button"]', but never found it. Queried from element: <span.ms-TagItem-text.>`],
|
||||
comment: 'TEST BUG - Its been fixed, should see this working soon.',
|
||||
},
|
||||
{
|
||||
and: [`Chat turn 2 should be an exact match to: The user asks a silly question, however, we found The user asks another question instead`],
|
||||
comment: 'UI BUG is fixed - Test code modified to pass, should see this working soon.',
|
||||
bugs: [2265]
|
||||
},
|
||||
{
|
||||
testName: 'Regression-Log',
|
||||
and: [
|
||||
`Timed out retrying: Expected to find content:`,
|
||||
`within the element: <div.wc-message-content> but never did.`,
|
||||
],
|
||||
bugs: [2197]
|
||||
},
|
||||
{
|
||||
and: [
|
||||
`cy.visit() failed trying to load:`,
|
||||
`http://localhost:3000/`
|
||||
],
|
||||
comment: 'This happens from time to time and there is no known fix for it.',
|
||||
},
|
||||
{
|
||||
searchBy: ERROR_PANEL,
|
||||
and: [`Creating Application Failed Request failed with status code 400 "Bad Request {"Locale":["The Locale field is required."]}`],
|
||||
bugs: [2408],
|
||||
},
|
||||
{
|
||||
searchBy: ERROR_PANEL,
|
||||
and: [`Running extractor Failed Request failed with status code 502 "Bad Gateway "LUIS error:`],
|
||||
comment: 'Network Issues - It should eventually fix itself.',
|
||||
},
|
||||
{
|
||||
searchBy: FULL_LOG,
|
||||
and: [
|
||||
`Failure Message: You attempted to select the phrase:`,
|
||||
`, but it was not found in the input:`,
|
||||
],
|
||||
bugs: [2422],
|
||||
comment: 'Bug in Test_SelectWord',
|
||||
},
|
||||
{
|
||||
testName: 'Regression-BugRepro-Bug2319Repro.spec.js',
|
||||
and: [`Expected to find 'X' in the text chat pane, instead we found 'ERROR: Missing ConversationReference' at index: 4`],
|
||||
bugs: [2319],
|
||||
},
|
||||
{
|
||||
testName: 'Regression-Log-Abandon.spec.js',
|
||||
// Using OR since there will probably be other variations on the failure message associated with this bug
|
||||
or: [`Timed out retrying: 3 rows found in the training grid, however we were expecting 2`],
|
||||
bugs: [2212],
|
||||
},
|
||||
{
|
||||
testName: 'Regression-BugRepro-Bug2119Repro.spec.js',
|
||||
searchBy: FULL_LOG,
|
||||
// Using OR since there will probably be other variations on the failure message associated with this bug
|
||||
or: [`TextContentWithoutNewlines - Raw Text Content: "User utterance **#**6"`],
|
||||
bugs: [2423],
|
||||
comment: 'PLEASE UPDATE BUG REPORT with a count on any instances of this bug that you find.',
|
||||
},
|
||||
{
|
||||
testName: 'Regression-Train-ApiMemoryManipulation.spec.js',
|
||||
and: [
|
||||
`Expected to find element: '[data-testid="action-scorer-button-clickable"]', but never found it. Queried from element: <div.ms-DetailsRow-fields`,
|
||||
{
|
||||
searchBy: FULL_LOG,
|
||||
and: [`FAILED - Test Case: 'API Memory Manipulation - Train - Train - Should remove one of the toppings and verify the resulting list is correct'`],
|
||||
}
|
||||
],
|
||||
bugs: [2416],
|
||||
comment: 'Another instance of that LUIS bug that mislabels Entities.',
|
||||
}
|
||||
]
|
||||
|
||||
// NOT USED BY THE CODE - AN EXAMPLE ONLY
|
||||
exampleTriageData = [
|
||||
{ // This is a complex example that shows how to search different strings found in the test artifacts.
|
||||
//
|
||||
and: [
|
||||
`Expected to find element:`, // must be in the Failure Message
|
||||
`but never found it.`, // and this must also be in the Failure Message
|
||||
{ // and this entire block must result in TRUE
|
||||
searchBy: FULL_LOG, // Search will now occur in the FULL_LOG
|
||||
or: [
|
||||
`This will NOT be found`, // Since we are now in an OR this one can be FALSE if the other is TRUE
|
||||
{ // Starting a new block that is ORed with the prior test
|
||||
and : [
|
||||
`Should import a model to test against and navigate to Train Dialogs view`, // this should also be found in the FULL_LOG
|
||||
{ // Starting a new block that is ANDed with the prior test
|
||||
searchBy: ERROR_PANEL,// Search will now occur in the ERROR_PANEL
|
||||
and: [`Creating Application Failed Request failed with status code 400 "Bad Request {"Locale":["The Locale field is required."]}`],
|
||||
},
|
||||
]
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
comment: 'This is a complex query example',
|
||||
},
|
||||
]
|
||||
|
|
@ -0,0 +1,252 @@
|
|||
const apiData = require('./ApiData')
|
||||
const cheerio = require('cheerio')
|
||||
|
||||
const FAILURE_MESSAGE = 'fm'
|
||||
const ERROR_PANEL = 'ep'
|
||||
const FULL_LOG = 'fl'
|
||||
|
||||
var triageData = undefined
|
||||
|
||||
exports.SetTriageData = SetTriageData
|
||||
exports.GetTriageDetailsAboutTestFailure = GetTriageDetailsAboutTestFailure
|
||||
|
||||
function SetTriageData(data) {
|
||||
triageData = data
|
||||
}
|
||||
|
||||
async function GetTriageDetailsAboutTestFailure(log) {
|
||||
if (!triageData) {
|
||||
throw new Error('You must first call "SetTriageData()" before calling GetTriageDetailsAboutTestFailure')
|
||||
}
|
||||
|
||||
let failureMessage
|
||||
let errorPanelText
|
||||
let fullLogText
|
||||
|
||||
// Relevant items in queryObj are:
|
||||
// searchBy: 'string' - must be one of: FAILURE_MESSAGE, ERROR_PANEL, FULL_LOG
|
||||
// and: 'array' - must contain either strings used in the search or another queryObj
|
||||
// or: 'array' - must contain either strings used in the search or another queryObj
|
||||
// It should contain only an 'and' array or an 'or' array, but not both.
|
||||
function query(queryObj, defaultSearchBy = FAILURE_MESSAGE) {
|
||||
// Wrapping the actual call so we can log the returned value with greater ease.
|
||||
let returnValue = _query()
|
||||
console.log(`query returned:`, returnValue, '\n')
|
||||
return returnValue
|
||||
|
||||
function _query() {
|
||||
// Determine which string will be searched...
|
||||
let searchBy = defaultSearchBy
|
||||
if (queryObj.searchBy) {
|
||||
searchBy = queryObj.searchBy
|
||||
}
|
||||
let sourceText
|
||||
switch(searchBy) {
|
||||
case FAILURE_MESSAGE:
|
||||
sourceText = failureMessage
|
||||
break;
|
||||
case ERROR_PANEL:
|
||||
sourceText = errorPanelText
|
||||
break;
|
||||
case FULL_LOG:
|
||||
sourceText = fullLogText
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unexpected searchBy value: '${searchBy}'`)
|
||||
}
|
||||
|
||||
if (!sourceText) {
|
||||
console.log(`sourceText for searchBy '${searchBy}' is undefined`)
|
||||
return false
|
||||
}
|
||||
//console.log(`sourceText: >>>${sourceText.substring(0, 140)}<<<`)
|
||||
|
||||
// Determine whether multiple conditions will be ANDed or ORed together to form the result...
|
||||
let and
|
||||
let queryArray
|
||||
if(queryObj.and) {
|
||||
and = true
|
||||
queryArray = queryObj.and
|
||||
} else if(queryObj.or) {
|
||||
and = false
|
||||
queryArray = queryObj.or
|
||||
}
|
||||
|
||||
console.log(`query`, searchBy, and ? 'AND' : 'OR', queryArray)
|
||||
|
||||
// Do the actual search here...
|
||||
let result
|
||||
for (let i = 0; i < queryArray.length; i++) {
|
||||
item = queryArray[i]
|
||||
console.log(`item:`, item)
|
||||
if (typeof item == 'string') {
|
||||
result = sourceText.includes(item)
|
||||
console.log(`string search`, result)
|
||||
} else if (typeof item == 'object') {
|
||||
result = query(item, searchBy)
|
||||
console.log(`object search`, result)
|
||||
} else {
|
||||
throw new Error(`Unexpected item type in Query Array: ${typeof item}`)
|
||||
}
|
||||
|
||||
if (and) {
|
||||
if (!result) { return false }
|
||||
} else { // this is an 'OR' test
|
||||
if (result) { return true }
|
||||
}
|
||||
}
|
||||
return and
|
||||
}
|
||||
}
|
||||
|
||||
function GetFailureMessage() {
|
||||
const searchForFailureMessage = '\nFailure Message: '
|
||||
let iStart
|
||||
if (log.key.endsWith('.spec.ts')) {
|
||||
// The .ts tests usually have multiple errors in them, so we search from the beginning
|
||||
// in order to find the first failure, which usually causes the other failures.
|
||||
iStart = fullLogText.indexOf(searchForFailureMessage)
|
||||
} else {
|
||||
iStart = fullLogText.lastIndexOf(searchForFailureMessage)
|
||||
}
|
||||
if (iStart == -1) {
|
||||
console.log(`GetTriageDetailsAboutTestFailure - not found error`)
|
||||
return 'ERROR: Failure Message was not found in this log file.'
|
||||
} else {
|
||||
iStart += searchForFailureMessage.length
|
||||
|
||||
let iEnd
|
||||
let iEnd1 = fullLogText.indexOf('\n-+-', iStart)
|
||||
let iEnd2 = fullLogText.indexOf('\n***', iStart)
|
||||
if (iEnd1 == -1 && iEnd2 == -1) {
|
||||
iEnd = fullLogText.length
|
||||
} else {
|
||||
iEnd = Math.min(iEnd1, iEnd2)
|
||||
}
|
||||
|
||||
return fullLogText.substring(iStart, iEnd)
|
||||
}
|
||||
}
|
||||
|
||||
function GetErrorPanelText() {
|
||||
const errorPanelMarker = 'Error Panel found in Current HTML'
|
||||
const index = fullLogText.indexOf(errorPanelMarker)
|
||||
if (index == -1) { return undefined }
|
||||
|
||||
let start = fullLogText.indexOf('\n', index + errorPanelMarker.length) + 6
|
||||
let end = fullLogText.indexOf('\n-+- ', start)
|
||||
let errorMessage = fullLogText.substring(start, end).trim()
|
||||
|
||||
console.log(`start: ${start} - end: ${end}`)
|
||||
console.log('Extracted:\n' + errorMessage + '\n')
|
||||
|
||||
// Adding spaces to the end of each potential text string just to be sure there is a break between words.
|
||||
// Later we will remove any extra spaces this might create.
|
||||
errorMessage = errorMessage.replace(/<\//g, ' </')
|
||||
|
||||
const $ = cheerio.load(errorMessage)
|
||||
let text = $('div.cl-errorpanel').text().trim().replace(/\\"/g, '"').replace(/ |\\n/g, ' ')
|
||||
console.log(`Error Message:\n${text}<===\n`)
|
||||
return text
|
||||
}
|
||||
|
||||
console.log(`GetTriageDetailsAboutTestFailure - start`)
|
||||
return await apiData.Get(log.url).then(data => {
|
||||
try {
|
||||
fullLogText = data
|
||||
// console.log(fullLogText)
|
||||
// console.log()
|
||||
|
||||
failureMessage = GetFailureMessage()
|
||||
errorPanelText = GetErrorPanelText()
|
||||
|
||||
let returnValue = {
|
||||
message: failureMessage,
|
||||
errorPanelText: errorPanelText,
|
||||
}
|
||||
|
||||
for(let i = 0; i < triageData.length; i++) {
|
||||
console.log(`GetTriageDetailsAboutTestFailure - for i=${i}`)
|
||||
// testName is an "AND" condition with the other query conditions.
|
||||
if (triageData[i].testName) {
|
||||
if (!log.key.includes(triageData[i].testName)) {
|
||||
console.log(`GetTriageDetailsAboutTestFailure - 'testName' - continue not a match`)
|
||||
continue // because this one is not a match
|
||||
}
|
||||
}
|
||||
|
||||
if (query(triageData[i])) {
|
||||
returnValue.knownIssue = true
|
||||
returnValue.bugs = triageData[i].bugs
|
||||
returnValue.comment = triageData[i].comment
|
||||
break
|
||||
}
|
||||
}
|
||||
console.log(`GetTriageDetailsAboutTestFailure returns:`)
|
||||
console.log(returnValue)
|
||||
return returnValue
|
||||
}
|
||||
catch(error) {
|
||||
console.log(`!!! ERROR: ${error.message}`)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// UNIT TESTS - These are triggered ONLY when running this as a standalone module.
|
||||
if (require.main === module) {
|
||||
(async function () {
|
||||
const triageData = [
|
||||
{
|
||||
testName: 'Regression-Log',
|
||||
and: [
|
||||
`Timed out retrying: Expected to find content:`,
|
||||
`within the element: <div.wc-message-content> but never did.`,
|
||||
],
|
||||
bugs: [2197]
|
||||
},
|
||||
{
|
||||
and: [
|
||||
`cy.visit() failed trying to load:`,
|
||||
`http://localhost:3000/`
|
||||
],
|
||||
comment: 'This happens from time to time and there is no known fix for it.',
|
||||
},
|
||||
{
|
||||
and: [`Expected to find element:`, `but never found it.`, 'No WAY!'],
|
||||
comment: 'This should NEVER be the answer',
|
||||
},
|
||||
{ // Should be a match
|
||||
and: [
|
||||
`Expected to find element:`,
|
||||
`but never found it.`,
|
||||
{
|
||||
searchBy: FULL_LOG,
|
||||
or: [
|
||||
`This will NOT be found`,
|
||||
{
|
||||
and : [
|
||||
`Should import a model to test against and navigate to Train Dialogs view`,
|
||||
{
|
||||
searchBy: ERROR_PANEL,
|
||||
and: [`Creating Application Failed Request failed with status code 400 "Bad Request {"Locale":["The Locale field is required."]}`],
|
||||
},
|
||||
]},
|
||||
],
|
||||
},
|
||||
],
|
||||
comment: 'This can be the answer',
|
||||
},
|
||||
{ // Should be a match
|
||||
searchBy: ERROR_PANEL,
|
||||
and: [`Creating Application Failed Request failed with status code 400 "Bad Request {"Locale":["The Locale field is required."]}"`],
|
||||
comment: 'This is the correct answer'
|
||||
}
|
||||
]
|
||||
|
||||
SetTriageData(triageData)
|
||||
|
||||
await GetTriageDetailsAboutTestFailure({
|
||||
key: 'WeDoNotNeedToMatchThis',
|
||||
url: 'https://5509-94457606-gh.circle-artifacts.com/0/root/project/results/cypress/Regression-EditAndBranching-LastTurnUndo.spec.js.19.12.13.01.21.57..846.log'})
|
||||
}())
|
||||
}
|
||||
Загрузка…
Ссылка в новой задаче