diff --git a/.gitignore b/.gitignore index edfe38ff..a4f2e40e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +.env + # Dependencies node_modules/ diff --git a/packages/ui/src/testResultReporting/TestResultAnalyzer.js b/packages/ui/src/testResultReporting/TestResultAnalyzer.js index 0861b01a..8e1b3bba 100644 --- a/packages/ui/src/testResultReporting/TestResultAnalyzer.js +++ b/packages/ui/src/testResultReporting/TestResultAnalyzer.js @@ -1,3 +1,20 @@ +// USAGE NOTES: +// +// 1) Go to https://circleci.com/account/api +// 2) Create a new token, copy to the clipboard, and paste into an ".env" file in the root folder on a new line like this... +// circle-token=[paste-token-string-here] +// 3) Run the tool from the root folder like this... +// npm run viewTestResults [build-number] +// You will find the build number on the page https://circleci.com/gh/microsoft/conversationlearner. +// Be sure to select the build number that coresponds to either a "test-smoke" or "test-regression" run. +// +// The expected result from using this tool is you will get a lot logging spew in the command window +// as the tool makes progress processing the test results artifacts. Once it completes, it will open +// the resulting report in your default browser window. +// +// The data used to triage each test failure is found in this folder "TriageData.js", it contains plenty +// of examples that you can follow to expand it as necessary. + const ttf = require('./TriageTestFailure') const apiData = require('./ApiData') const fs = require('fs') @@ -26,7 +43,9 @@ ttf.SetTriageData(triageData); buildNumber = process.argv[2] - const artifacts = await apiData.Get(`https://circleci.com/api/v1.1/project/github/microsoft/conversationLearner/${buildNumber}/artifacts?circle-token=2ad1e457047948114cb3bbb1957d6f90c1e2ee25`) + const circleCiToken = GetCircleCiToken() + + const artifacts = await apiData.Get(`https://circleci.com/api/v1.1/project/github/microsoft/conversationLearner/${buildNumber}/artifacts?circle-token=${circleCiToken}`) MoveArtifactJsonIntoArrays() console.log('Processing the Failed Test Results ----------------------------------') await ProcessFailingTestArtifacts() @@ -42,6 +61,25 @@ ttf.SetTriageData(triageData); child_process.exec(outputPath) + // --- End of Main process - worker functions below ----------------- + + function GetCircleCiToken() { + const envFileContents = fs.readFileSync('.env', { encoding: 'utf8' }) + const CIRCLECI_TOKEN = 'circle-token=' + const iStart = envFileContents.indexOf(CIRCLECI_TOKEN) + //console.log(typeof envFileContents, '\n', envFileContents) + if (iStart == -1) { + throw new Error('circle-token= is missing from the .env file') + } + let iEnd = envFileContents.indexOf('\n', iStart + CIRCLECI_TOKEN.length) + if (iEnd == -1) { + iEnd = envFileContents.length + } + const returnValue = envFileContents.substring(iStart + CIRCLECI_TOKEN.length, iEnd).trim() + //console.log(returnValue) + return returnValue + } + function MoveArtifactJsonIntoArrays() { artifacts.forEach(artifact => { const suffix = artifact.url.substring(artifact.url.length - 3) @@ -104,11 +142,7 @@ ttf.SetTriageData(triageData); console.log(`ProcessFailingTestArtifacts - going to await GetTriageDetailsAboutTestFailure`) failureDetails = await ttf.GetTriageDetailsAboutTestFailure(log) - if (typeof failureDetails == 'string') { - console.log(`ProcessFailingTestArtifacts got failureDetails: ${failureDetails}`) - } else { - console.log(`ProcessFailingTestArtifacts got failureDetails: { message: ${failureDetails.message}, bugs: ${failureDetails.bugs}, comment: ${failureDetails.comment} }`) - } + console.log(`ProcessFailingTestArtifacts got failureDetails: { message: ${failureDetails.message}, bugs: ${failureDetails.bugs}, comment: ${failureDetails.comment} }`) const mp4 = mp4s.find(mp4 => mp4.key === png.key) if (!mp4) { @@ -157,51 +191,6 @@ ttf.SetTriageData(triageData); }) } - function OLD_RenderResults() { - console.log(`${unknownTestFailures.length} UNKNOWN TEST FAILURES -------------------------------------------------------`) - unknownTestFailures.forEach(unknownTestFailure => { - console.log(`unknownTestFailures.push({ - testName: '${unknownTestFailure.testName}', - key: ${unknownTestFailure.key}, - snapshotUrl: '${unknownTestFailure.snapshotUrl}', - videoUrl: '${unknownTestFailure.videoUrl}', - logUrl: '${unknownTestFailure.logUrl}', - failureMessage: '${unknownTestFailure.failureMessage}', - })`) - }) - - console.log(`${knownTestFailures.length} KNOWN TEST FAILURES ---------------------------------------------------------`) - knownTestFailures.forEach(knownTestFailure => { - console.log(`knownTestFailures.push({ - testName: '${knownTestFailure.testName}', - key: ${knownTestFailure.key}, - snapshotUrl: '${knownTestFailure.snapshotUrl}', - videoUrl: '${knownTestFailure.videoUrl}', - logUrl: '${knownTestFailure.logUrl}', - failureMessage: '${knownTestFailure.failureMessage}', - bugs: '${knownTestFailure.bugs}', - })`) - }) - - console.log(`${passingTests.length} PASSING TESTS ---------------------------------------------------------------`) - passingTests.forEach(passingTest => { - console.log(`passingTests.push({ - testName: '${passingTest.testName}', - videoUrl: '${passingTest.videoUrl}', - logUrl: '${passingTest.logUrl}', - })`) - }) - - console.log(`${errors.length} ERRORS ---------------------------------------------------------------`) - errors.forEach(error => { - console.log(`errors.push({ - testName: '${error.testName}', - key: '${error.key}', - url: '${error.url}', - })`) - }) - } - function ReplaceSpecialHtmlCharacters(text) { return text.replace(/&/g, "&").replace(/>/g, ">").replace(/ { try { + if (!data || typeof data != 'string') { + console.log(`GetTriageDetailsAboutTestFailure - Error reading log file. Returned data is either undefined or not in the expected form`) + console.log('Log:', log) + console.log('data:', data) + return { + message: `failed to find the expected data in: ${log.url}`, + comment: 'Error Processing this Test Failure - try re-running the tool', + } + } + fullLogText = data // console.log(fullLogText) // console.log() @@ -161,11 +171,11 @@ async function GetTriageDetailsAboutTestFailure(log) { errorPanelText = GetErrorPanelText() let returnValue = { - message: failureMessage, + message: failureMessage, errorPanelText: errorPanelText, } - - for(let i = 0; i < triageData.length; i++) { + + for (let i = 0; i < triageData.length; i++) { console.log(`GetTriageDetailsAboutTestFailure - for i=${i}`) // testName is an "AND" condition with the other query conditions. if (triageData[i].testName) { @@ -186,8 +196,12 @@ async function GetTriageDetailsAboutTestFailure(log) { console.log(returnValue) return returnValue } - catch(error) { + catch (error) { console.log(`!!! ERROR: ${error.message}`) + return { + message: `failed to find the expected data in: ${log.url}`, + comment: 'Error Processing this Test Failure - try re-running the tool', + } } }) } @@ -217,20 +231,21 @@ if (require.main === module) { }, { // Should be a match and: [ - `Expected to find element:`, + `Expected to find element:`, `but never found it.`, { searchBy: FULL_LOG, or: [ `This will NOT be found`, - { - and : [ - `Should import a model to test against and navigate to Train Dialogs view`, - { - searchBy: ERROR_PANEL, - and: [`Creating Application Failed Request failed with status code 400 "Bad Request {"Locale":["The Locale field is required."]}`], - }, - ]}, + { + and: [ + `Should import a model to test against and navigate to Train Dialogs view`, + { + searchBy: ERROR_PANEL, + and: [`Creating Application Failed Request failed with status code 400 "Bad Request {"Locale":["The Locale field is required."]}`], + }, + ] + }, ], }, ], @@ -242,11 +257,12 @@ if (require.main === module) { comment: 'This is the correct answer' } ] - + SetTriageData(triageData) await GetTriageDetailsAboutTestFailure({ key: 'WeDoNotNeedToMatchThis', - url: 'https://5509-94457606-gh.circle-artifacts.com/0/root/project/results/cypress/Regression-EditAndBranching-LastTurnUndo.spec.js.19.12.13.01.21.57..846.log'}) + url: 'https://5509-94457606-gh.circle-artifacts.com/0/root/project/results/cypress/Regression-EditAndBranching-LastTurnUndo.spec.js.19.12.13.01.21.57..846.log' + }) }()) } \ No newline at end of file