Fix CircleCI Exposed Token Issue in Test Code (#17)

* fix: removed the exposed secret * fix: adding some usage comments * fix: improved error handling
2019-12-22 17:57:52 -08:00 · 2019-12-22 17:57:52 -08:00 · ed4aedcda5
--- a/.gitignore
+++ b/.gitignore
@ -1,3 +1,5 @@
+.env
+
 # Dependencies
 node_modules/

--- a/packages/ui/src/testResultReporting/TestResultAnalyzer.js
+++ b/packages/ui/src/testResultReporting/TestResultAnalyzer.js
@ -1,3 +1,20 @@
+// USAGE NOTES:
+//
+// 1) Go to https://circleci.com/account/api
+// 2) Create a new token, copy to the clipboard, and paste into an ".env" file in the root folder on a new line like this...
+//      circle-token=[paste-token-string-here]
+// 3) Run the tool from the root folder like this...
+//      npm run viewTestResults [build-number]
+//    You will find the build number on the page https://circleci.com/gh/microsoft/conversationlearner.
+//    Be sure to select the build number that coresponds to either a "test-smoke" or "test-regression" run.
+//
+//    The expected result from using this tool is you will get a lot logging spew in the command window
+//    as the tool makes progress processing the test results artifacts. Once it completes, it will open
+//    the resulting report in your default browser window.
+//
+// The data used to triage each test failure is found in this folder "TriageData.js", it contains plenty
+// of examples that you can follow to expand it as necessary.
+
 const ttf = require('./TriageTestFailure')
 const apiData = require('./ApiData')
 const fs = require('fs')
@ -26,7 +43,9 @@ ttf.SetTriageData(triageData);

  buildNumber = process.argv[2]

-  const artifacts = await apiData.Get(`https://circleci.com/api/v1.1/project/github/microsoft/conversationLearner/${buildNumber}/artifacts?circle-token=2ad1e457047948114cb3bbb1957d6f90c1e2ee25`)
+  const circleCiToken = GetCircleCiToken()
+
+  const artifacts = await apiData.Get(`https://circleci.com/api/v1.1/project/github/microsoft/conversationLearner/${buildNumber}/artifacts?circle-token=${circleCiToken}`)
  MoveArtifactJsonIntoArrays()
  console.log('Processing the Failed Test Results ----------------------------------')
  await ProcessFailingTestArtifacts()
@ -42,6 +61,25 @@ ttf.SetTriageData(triageData);

  child_process.exec(outputPath)

+  // --- End of Main process - worker functions below -----------------
+
+  function GetCircleCiToken() {
+    const envFileContents = fs.readFileSync('.env', { encoding: 'utf8' })
+    const CIRCLECI_TOKEN = 'circle-token='
+    const iStart = envFileContents.indexOf(CIRCLECI_TOKEN)
+    //console.log(typeof envFileContents, '\n', envFileContents)
+    if (iStart == -1) {
+      throw new Error('circle-token= is missing from the .env file')
+    }
+    let iEnd = envFileContents.indexOf('\n', iStart + CIRCLECI_TOKEN.length)
+    if (iEnd == -1) {
+      iEnd = envFileContents.length
+    }
+    const returnValue = envFileContents.substring(iStart + CIRCLECI_TOKEN.length, iEnd).trim()
+    //console.log(returnValue)
+    return returnValue
+  }
+
  function MoveArtifactJsonIntoArrays() {
    artifacts.forEach(artifact => {
      const suffix = artifact.url.substring(artifact.url.length - 3)
@ -104,11 +142,7 @@ ttf.SetTriageData(triageData);

      console.log(`ProcessFailingTestArtifacts - going to await GetTriageDetailsAboutTestFailure`)
      failureDetails = await ttf.GetTriageDetailsAboutTestFailure(log)
-      if (typeof failureDetails == 'string') {
-        console.log(`ProcessFailingTestArtifacts got failureDetails: ${failureDetails}`)
-      } else {
-        console.log(`ProcessFailingTestArtifacts got failureDetails: { message: ${failureDetails.message}, bugs: ${failureDetails.bugs}, comment: ${failureDetails.comment} }`)
-      }
+      console.log(`ProcessFailingTestArtifacts got failureDetails: { message: ${failureDetails.message}, bugs: ${failureDetails.bugs}, comment: ${failureDetails.comment} }`)

      const mp4 = mp4s.find(mp4 => mp4.key === png.key)
      if (!mp4) {
@ -157,51 +191,6 @@ ttf.SetTriageData(triageData);
    })
  }

-  function OLD_RenderResults() {
-    console.log(`${unknownTestFailures.length} UNKNOWN TEST FAILURES -------------------------------------------------------`)
-    unknownTestFailures.forEach(unknownTestFailure => {
-      console.log(`unknownTestFailures.push({
-        testName: '${unknownTestFailure.testName}',
-        key: ${unknownTestFailure.key},
-        snapshotUrl: '${unknownTestFailure.snapshotUrl}',
-        videoUrl: '${unknownTestFailure.videoUrl}',
-        logUrl: '${unknownTestFailure.logUrl}',
-        failureMessage: '${unknownTestFailure.failureMessage}',
-      })`)
-    })
-
-    console.log(`${knownTestFailures.length} KNOWN TEST FAILURES ---------------------------------------------------------`)
-    knownTestFailures.forEach(knownTestFailure => {
-      console.log(`knownTestFailures.push({
-        testName: '${knownTestFailure.testName}',
-        key: ${knownTestFailure.key},
-        snapshotUrl: '${knownTestFailure.snapshotUrl}',
-        videoUrl: '${knownTestFailure.videoUrl}',
-        logUrl: '${knownTestFailure.logUrl}',
-        failureMessage: '${knownTestFailure.failureMessage}',
-        bugs: '${knownTestFailure.bugs}',
-      })`)
-    })
-
-    console.log(`${passingTests.length} PASSING TESTS ---------------------------------------------------------------`)
-    passingTests.forEach(passingTest => {
-      console.log(`passingTests.push({
-        testName: '${passingTest.testName}',
-        videoUrl: '${passingTest.videoUrl}',
-        logUrl: '${passingTest.logUrl}',
-      })`)
-    })
-
-    console.log(`${errors.length} ERRORS ---------------------------------------------------------------`)
-    errors.forEach(error => {
-      console.log(`errors.push({
-        testName: '${error.testName}',
-        key: '${error.key}',
-        url: '${error.url}',
-      })`)
-    })
-  }
-
  function ReplaceSpecialHtmlCharacters(text) {
    return text.replace(/&/g, "&amp;").replace(/>/g, "&gt;").replace(/</g, "&lt;")
  }
--- a/packages/ui/src/testResultReporting/TriageData.js
+++ b/packages/ui/src/testResultReporting/TriageData.js
@ -13,6 +13,7 @@ exports.triageData = [
  {
    and: [`Timed out retrying: Expected to find element: 'button.ms-Dropdown-item[title="Enum"]', but never found it.`],
    bugs: [2409],
+    comment: 'Fails to create an entity - only on CircleCI',
  },
  {
    and: [`Bugs 2389 & 2400 - Entity Detection panel shows a phrase that is different than the user's utterance.`],
--- a/packages/ui/src/testResultReporting/TriageTestFailure.js
+++ b/packages/ui/src/testResultReporting/TriageTestFailure.js
@ -33,15 +33,15 @@ async function GetTriageDetailsAboutTestFailure(log) {
    let returnValue = _query()
    console.log(`query returned:`, returnValue, '\n')
    return returnValue
-    
+
    function _query() {
      // Determine which string will be searched...
      let searchBy = defaultSearchBy
      if (queryObj.searchBy) {
-        searchBy = queryObj.searchBy  
+        searchBy = queryObj.searchBy
      }
      let sourceText
-      switch(searchBy) {
+      switch (searchBy) {
        case FAILURE_MESSAGE:
          sourceText = failureMessage
          break;
@ -54,7 +54,7 @@ async function GetTriageDetailsAboutTestFailure(log) {
        default:
          throw new Error(`Unexpected searchBy value: '${searchBy}'`)
      }
-      
+
      if (!sourceText) {
        console.log(`sourceText for searchBy '${searchBy}' is undefined`)
        return false
@ -64,14 +64,14 @@ async function GetTriageDetailsAboutTestFailure(log) {
      // Determine whether multiple conditions will be ANDed or ORed together to form the result...
      let and
      let queryArray
-      if(queryObj.and) {
+      if (queryObj.and) {
        and = true
        queryArray = queryObj.and
-      } else if(queryObj.or) {
+      } else if (queryObj.or) {
        and = false
        queryArray = queryObj.or
      }
-      
+
      console.log(`query`, searchBy, and ? 'AND' : 'OR', queryArray)

      // Do the actual search here...
@ -88,8 +88,8 @@ async function GetTriageDetailsAboutTestFailure(log) {
        } else {
          throw new Error(`Unexpected item type in Query Array: ${typeof item}`)
        }
-        
-        if (and) { 
+
+        if (and) {
          if (!result) { return false }
        } else { // this is an 'OR' test
          if (result) { return true }
@ -98,7 +98,7 @@ async function GetTriageDetailsAboutTestFailure(log) {
      return and
    }
  }
-  
+
  function GetFailureMessage() {
    const searchForFailureMessage = '\nFailure Message: '
    let iStart
@ -132,18 +132,18 @@ async function GetTriageDetailsAboutTestFailure(log) {
    const errorPanelMarker = 'Error Panel found in Current HTML'
    const index = fullLogText.indexOf(errorPanelMarker)
    if (index == -1) { return undefined }
-  
+
    let start = fullLogText.indexOf('\n', index + errorPanelMarker.length) + 6
    let end = fullLogText.indexOf('\n-+- ', start)
    let errorMessage = fullLogText.substring(start, end).trim()
-  
+
    console.log(`start: ${start} - end: ${end}`)
    console.log('Extracted:\n' + errorMessage + '\n')
-  
+
    // Adding spaces to the end of each potential text string just to be sure there is a break between words.
    // Later we will remove any extra spaces this might create.
    errorMessage = errorMessage.replace(/<\//g, ' </')
-  
+
    const $ = cheerio.load(errorMessage)
    let text = $('div.cl-errorpanel').text().trim().replace(/\\"/g, '"').replace(/  |\\n/g, ' ')
    console.log(`Error Message:\n${text}<===\n`)
@ -153,6 +153,16 @@ async function GetTriageDetailsAboutTestFailure(log) {
  console.log(`GetTriageDetailsAboutTestFailure - start`)
  return await apiData.Get(log.url).then(data => {
    try {
+      if (!data || typeof data != 'string') {
+        console.log(`GetTriageDetailsAboutTestFailure - Error reading log file. Returned data is either undefined or not in the expected form`)
+        console.log('Log:', log)
+        console.log('data:', data)
+        return {
+          message: `failed to find the expected data in: ${log.url}`,
+          comment: 'Error Processing this Test Failure - try re-running the tool',
+        }
+      }
+
      fullLogText = data
      // console.log(fullLogText)
      // console.log()
@ -161,11 +171,11 @@ async function GetTriageDetailsAboutTestFailure(log) {
      errorPanelText = GetErrorPanelText()

      let returnValue = {
-        message: failureMessage, 
+        message: failureMessage,
        errorPanelText: errorPanelText,
      }
-  
-      for(let i = 0; i < triageData.length; i++) {
+
+      for (let i = 0; i < triageData.length; i++) {
        console.log(`GetTriageDetailsAboutTestFailure - for i=${i}`)
        // testName is an "AND" condition with the other query conditions.
        if (triageData[i].testName) {
@ -186,8 +196,12 @@ async function GetTriageDetailsAboutTestFailure(log) {
      console.log(returnValue)
      return returnValue
    }
-    catch(error) {
+    catch (error) {
      console.log(`!!! ERROR: ${error.message}`)
+      return {
+        message: `failed to find the expected data in: ${log.url}`,
+        comment: 'Error Processing this Test Failure - try re-running the tool',
+      }
    }
  })
 }
@ -217,20 +231,21 @@ if (require.main === module) {
      },
      { // Should be a match
        and: [
-          `Expected to find element:`, 
+          `Expected to find element:`,
          `but never found it.`,
          {
            searchBy: FULL_LOG,
            or: [
              `This will NOT be found`,
-              { 
-                and : [
-                `Should import a model to test against and navigate to Train Dialogs view`,
-                { 
-                  searchBy: ERROR_PANEL,
-                  and: [`Creating Application Failed Request failed with status code 400 "Bad Request {"Locale":["The Locale field is required."]}`],
-                },
-              ]},
+              {
+                and: [
+                  `Should import a model to test against and navigate to Train Dialogs view`,
+                  {
+                    searchBy: ERROR_PANEL,
+                    and: [`Creating Application Failed Request failed with status code 400 "Bad Request {"Locale":["The Locale field is required."]}`],
+                  },
+                ]
+              },
            ],
          },
        ],
@ -242,11 +257,12 @@ if (require.main === module) {
        comment: 'This is the correct answer'
      }
    ]
-   
+
    SetTriageData(triageData)

    await GetTriageDetailsAboutTestFailure({
      key: 'WeDoNotNeedToMatchThis',
-      url: 'https://5509-94457606-gh.circle-artifacts.com/0/root/project/results/cypress/Regression-EditAndBranching-LastTurnUndo.spec.js.19.12.13.01.21.57..846.log'})
+      url: 'https://5509-94457606-gh.circle-artifacts.com/0/root/project/results/cypress/Regression-EditAndBranching-LastTurnUndo.spec.js.19.12.13.01.21.57..846.log'
+    })
  }())
 }