Merge pull request #13241 from electron/test-releases

spec: run tests on release builds

.circleci/config.yml

@@ -6,7 +6,7 @@ build-steps: &build-steps
         command: |
           if [ -n "${RUN_RELEASE_BUILD}" ]; then
             echo 'release build triggered from api'
-            echo 'export ELECTRON_RELEASE=1 TRIGGERED_BY_API=1' >> $BASH_ENV
+            echo 'export ELECTRON_RELEASE=1 UPLOAD_TO_S3=1' >> $BASH_ENV
           fi
     - run:
        name: Bootstrap
@@ -43,10 +43,10 @@ build-steps: &build-steps
     - run:
         name: Upload distribution
         command: |
-          if [ "$ELECTRON_RELEASE" == "1" ] && [ "$TRIGGERED_BY_API" != "1" ]; then
+          if [ "$ELECTRON_RELEASE" == "1" ] && [ "$UPLOAD_TO_S3" != "1" ]; then
             echo 'Uploading Electron release distribution to github releases'
             script/upload.py
-          elif [ "$ELECTRON_RELEASE" == "1" ] && [ "$TRIGGERED_BY_API" == "1" ]; then
+          elif [ "$ELECTRON_RELEASE" == "1" ] && [ "$UPLOAD_TO_S3" == "1" ]; then
             echo 'Uploading Electron release distribution to s3'
             script/upload.py --upload_to_s3
           else
@@ -55,7 +55,7 @@ build-steps: &build-steps
     - run:
         name: Setup for headless testing
         command: |
-          if [ "$ELECTRON_RELEASE" != "1" ] && [ "$RUN_HEADLESS_TESTS" == "true" ]; then
+          if [ "$RUN_HEADLESS_TESTS" == "true" ]; then
             echo 'Setup for headless testing'
             sh -e /etc/init.d/xvfb start
           else
@@ -67,30 +67,34 @@ build-steps: &build-steps
           MOCHA_FILE: junit/test-results.xml
           MOCHA_REPORTER: mocha-junit-reporter
         command: |
-          if [ "$ELECTRON_RELEASE" != "1" ] && [ "$RUN_TESTS" == "true" ]; then
+          if [ "$RUN_TESTS" == "true" ]; then
-            echo 'Testing Electron debug build'
+            if [ "$ELECTRON_RELEASE" != "1" ]; then
-            mkdir junit
+              echo 'Testing Electron debug build'
-            script/test.py --ci
+              mkdir junit
-          else
+              script/test.py --ci --rebuild_native_modules
-            if [ "$ELECTRON_RELEASE" == "1" ]; then
-              echo 'Skipping testing on release build'
             else
-              echo 'Skipping tests due to configuration'
+              echo 'Testing Electron release build'
+              mkdir junit
+              script/test.py --ci --rebuild_native_modules -c R
             fi
+          else
+           echo 'Skipping tests due to configuration'
           fi
     - run:
         name: Verify FFmpeg
         command: |
-          if [ "$ELECTRON_RELEASE" != "1" ] && [ "$RUN_TESTS" == "true" ]; then
+          if [ "$RUN_TESTS" == "true" ]; then
-            echo 'Verifying ffmpeg on debug build'
+            if [ "$ELECTRON_RELEASE" != "1" ]; then
-            script/verify-ffmpeg.py
+              echo 'Verifying ffmpeg on debug build'
-          else
+              script/verify-ffmpeg.py
-            if [ "$ELECTRON_RELEASE" == "1" ]; then
-              echo 'Skipping verify ffmpeg on release build'
             else
-              echo 'Skipping tests due to configuration'
+              echo 'Verifying ffmpeg on release build'
+              script/verify-ffmpeg.py -R
             fi
+          else
+            echo 'Skipping tests due to configuration'
           fi
+
     - run:
         name: Generate Typescript Definitions
         command: |
@@ -311,3 +315,12 @@ workflows:
       - electron-linux-arm64-release-nightly
       - electron-linux-ia32-release-nightly
       - electron-linux-x64-release-nightly
+
+experimental:
+  notify:
+    branches:
+      only:
+        - master
+        - 2-0-x
+        - 1-8-x
+        - 1-7-x

script/ci-release-build.js

@@ -1,6 +1,7 @@
 const assert = require('assert')
 const request = require('request')
 const buildAppVeyorURL = 'https://windows-ci.electronjs.org/api/builds'
+const vstsURL = 'https://github.visualstudio.com/electron/_apis/build'
 
 const circleCIJobs = [
   'electron-linux-arm',
@@ -10,6 +11,11 @@ const circleCIJobs = [
   'electron-linux-x64'
 ]
 
+const vstsJobs = [
+  'electron-release-mas-x64',
+  'electron-release-osx-x64'
+]
+
 async function makeRequest (requestOptions, parseResponse) {
   return new Promise((resolve, reject) => {
     request(requestOptions, (err, res, body) => {
@@ -62,7 +68,7 @@ async function circleCIcall (buildUrl, targetBranch, job, options) {
   }, true).catch(err => {
     console.log('Error calling CircleCI:', err)
   })
-  console.log(`Check ${circleResponse.build_url} for status. (${job})`)
+  console.log(`CircleCI release build request for ${job} successful.  Check ${circleResponse.build_url} for status.`)
 }
 
 async function buildAppVeyor (targetBranch, options) {
@@ -113,6 +119,70 @@ function buildCircleCI (targetBranch, options) {
   }
 }
 
+async function buildVSTS (targetBranch, options) {
+  if (options.job) {
+    assert(vstsJobs.includes(options.job), `Unknown CI job name: ${options.job}.`)
+  }
+  console.log(`Triggering VSTS to run build on branch: ${targetBranch} with release flag.`)
+  assert(process.env.VSTS_TOKEN, 'VSTS_TOKEN not found in environment')
+  let environmentVariables = {}
+
+  if (!options.ghRelease) {
+    environmentVariables.UPLOAD_TO_S3 = 1
+  }
+
+  if (options.automaticRelease) {
+    environmentVariables.AUTO_RELEASE = 'true'
+  }
+
+  let requestOpts = {
+    url: `${vstsURL}/definitions?api-version=4.1`,
+    auth: {
+      user: '',
+      password: process.env.VSTS_TOKEN
+    },
+    headers: {
+      'Content-Type': 'application/json'
+    }
+  }
+  let vstsResponse = await makeRequest(requestOpts, true).catch(err => {
+    console.log('Error calling VSTS to get build definitions:', err)
+  })
+  let buildsToRun = []
+  if (options.job) {
+    buildsToRun = vstsResponse.value.filter(build => build.name === options.job)
+  } else {
+    buildsToRun = vstsResponse.value.filter(build => vstsJobs.includes(build.name))
+  }
+  buildsToRun.forEach((build) => callVSTSBuild(build, targetBranch, environmentVariables))
+}
+
+async function callVSTSBuild (build, targetBranch, environmentVariables) {
+  let buildBody = {
+    definition: build,
+    sourceBranch: targetBranch
+  }
+  if (Object.keys(environmentVariables).length !== 0) {
+    buildBody.parameters = JSON.stringify(environmentVariables)
+  }
+  let requestOpts = {
+    url: `${vstsURL}/builds?api-version=4.1`,
+    auth: {
+      user: '',
+      password: process.env.VSTS_TOKEN
+    },
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify(buildBody),
+    method: 'POST'
+  }
+  let vstsResponse = await makeRequest(requestOpts, true).catch(err => {
+    console.log(`Error calling VSTS for job ${build.name}`, err)
+  })
+  console.log(`VSTS release build request for ${build.name} successful. Check ${vstsResponse._links.web.href} for status.`)
+}
+
 function runRelease (targetBranch, options) {
   if (options.ci) {
     switch (options.ci) {
@@ -124,10 +194,19 @@ function runRelease (targetBranch, options) {
         buildAppVeyor(targetBranch, options)
         break
       }
+      case 'VSTS': {
+        buildVSTS(targetBranch, options)
+        break
+      }
+      default: {
+        console.log(`Error! Unknown CI: ${options.ci}.`)
+        process.exit(1)
+      }
     }
   } else {
     buildCircleCI(targetBranch, options)
     buildAppVeyor(targetBranch, options)
+    buildVSTS(targetBranch, options)
   }
 }
 
@@ -140,7 +219,7 @@ if (require.main === module) {
   const targetBranch = args._[0]
   if (args._.length < 1) {
     console.log(`Trigger CI to build release builds of electron.
-    Usage: ci-release-build.js [--job=CI_JOB_NAME] [--ci=CircleCI|AppVeyor] [--ghRelease] [--automaticRelease] TARGET_BRANCH
+    Usage: ci-release-build.js [--job=CI_JOB_NAME] [--ci=CircleCI|AppVeyor|VSTS] [--ghRelease] [--automaticRelease] TARGET_BRANCH
     `)
     process.exit(0)
   }

script/upload.py

@@ -212,6 +212,7 @@ def create_release_draft(github, tag):
 
 
 def upload_electron(github, release, file_path, args):
+  filename = os.path.basename(file_path)
 
   # if upload_to_s3 is set, skip github upload.
   if args.upload_to_s3:
@@ -221,10 +222,11 @@ def upload_electron(github, release, file_path, args):
     s3put(bucket, access_key, secret_key, os.path.dirname(file_path),
           key_prefix, [file_path])
     upload_sha256_checksum(release['tag_name'], file_path, key_prefix)
+    s3url = 'https://gh-contractor-zcbenz.s3.amazonaws.com'
+    print '{0} uploaded to {1}/{2}/{0}'.format(filename, s3url, key_prefix)
     return
 
   # Delete the original file before uploading in CI.
-  filename = os.path.basename(file_path)
   if os.environ.has_key('CI'):
     try:
       for asset in release['assets']:

vsts.yml

@@ -33,7 +33,7 @@ steps:
   condition: and(succeeded(), eq(variables['ELECTRON_RELEASE'], '1'))
 
 - bash: |
-    if [ "$TRIGGERED_BY_API" != "1" ]; then
+    if [ "$UPLOAD_TO_S3" != "1" ]; then
       echo 'Uploading Electron release distribution to github releases'
       ELECTRON_S3_BUCKET="$(s3_bucket)" ELECTRON_S3_ACCESS_KEY="$(s3_access_key)" ELECTRON_S3_SECRET_KEY="$(s3_secret_key)" ELECTRON_GITHUB_TOKEN="$(github_token)" script/upload.py
     else
@@ -48,16 +48,31 @@ steps:
     mkdir junit
     export MOCHA_FILE="junit/test-results.xml"
     export MOCHA_REPORTER="mocha-junit-reporter"
-    script/test.py --ci
+    script/test.py --ci --rebuild_native_modules
   name: Test
   condition: and(succeeded(), ne(variables['ELECTRON_RELEASE'], '1'))
 
+- bash: |
+    echo 'Testing Electron release build'
+    mkdir junit
+    export MOCHA_FILE="junit/test-results.xml"
+    export MOCHA_REPORTER="mocha-junit-reporter"
+    script/test.py --ci --rebuild_native_modules -c R
+  name: Test
+  condition: and(succeeded(), eq(variables['ELECTRON_RELEASE'], '1'))
+
 - bash: |
     echo 'Verifying ffmpeg on debug build'
     script/verify-ffmpeg.py
   name: Verify_FFmpeg
   condition: and(succeeded(), ne(variables['ELECTRON_RELEASE'], '1'))
 
+- bash: |
+    echo 'Verifying ffmpeg on release build'
+    script/verify-ffmpeg.py -R
+  name: Verify_FFmpeg
+  condition: and(succeeded(), eq(variables['ELECTRON_RELEASE'], '1'))
+
 - task: PublishTestResults@2
   displayName: Publish Test Results
   inputs:
@@ -65,4 +80,12 @@ steps:
     searchFolder: junit
   condition: and(always(), ne(variables['ELECTRON_RELEASE'], '1'))
 
+- task: kasunkodagoda.slack-notification.slack-notification-task.SlackNotification@3
+  displayName: Post Slack Notification
+  inputs:
+    SlackApiToken: '$(slack_token)'
+    Channel: '#bot-nightly-releases'
+    Message: '$(Build.DefinitionName)-$(Build.BuildNumber) finished with a $(Agent.JobStatus) status.'
+  condition: and(always(), eq(variables['Build.Reason'], 'Schedule'))
+
 - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3