Bug 1540284 [wpt PR 16108] - [Azure Pipelines] Run scheduled runs of Safari stable, a=testonly

Automatic update from web-platform-tests
[Azure Pipelines] Run daily run of Safari stable (#16108)

Rather than adding support for getting the version of Safari stable using
PlistBuddy in tools/wpt/browser.py, pass in the version with
`--browser-version` and make sure that it is not overwritten internally.

The next stable Safari release should support `safaridriver --version`.
--

wpt-commits: 146c7368173ea5336d39049788d993f93774eca6
wpt-pr: 16108

testing/web-platform/tests/.azure-pipelines.yml

@@ -198,6 +198,40 @@ jobs:
     dependsOn: results_edge
     artifactName: edge-results
 
+- job: results_safari
+  displayName: 'all tests (Safari)'
+  condition: eq(variables['Build.Reason'], 'Schedule')
+  strategy:
+    parallel: 4 # chosen to make runtime ~2h
+  timeoutInMinutes: 360
+  pool:
+    vmImage: 'macOS-10.13'
+  steps:
+  - template: tools/ci/azure/checkout.yml
+  - template: tools/ci/azure/pip_install.yml
+    parameters:
+      packages: virtualenv
+  - template: tools/ci/azure/install_fonts.yml
+  - template: tools/ci/azure/install_certs.yml
+  - template: tools/ci/azure/install_safari.yml
+    parameters:
+      channel: stable
+  - template: tools/ci/azure/update_hosts.yml
+  - template: tools/ci/azure/update_manifest.yml
+  - script: |
+      # TODO: drop this when `safaridriver --version` is supported.
+      SAFARI_VERSION="$(/usr/libexec/PlistBuddy -c 'Print :CFBundleShortVersionString' /Applications/Safari.app/Contents/Info.plist) ($(/usr/libexec/PlistBuddy -c 'Print :CFBundleVersion' /Applications/Safari.app/Contents/Info.plist))"
+      no_proxy='*' ./wpt run --no-manifest-update --no-restart-on-unexpected --no-fail-on-unexpected --this-chunk=$(System.JobPositionInPhase) --total-chunks=$(System.TotalJobsInPhase) --chunk-type hash --log-wptreport $(Build.ArtifactStagingDirectory)/wpt_report_$(System.JobPositionInPhase).json --log-wptscreenshot $(Build.ArtifactStagingDirectory)/wpt_screenshot_$(System.JobPositionInPhase).txt --browser-version "$SAFARI_VERSION" safari
+    displayName: 'Run tests'
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish results'
+    inputs:
+      artifactName: 'safari-results'
+- template: tools/ci/azure/fyi_hook.yml
+  parameters:
+    dependsOn: results_safari
+    artifactName: safari-results
+
 - job: results_safari_preview
   displayName: 'all tests (Safari Technology Preview)'
   condition: |

testing/web-platform/tests/tools/ci/azure/install_safari.yml

@@ -1,9 +1,19 @@
+parameters:
+  channel: preview
+
 steps:
-- script: |
-    # This is equivalent to `Homebrew/homebrew-cask-versions/safari-technology-preview`,
-    # but the raw URL is used to bypass caching.
-    HOMEBREW_NO_AUTO_UPDATE=1 brew cask install https://raw.githubusercontent.com/Homebrew/homebrew-cask-versions/master/Casks/safari-technology-preview.rb
-    # https://web-platform-tests.org/running-tests/safari.html
-    sudo "/Applications/Safari Technology Preview.app/Contents/MacOS/safaridriver" --enable
-    defaults write com.apple.Safari WebKitJavaScriptCanOpenWindowsAutomatically 1
-  displayName: 'Install Safari Technology Preview'
+- ${{ if eq(parameters.channel, 'preview') }}:
+  - script: |
+      # This is equivalent to `Homebrew/homebrew-cask-versions/safari-technology-preview`,
+      # but the raw URL is used to bypass caching.
+      HOMEBREW_NO_AUTO_UPDATE=1 brew cask install https://raw.githubusercontent.com/Homebrew/homebrew-cask-versions/master/Casks/safari-technology-preview.rb
+      # https://web-platform-tests.org/running-tests/safari.html
+      sudo "/Applications/Safari Technology Preview.app/Contents/MacOS/safaridriver" --enable
+      defaults write com.apple.Safari WebKitJavaScriptCanOpenWindowsAutomatically 1
+    displayName: 'Install Safari Technology Preview'
+- ${{ if eq(parameters.channel, 'stable') }}:
+  - script: |
+      # https://web-platform-tests.org/running-tests/safari.html
+      sudo safaridriver --enable
+      defaults write com.apple.Safari WebKitJavaScriptCanOpenWindowsAutomatically 1
+    displayName: 'Configure Safari'

testing/web-platform/tests/tools/wpt/run.py

@@ -568,8 +568,12 @@ def setup_wptrunner(venv, prompt=True, install_browser=False, **kwargs):
     if not venv.skip_virtualenv_setup:
         venv.install_requirements(os.path.join(wptrunner_path, "requirements.txt"))
 
-    kwargs['browser_version'] = setup_cls.browser.version(binary=kwargs.get("binary"),
-                                                          webdriver_binary=kwargs.get("webdriver_binary"))
+    # Only update browser_version if it was not given as a command line
+    # argument, so that it can be overridden on the command line.
+    if not kwargs["browser_version"]:
+        kwargs["browser_version"] = setup_cls.browser.version(binary=kwargs.get("binary"),
+                                                              webdriver_binary=kwargs.get("webdriver_binary"))
+
     return kwargs