Merge pull request #522 from electron/use-sccache

Use sccache to speed up builds

.circleci/config.yml

@@ -14,13 +14,16 @@ jobs:
           command: script/bootstrap
       - run:
           name: Update
-          command: script/update --clean -t $TARGET_ARCH
+          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache
       - run:
           name: Build shared library
           command: script/build -t $TARGET_ARCH -c $COMPONENT
       - run:
           name: Build FFmpeg
           command: script/build -t $TARGET_ARCH -c ffmpeg
+      - run:
+          name: Check sccache stats after build
+          command: script/sccache -s
       - run:
           name: Create distribution
           command: script/create-dist -t $TARGET_ARCH -c $COMPONENT
@@ -86,13 +89,16 @@ jobs:
           command: script/bootstrap
       - run:
           name: Update
-          command: script/update --clean -t $TARGET_ARCH
+          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache
       - run:
           name: Build shared library
           command: script/build -t $TARGET_ARCH -c $COMPONENT
       - run:
           name: Build FFmpeg
           command: script/build -t $TARGET_ARCH -c ffmpeg
+      - run:
+          name: Check sccache stats after build
+          command: script/sccache -s
       - run:
           name: Create distribution
           command: script/create-dist -t $TARGET_ARCH -c $COMPONENT
@@ -158,7 +164,7 @@ jobs:
           command: script/bootstrap
       - run:
           name: Update
-          command: script/update --clean -t $TARGET_ARCH
+          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache
       - run:
           name: Build mksnapshot
           command: script/build -t $TARGET_ARCH -c native_mksnapshot
@@ -168,6 +174,9 @@ jobs:
       - run:
           name: Build FFmpeg
           command: script/build -t $TARGET_ARCH -c ffmpeg
+      - run:
+          name: Check sccache stats after build
+          command: script/sccache -s
       - run:
           name: Create shared distribution
           command: script/create-dist -t $TARGET_ARCH -c $COMPONENT
@@ -238,7 +247,7 @@ jobs:
           command: script/bootstrap
       - run:
           name: Update
-          command: script/update --clean -t $TARGET_ARCH
+          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache
       - run:
           name: Build mksnapshot
           command: script/build -t $TARGET_ARCH -c native_mksnapshot
@@ -248,6 +257,9 @@ jobs:
       - run:
           name: Build FFmpeg
           command: script/build -t $TARGET_ARCH -c ffmpeg
+      - run:
+          name: Check sccache stats after build
+          command: script/sccache -s
       - run:
           name: Create shared distribution
           command: script/create-dist -t $TARGET_ARCH -c $COMPONENT
@@ -318,13 +330,16 @@ jobs:
           command: script/bootstrap
       - run:
           name: Update
-          command: script/update --clean -t $TARGET_ARCH
+          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache
       - run:
           name: Build shared library
           command: script/build -t $TARGET_ARCH -c $COMPONENT
       - run:
           name: Build FFmpeg
           command: script/build -t $TARGET_ARCH -c ffmpeg
+      - run:
+          name: Check sccache stats after build
+          command: script/sccache -s
       - run:
           name: Create distribution
           command: script/create-dist -t $TARGET_ARCH -c $COMPONENT
@@ -355,6 +370,7 @@ jobs:
       - run:
           name: Update
           command: script/update --clean -t $TARGET_ARCH
+
       - run:
           name: Build static library
           command: script/build -t $TARGET_ARCH -c $COMPONENT

appveyor.yml

@@ -1,32 +0,0 @@
-version: 1.0.{build}
-build_cloud: libcc-20
-image: libcc-20-vs2017
-clone_folder: c:\build
-environment:
-  matrix:
-  - TARGET_ARCH: ia32
-    COMPONENT: shared_library
-  - TARGET_ARCH: x64
-    COMPONENT: shared_library
-  - TARGET_ARCH: ia32
-    COMPONENT: static_library
-  - TARGET_ARCH: x64
-    COMPONENT: static_library
-build_script:
-- ps: >-
-    if(($env:APPVEYOR_PULL_REQUEST_HEAD_REPO_NAME -split "/")[0] -eq ($env:APPVEYOR_REPO_NAME -split "/")[0]) {
-      Write-warning "Skipping PR build for branch"; Exit-AppveyorBuild
-    } else {
-      script\cibuild.ps1
-      if ($? -ne 'True') {
-        throw "Build failed with exit code $?"
-      } else {
-        "Build succeeded."
-      }
-    }
-test: off
-artifacts:
-- path: libchromiumcontent.zip
-  name: libchromiumcontent.zip
-- path: libchromiumcontent-static.zip
-  name: libchromiumcontent-static.zip

script/cibuild

@@ -18,6 +18,7 @@ S3_CREDENTIALS_FILE = os.path.join(JENKINS_ROOT, 'config', 's3credentials')
 def main():
   args = parse_args()
   skip_upload = args.skip_upload
+  use_sccache = args.use_sccache
 
   if (not skip_upload and
      not 'LIBCHROMIUMCONTENT_S3_ACCESS_KEY' in os.environ and
@@ -34,23 +35,25 @@ def main():
 
   if 'TARGET_ARCH' in os.environ:
     args = ['-t', os.environ['TARGET_ARCH']]
-    return run_ci(args, skip_upload, os.environ['COMPONENT'])
+    return run_ci(args, skip_upload, use_sccache, os.environ['COMPONENT'])
 
   if sys.platform in ['win32', 'cygwin']:
-    return (run_ci(['-t', 'x64'], skip_upload) or
-            run_ci(['-t', 'ia32'], skip_upload))
+    return (run_ci(['-t', 'x64'], skip_upload, use_sccache) or
+            run_ci(['-t', 'ia32'], skip_upload, use_sccache))
   elif sys.platform == 'linux2':
-    return (run_ci(['-t', 'x64'], skip_upload) or
-            run_ci(['-t', 'ia32'], skip_upload) or
-            run_ci(['-t', 'arm'], skip_upload) or
-            run_ci(['-t', 'arm64'], skip_upload))
+    return (run_ci(['-t', 'x64'], skip_upload, use_sccache) or
+            run_ci(['-t', 'ia32'], skip_upload, use_sccache) or
+            run_ci(['-t', 'arm'], skip_upload, use_sccache) or
+            run_ci(['-t', 'arm64'], skip_upload, use_sccache))
   else:
-    return run_ci([], skip_upload)
+    return run_ci([], skip_upload, use_sccache)
 
 def parse_args():
   parser = argparse.ArgumentParser(description='Run CI build')
   parser.add_argument('-s', '--skip_upload', action='store_true',
                       help='Skip uploading to S3')
+  parser.add_argument('--use_sccache', action='store_true',
+                      help='Use sccache binary stored in with the libcc repo.')
   return parser.parse_args()
 
 def copy_to_environment(credentials_file):
@@ -72,9 +75,12 @@ def os_version():
     return platform.platform()
 
 
-def run_ci(args, skip_upload, component=None):
+def run_ci(args, skip_upload, use_sccache, component=None):
   build_args = []
   component_args = []
+  update_args = ['--clean'] + args
+  if use_sccache:
+    update_args += ['--use-bundled-sccache']
   if component is not None:
     component_args = ['-c', component]
     build_args += component_args + ['ffmpeg']
@@ -88,7 +94,7 @@ def run_ci(args, skip_upload, component=None):
     os.environ.update(vs_env)
 
   return (run_script('bootstrap') or
-          run_script('update', ['--clean'] + args) or
+          run_script('update', update_args) or
           run_script('build', args + build_args) or
           run_script('create-dist', args + component_args) or
           run_script('upload', args, skip_upload))

script/cibuild.ps1

@@ -1,4 +1,4 @@
-param([switch]$skipUpload)
+param([switch]$skipUpload, [switch]$useSccache)
 function Run-Command([scriptblock]$Command, [switch]$Fatal, [switch]$Quiet) {
   $output = ""
   try {
@@ -33,8 +33,13 @@ function Run-Command([scriptblock]$Command, [switch]$Fatal, [switch]$Quiet) {
 }
 
 Write-Output ""
+$CommandLine = "python .\script\cibuild"
 if ($skipUpload) {
-  Run-Command -Fatal { python .\script\cibuild --skip_upload }
-} else {
-  Run-Command -Fatal { python .\script\cibuild }
+  $CommandLine += " --skip_upload"
 }
+
+if ($useSccache) {
+  $CommandLine += " --use_sccache"
+}
+$CICommand = [ScriptBlock]::Create($CommandLine)
+Run-Command -Fatal $CICommand

script/lib/sccache.py

@@ -5,7 +5,7 @@ import sys
 from config import TOOLS_DIR
 
 
-VERSION = '0.2.6'
+VERSION = 'aad2120'
 SUPPORTED_PLATFORMS = {
   'cygwin': 'windows',
   'darwin': 'mac',

script/sccache

@@ -1,12 +1,34 @@
 #!/usr/bin/env python
 
+import argparse
+import os
 import sys
 
 import lib.sccache as sccache
 
+def parse_args():
+  parser = argparse.ArgumentParser(description='Run sccache`')
+  parser.add_argument('--azure_container', help='Name of azure container to use')
+  parser.add_argument('--azure_connection', help='Name of azure connection to use')
+  parser.add_argument('-s', '--show-stats', action='store_true', help='Show cache statistics')
+  parser.add_argument('--start-server', action='store_true', help='start background server')
+  parser.add_argument('--stop-server', action='store_true', help='stop background server')
+  parser.add_argument('-V', '--version', action='store_true', help='Prints version information')
+  parser.add_argument('-z', '--zero-stats', action='store_true', help='zero statistics counters')
+  return parser.parse_args()
 
 def main(args_list):
-  return sccache.run(*args_list)
+  args = parse_args()
+  sccache_args = list(args_list)
+  if args.azure_container:
+    os.environ['SCCACHE_AZURE_BLOB_CONTAINER'] = args.azure_container
+    sccache_args.remove('--azure_container')
+    sccache_args.remove(args.azure_container)
+  if args.azure_connection:
+    os.environ['SCCACHE_AZURE_CONNECTION_STRING'] = args.azure_connection
+    sccache_args.remove('--azure_connection')
+    sccache_args.remove(args.azure_connection)
+  return sccache.run(*sccache_args)
 
 
 if __name__ == '__main__':

tools/sccache/README.md

@@ -3,7 +3,7 @@
 Shared Compilation Cache  
 https://github.com/mozilla/sccache
 
-### Building a portable binary version 0.2.6 on Mac
+### Building a portable binary version on Mac
 
 The goal is to build a binary that would not require `openssl` to be installed in the system. The project's docs provide [build instructions](https://github.com/mozilla/sccache#building-portable-binaries)  for that but they're quite laconic.
 
@@ -12,8 +12,8 @@ Old versions might not be available via [Homebrew](https://brew.sh) but `brew in
 2. Clone the repo:  
 `git clone https://github.com/mozilla/sccache.git`
 3. `cd sccache`
-4. Checkout version 0.2.6:  
-`git fetch --tags && git checkout 0.2.6`
+4. Checkout version aad212037ce6340839686da8a1bdf5455b143b1c:  
+`git fetch --tags && git checkout aad212037ce6340839686da8a1bdf5455b143b1c`
 5. `export OPENSSL_STATIC=yes`
 6. Build the project in a release mode:  
 `cargo build --release`.

vsts.yml

@@ -47,18 +47,26 @@ phases:
 
   - bash: |
      echo "===Updating for $TARGET_ARCH===" > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
-     script/update --clean -t $TARGET_ARCH > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
+     if [ "${COMPONENT}" == "shared_library" ]; then
+       script/update --clean -t $TARGET_ARCH --use-bundled-sccache > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
+     else
+       script/update --clean -t $TARGET_ARCH > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
+     fi
     name: Update
 
   - bash: |
+     if [ "${COMPONENT}" == "shared_library" ]; then
+       script/sccache --start-server --azure_container "$(SCCACHE_AZURE_BLOB_CONTAINER)" --azure_connection "$(SCCACHE_AZURE_CONNECTION_STRING)" > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
+       script/sccache -s > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
+     fi
      echo "===Building $COMPONENT for $TARGET_ARCH===" > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
      script/build -t $TARGET_ARCH -c $COMPONENT > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
-    name: Build_library
-
-  - bash: |
      echo "===Building ffmpeg for $TARGET_ARCH===" > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
      script/build -t $TARGET_ARCH -c ffmpeg > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
-    name: Build_ffmpeg
+     if [ "${COMPONENT}" == "shared_library" ]; then
+       script/sccache --stop-server > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
+     fi
+    name: Build
 
   - bash: |
      echo "===Create $COMPONENT distribution for $TARGET_ARCH===" > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
@@ -98,3 +106,4 @@ phases:
       fi
 
   - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+    condition: always()