Use packaged chromium source for CI

Patch so that we can exclude native_client source
2018-04-24 14:51:42 -04:00 · 2018-04-24 14:51:42 -04:00 · 44fb62a081
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -14,7 +14,8 @@ jobs:
          command: script/bootstrap
      - run:
          name: Update
-          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache
+          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache --use_packaged_src --upload_packaged_src
+          no_output_timeout: 60m
      - run:
          name: Build shared library
          command: script/build -t $TARGET_ARCH -c $COMPONENT
@ -53,7 +54,8 @@ jobs:
          command: script/bootstrap
      - run:
          name: Update
-          command: script/update --clean -t $TARGET_ARCH
+          command: script/update --clean -t $TARGET_ARCH --use_packaged_src
+          no_output_timeout: 20m
      - run:
          name: Build static library
          command: script/build -t $TARGET_ARCH -c $COMPONENT
@ -90,7 +92,7 @@ jobs:
          command: script/bootstrap
      - run:
          name: Update
-          command: script/update --clean -t $TARGET_ARCH
+          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache --use_packaged_src
      - run:
          name: Build static library
          command: script/build -t $TARGET_ARCH -c $COMPONENT
@ -134,7 +136,8 @@ jobs:
          command: script/bootstrap
      - run:
          name: Update
-          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache
+          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache --use_packaged_src
+          no_output_timeout: 20m
      - run:
          name: Build shared library
          command: script/build -t $TARGET_ARCH -c $COMPONENT
@ -173,7 +176,8 @@ jobs:
          command: script/bootstrap
      - run:
          name: Update
-          command: script/update --clean -t $TARGET_ARCH
+          command: script/update --clean -t $TARGET_ARCH --use_packaged_src
+          no_output_timeout: 20m
      - run:
          name: Build static library
          command: script/build -t $TARGET_ARCH -c $COMPONENT
@ -210,7 +214,7 @@ jobs:
          command: script/bootstrap
      - run:
          name: Update
-          command: script/update --clean -t $TARGET_ARCH
+          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache --use_packaged_src
      - run:
          name: Build static library
          command: script/build -t $TARGET_ARCH -c $COMPONENT
@ -235,7 +239,7 @@ jobs:
      - store_test_results:
          path: test_reports
      - store_artifacts:
-          path: test_reports          
+          path: test_reports

  libchromiumcontent-linux-arm-shared:
    docker:
@ -251,7 +255,8 @@ jobs:
          command: script/bootstrap
      - run:
          name: Update
-          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache
+          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache --use_packaged_src
+          no_output_timeout: 20m
      - run:
          name: Build mksnapshot
          command: script/build -t $TARGET_ARCH -c native_mksnapshot
@ -298,7 +303,8 @@ jobs:
          command: script/bootstrap
      - run:
          name: Update
-          command: script/update --clean -t $TARGET_ARCH
+          command: script/update --clean -t $TARGET_ARCH --use_packaged_src
+          no_output_timeout: 20m
      - run:
          name: Build static library
          command: script/build -t $TARGET_ARCH -c $COMPONENT
@ -334,7 +340,8 @@ jobs:
          command: script/bootstrap
      - run:
          name: Update
-          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache
+          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache --use_packaged_src
+          no_output_timeout: 20m
      - run:
          name: Build mksnapshot
          command: script/build -t $TARGET_ARCH -c native_mksnapshot
@ -381,7 +388,8 @@ jobs:
          command: script/bootstrap
      - run:
          name: Update
-          command: script/update --clean -t $TARGET_ARCH
+          command: script/update --clean -t $TARGET_ARCH --use_packaged_src
+          no_output_timeout: 20m
      - run:
          name: Build static library
          command: script/build -t $TARGET_ARCH -c $COMPONENT
@ -417,7 +425,8 @@ jobs:
          command: script/bootstrap
      - run:
          name: Update
-          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache
+          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache --use_packaged_src
+          no_output_timeout: 20m
      - run:
          name: Build shared library
          command: script/build -t $TARGET_ARCH -c $COMPONENT
@ -456,8 +465,8 @@ jobs:
          command: script/bootstrap
      - run:
          name: Update
-          command: script/update --clean -t $TARGET_ARCH
-
+          command: script/update --clean -t $TARGET_ARCH --use_packaged_src
+          no_output_timeout: 20m
      - run:
          name: Build static library
          command: script/build -t $TARGET_ARCH -c $COMPONENT
--- a/chromiumcontent/args/native_mksnapshot.gn
+++ b/chromiumcontent/args/native_mksnapshot.gn
@ -5,6 +5,7 @@ is_debug = false
 enable_linux_installer = false
 v8_use_snapshot = true
 v8_enable_i18n_support = false
+enable_nacl = false
 if (target_cpu == "arm") {
  v8_snapshot_toolchain="//build/toolchain/linux:clang_arm"
 }
--- a/patches/common/chromium/.patches.yaml
+++ b/patches/common/chromium/.patches.yaml
@ -341,3 +341,9 @@ patches:
    in electron (and other applications). This patch provides a way to disable
    the redraw locking mechanism, which fixes these issues. The electron issue
    can be found at https://github.com/electron/electron/issues/1821
+-
+  owners: jkleinsc
+  file: backport_4dabe8070550.patch
+  description:  |
+    Backports part of 4dabe8070550cb4014d1f996aa1a7e130bef42e1 so that we don't
+    need the native_client source tree.
--- a/patches/common/chromium/backport_4dabe8070550.patch
+++ b/patches/common/chromium/backport_4dabe8070550.patch
@ -0,0 +1,19 @@
+diff --git a/BUILD.gn b/BUILD.gn
+index df54a977..38a4956 100644
+--- a/BUILD.gn
+++ b/BUILD.gn
+@@ -688,10 +688,10 @@ group("gn_all") {
+       if (!is_android) {
+         deps += [ "//chrome/test:load_library_perf_tests" ]
+       }
+-      deps += [
+-        "//native_client/src/trusted/platform_qualify:vcpuid",
+-        "//third_party/libjpeg_turbo:simd_asm",
+-      ]
+      if (enable_nacl) {
+        deps += [ "//native_client/src/trusted/platform_qualify:vcpuid" ]
+      }
+      deps += [ "//third_party/libjpeg_turbo:simd_asm" ]
+     }
+     if (is_linux && current_toolchain == host_toolchain) {
+       deps += [ "//v8:v8_shell" ]
--- a/script/cibuild
+++ b/script/cibuild
@ -87,9 +87,11 @@ def run_ci(target_arch, skip_upload, use_sccache, build_tests, component=None):
    args += ['-t', target_arch]
  build_args = []
  component_args = []
-  update_args = ['--clean'] + args
+  update_args = ['--clean', '--use_packaged_src'] + args
  if use_sccache:
    update_args += ['--use-bundled-sccache']
+  if target_arch == 'x64':
+    update_args += ['--upload_packaged_src']
  if component is not None:
    component_args = ['-c', component]
    build_args += component_args + ['ffmpeg']
--- a/script/download
+++ b/script/download
@ -6,22 +6,11 @@ import errno
 import os
 import subprocess
 import sys
-import tempfile
-import urllib2
-import tarfile

 import lib.filesystem as filesystem
 import lib.git as git

-from lib.config import SOURCE_ROOT, TOOLS_DIR
-
-
-PLATFORM_KEY = {
-  'cygwin': 'win',
-  'darwin': 'osx',
-  'linux2': 'linux',
-  'win32': 'win',
-}[sys.platform]
+from lib.config import PLATFORM_KEY, SOURCE_ROOT, TOOLS_DIR

 SHARED_LIBRARY_FILENAME = 'libchromiumcontent.tar.bz2'
 STATIC_LIBRARY_FILENAME = 'libchromiumcontent-static.tar.bz2'
@ -125,24 +114,7 @@ def download(destination, base_url, commit, filename):
  sys.stderr.write('Downloading {0}...\n'.format(filename))
  sys.stderr.flush()
  url = '{0}/{1}/{2}'.format(base_url, commit, filename)
-  download_and_extract(destination, url)
-
-
-def download_and_extract(destination, url):
-  print url
-  with tempfile.TemporaryFile() as t:
-    with contextlib.closing(urllib2.urlopen(url)) as u:
-      while True:
-        chunk = u.read(1024*1024)
-        if not len(chunk):
-          break
-        sys.stderr.write('.')
-        sys.stderr.flush()
-        t.write(chunk)
-    sys.stderr.write('\nExtracting...\n')
-    sys.stderr.flush()
-    with tarfile.open(fileobj=t, mode='r:bz2') as z:
-      z.extractall(destination)
+  filesystem.download_and_extract(destination, url)


 def generate_filenames_gypi(target_file, libcc_source_path,
--- a/script/lib/config.py
+++ b/script/lib/config.py
@ -2,6 +2,7 @@

 import os
 import platform
+import sys

 SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
 SRC_DIR = os.path.join(SOURCE_ROOT, 'src')
@ -22,6 +23,13 @@ IS_ARM64_HOST = platform.machine() == 'aarch64'
 # Whether the host system is an arm64 machine
 IS_ARMV7_HOST = platform.machine() == 'armv7l'

+PLATFORM_KEY = {
+  'cygwin': 'win',
+  'darwin': 'osx',
+  'linux2': 'linux',
+  'win32': 'win',
+}[sys.platform]
+
 def set_mips64el_env(env):
  gcc_dir = os.path.join(VENDOR_DIR, MIPS64EL_GCC)
  ldlib_dirs = [
--- a/script/lib/filesystem.py
+++ b/script/lib/filesystem.py
@ -1,9 +1,14 @@
 """Filesystem related helper functions.
 """

+import contextlib
 import errno
 import os
 import shutil
+import sys
+import tarfile
+import tempfile
+import urllib2


 def mkdir_p(path):
@ -36,3 +41,19 @@ def safe_unlink(path):
  except OSError as e:
    if e.errno != errno.ENOENT:
      raise
+
+def download_and_extract(destination, url):
+  print url
+  with tempfile.TemporaryFile() as t:
+    with contextlib.closing(urllib2.urlopen(url)) as u:
+      while True:
+        chunk = u.read(1024*1024)
+        if not len(chunk):
+          break
+        sys.stderr.write('.')
+        sys.stderr.flush()
+        t.write(chunk)
+    sys.stderr.write('\nExtracting...\n')
+    sys.stderr.flush()
+    with tarfile.open(fileobj=t, mode='r:bz2') as z:
+      z.extractall(destination)
--- a/script/update
+++ b/script/update
@ -9,6 +9,7 @@ import subprocess
 import sys
 import urllib2
 import platform
+import tarfile

 import lib.filesystem as filesystem
 import lib.sccache as sccache
@ -16,12 +17,13 @@ import lib.util as util

 from lib.config import MIPS64EL_GCC, MIPS64EL_GCC_URL, MIPS64EL_SYSROOT, \
                       set_mips64el_env, IS_ARM64_HOST, IS_ARMV7_HOST, \
-                       SOURCE_ROOT, VENDOR_DIR, DEPOT_TOOLS_DIR, SRC_DIR
-
+                       SOURCE_ROOT, VENDOR_DIR, DEPOT_TOOLS_DIR, SRC_DIR, \
+                       PLATFORM_KEY

 CHROMIUMCONTENT_SOURCE_DIR = os.path.join(SOURCE_ROOT, 'chromiumcontent')
 CHROMIUMCONTENT_DESTINATION_DIR = os.path.join(
    SRC_DIR, 'libchromiumcontent', 'chromiumcontent')
+CHROMIUMCONTENT_SRC_URL = 'https://s3.amazonaws.com/github-janky-artifacts/libchromiumcontent'

 DEBIAN_MIRROR = 'http://ftp.jp.debian.org/debian/pool/main/'
 BINTOOLS_NAME = 'c/cross-binutils/binutils-aarch64-linux-gnu_2.25-5_amd64.deb'
@ -38,6 +40,14 @@ solutions = [
 ]
 {cache_dir_line}
 '''
+SRC_FILES_TO_IGNORE = [
+  os.path.join('src', 'native_client'),
+  os.path.join('src', 'third_party', 'WebKit', 'LayoutTests'),
+  os.path.join('src', 'third_party', 'catapult', 'tracing', 'test_data'),
+  os.path.join('src', 'third_party', 'deqp'),
+  os.path.join('src', 'third_party', 'sqlite', 'sqlite-src-3200100', 'test'),
+  os.path.join('src', 'third_party', 'sqlite', 'src', 'test')
+]

 def main():
  args = parse_args()
@ -59,8 +69,12 @@ def main():
  target_arch = args.target_arch
  nohooks = target_arch == 'arm64' and IS_ARM64_HOST or target_arch == 'arm' and IS_ARMV7_HOST

+
  if args.run_gclient:
-    gclient_sync(chromium_version(), args.clean, git_cache, nohooks)
+    if args.use_packaged_src:
+      get_packaged_src(args.clean, git_cache, nohooks, args.upload_packaged_src)
+    else:
+      gclient_sync(chromium_version(), args.clean, git_cache, nohooks)
    if args.source_only:
      return

@ -93,6 +107,10 @@ def parse_args():
                      help='Skips update depot tools on Windows, easier to build locally')
  parser.add_argument('--source_only', action='store_true',
                      help='Only sync the Chromium source code')
+  parser.add_argument('--use_packaged_src', action='store_true',
+                      help='Get chromium source as packaged file')
+  parser.add_argument('--upload_packaged_src', action='store_true',
+                      help='Upload packaged chromium source if it does not exist')

  cc_wrapper_group = parser.add_mutually_exclusive_group()
  cc_wrapper_group.add_argument('--cc_wrapper',
@ -299,6 +317,44 @@ def download(url, filename):
          break
        f.write(chunk)

+def get_packaged_src(clean, git_cache, nohooks, upload_src):
+  src_version = chromium_version()
+  tar_name = 'src.tar.bz2'
+  package_url = '{0}/{1}/src/{2}/{3}'.format(CHROMIUMCONTENT_SRC_URL,
+                                         PLATFORM_KEY, src_version, tar_name)
+  try:
+    print('Trying to download ' + package_url)
+    filesystem.download_and_extract(SOURCE_ROOT, package_url)
+    os.environ['CHROMIUM_BUILDTOOLS_PATH'] = os.path.join(SRC_DIR, 'buildtools')
+  except urllib2.URLError as err:
+    print('Could not download {0}, syncing from git'.format(package_url))
+    gclient_sync(src_version, clean, git_cache, nohooks)
+    if upload_src:
+      print('Packaging source to ' + tar_name + '.')
+      tar_path =  os.path.join(SOURCE_ROOT, tar_name)
+      filesystem.safe_unlink(tar_path)
+      with util.scoped_cwd(SOURCE_ROOT):
+        tar_file = tarfile.open(tar_path, 'w:bz2')
+        tar_file.add('src', exclude=exclude_packaged_files)
+        tar_file.close()
+      if ('LIBCHROMIUMCONTENT_S3_ACCESS_KEY' in os.environ and
+         'LIBCHROMIUMCONTENT_S3_BUCKET' in os.environ and
+         'LIBCHROMIUMCONTENT_S3_SECRET_KEY' in os.environ):
+        script_path = os.path.join(SOURCE_ROOT, 'script', 'upload')
+        script_arguments = ['--upload_packaged_src', '--chromium_version', src_version]
+        print('Uploading source to ' + package_url + '.')
+        subprocess.check_call([sys.executable, script_path] + script_arguments)
+  except:
+    print('Unexpected error downloading and extracting chromium source:', sys.exc_info()[0])
+    raise
+
+
+def exclude_packaged_files(filename):
+  if ".git" in filename:
+    return True
+  else:
+    return filename in SRC_FILES_TO_IGNORE
+
 if __name__ == '__main__':
  import sys
  sys.exit(main())
--- a/script/upload
+++ b/script/upload
@ -9,23 +9,16 @@ import sys

 import lib.git as git

+from lib.config import PLATFORM_KEY

 SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 BOTO_DIR = os.path.join(SOURCE_ROOT, 'vendor', 'boto')

-PLATFORM_KEY = {
-  'cygwin': 'win',
-  'darwin': 'osx',
-  'linux2': 'linux',
-  'win32': 'win',
-}[sys.platform]
-
-
 def main():
  args = parse_args()

  try:
-    upload(args.target_arch)
+    upload(args)
  except AssertionError as e:
    return e.message

@ -33,10 +26,14 @@ def main():
 def parse_args():
  parser = argparse.ArgumentParser(description='Upload distribution')
  parser.add_argument('-t', '--target_arch', default='x64', help='x64 or ia32')
+  parser.add_argument('--upload_packaged_src', action='store_true',
+                      help='Upload packaged source to s3')
+  parser.add_argument('--chromium_version',
+                      help='Chromium version for uploading packaged source to s3')
  return parser.parse_args()


-def upload(target_arch):
+def upload(args):
  os.chdir(SOURCE_ROOT)
  bucket, access_key, secret_key = s3_config()
  commit = git.get_head_commit(SOURCE_ROOT)
@ -46,10 +43,16 @@ def upload(target_arch):
  else:
    platform = PLATFORM_KEY

-  upload_files = glob.glob('libchromiumcontent*.tar.bz2') + glob.glob('native-mksnapshot.tar.bz2')
-  s3put(bucket, access_key, secret_key, SOURCE_ROOT,
-        'libchromiumcontent/{0}/{1}/{2}'.format(platform, target_arch, commit),
-        upload_files)
+  if args.upload_packaged_src:
+    upload_files = ['src.tar.bz2']
+    s3put(bucket, access_key, secret_key, SOURCE_ROOT,
+          'libchromiumcontent/{0}/src/{1}'.format(PLATFORM_KEY, args.chromium_version),
+          upload_files)
+  else:
+    upload_files = glob.glob('libchromiumcontent*.tar.bz2') + glob.glob('native-mksnapshot.tar.bz2')
+    s3put(bucket, access_key, secret_key, SOURCE_ROOT,
+          'libchromiumcontent/{0}/{1}/{2}'.format(platform, args.target_arch, commit),
+          upload_files)


 def s3_config():
--- a/vsts.yml
+++ b/vsts.yml
@ -46,18 +46,48 @@ phases:
     set -e
     echo "===Bootstrapping===" > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
     script/bootstrap > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
+     mkdir s3files
    name: Bootstrap

  - bash: |
     set -e
     echo "===Updating for $TARGET_ARCH===" > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
     if [ "${COMPONENT}" == "shared_library" ]; then
-       script/update --clean -t $TARGET_ARCH --use-bundled-sccache > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
+       if [ "${TARGET_TYPE}" == "osx" ]; then
+         # Only upload packaged source once, so use osx shared_library build to do so
+         echo "===Updating for $TARGET_ARCH=== uploading packaged src" > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
+         echo "##vso[task.setvariable variable=ChromiumVersion]`cat VERSION`"
+         script/update --clean -t $TARGET_ARCH --use-bundled-sccache --use_packaged_src --upload_packaged_src > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
+         if [ -f src.tar.bz2 ]; then
+           echo "Packaged source exists, setting it up for upload"
+           mv src.tar.bz2 s3files
+           echo "##vso[task.setvariable variable=UploadPackagedSrc]True"
+           echo "UploadPackagedSrc has value: $(UploadPackagedSrc)"
+         else
+           echo "Packaged source does NOT exist, skipping upload"
+           ls -la *
+         fi
+       else
+         script/update --clean -t $TARGET_ARCH --use-bundled-sccache --use_packaged_src > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
+       fi
     else
-       script/update --clean -t $TARGET_ARCH > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
+       script/update --clean -t $TARGET_ARCH --use_packaged_src > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
     fi
    name: Update

+  - task: AmazonWebServices.aws-vsts-tools.S3Upload.S3Upload@1
+    inputs:
+      awsCredentials: 'Libchromium Content S3'
+      regionName: 'us-east-1'
+      bucketName: 'github-janky-artifacts'
+      sourceFolder: s3files
+      globExpressions: 'src.tar.*'
+      targetFolder: 'libchromiumcontent/$(TARGET_TYPE)/src/$(ChromiumVersion)'
+      filesAcl: 'public-read'
+      logRequest: true
+      logResponse: true
+    condition: and(succeeded(), eq(variables['UploadPackagedSrc'], 'True'))
+
  - bash: |
     set -e
     if [ "${COMPONENT}" == "shared_library" ]; then
@ -81,7 +111,6 @@ phases:
     set -e
     echo "===Create $COMPONENT distribution for $TARGET_ARCH===" > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
     script/create-dist -t $TARGET_ARCH -c $COMPONENT > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
-     mkdir s3files
     mv libchromiumcontent* s3files
     mv buildlog.txt s3files
    name: Create_distribution
@ -107,7 +136,7 @@ phases:
      regionName: 'us-east-1'
      bucketName: 'github-janky-artifacts'
      sourceFolder: s3files
-      globExpressions: '*'
+      globExpressions: 'libchromiumcontent*'
      targetFolder: 'libchromiumcontent/$(TARGET_TYPE)/$(TARGET_ARCH)/$(gitcommit)'
      filesAcl: 'public-read'
      logRequest: true