build: [gn] add distributable zip target (#14093)

* build: [gn] add distributable zip target * build: update the CircleCI config - enable debug and testing builds on Mac - run Mac release builds nightly - run test for Mac release builds - use shared build machines configs * Add resources dir to zip on non mac platforms.
2018-08-21 10:06:28 -07:00 · 2018-08-21 10:06:28 -07:00 · f82f89b2a3
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -192,6 +192,11 @@ gn-build-steps: &gn-build-steps
        command: |
          cd src
          ninja -C out/Default electron:electron_app
+    - run:
+        name: Build dist.zip
+        command: |
+          cd src
+          ninja -C out/Default electron:electron_dist_zip
    - run:
        name: Check sccache stats after build
        command: $SCCACHE_WRAPPER -s
@ -217,6 +222,88 @@ gn-build-steps: &gn-build-steps
        path: src/junit
    - store_artifacts:
        path: src/junit
+    - store_artifacts:
+        path: src/out/Default/dist.zip
+
+gn-mac-build-steps: &gn-mac-build-steps
+  steps:
+    - run:
+        name: Setup depot tools
+        command: |
+          git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git
+          echo 'export PATH="$PATH:'"$PWD"'/depot_tools"' >> $BASH_ENV
+          echo 'export GIT_CACHE_PATH="$HOME/.gclient-cache"' >> $BASH_ENV
+    - run:
+        name: Install Node.js 10 on MacOS
+        command: |
+          echo 'Installing Node.js 10 for MacOS'
+          brew update
+          brew install node@10
+          brew install gnu-tar
+          echo 'export PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH"' >> $BASH_ENV
+    - checkout:
+        path: src/electron
+    - run:
+        name: Gclient sync
+        command: |
+          gclient config \
+            --name "src/electron" \
+            --unmanaged \
+            $GCLIENT_EXTRA_ARGS \
+            "$CIRCLE_REPOSITORY_URL"
+          gclient sync --with_branch_heads --with_tags
+    - run:
+        name: GN gen
+        command: |
+          cd src
+          SCCACHE_PATH="$PWD/libchromiumcontent/tools/sccache/aad2120/mac/sccache"
+          echo 'export SCCACHE_WRAPPER="'"$SCCACHE_PATH"'"' >> $BASH_ENV
+          echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/buildtools"' >> $BASH_ENV
+          source $BASH_ENV
+          gn gen out/Default --args='import("'$GN_CONFIG'") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS"
+    - run:
+        name: Ninja build
+        command: |
+          cd src
+          ninja -C out/Default electron:electron_app
+    - run:
+        name: Build dist.zip
+        command: |
+          cd src
+          ninja -C out/Default electron:electron_dist_zip
+    - run:
+        name: Check sccache stats after build
+        command: $SCCACHE_WRAPPER -s
+    - run:
+        name: Test
+        environment:
+          MOCHA_REPORTER: mocha-junit-reporter
+          MOCHA_FILE: junit/test-results.xml
+          ELECTRON_DISABLE_SECURITY_WARNINGS: 1
+        command: |
+          if [ "$RUN_TESTS" != "false" ]; then
+            cd src
+            ninja -C out/Default third_party/electron_node:headers
+            export npm_config_nodedir="$PWD/out/Default/gen/node_headers"
+            (cd electron/spec && npm install)
+            ./out/Default/Electron.app/Contents/MacOS/Electron electron/spec --ci --enable-logging
+          fi
+    - store_test_results:
+        path: src/junit
+    - store_artifacts:
+        path: src/junit
+    - store_artifacts:
+        path: src/out/Default/dist.zip
+
+gn-linux-build-machine: &gn-linux-build-machine
+  docker:
+    - image: electronbuilds/electron:0.0.8
+  resource_class: 2xlarge
+
+gn-mac-build-machine: &gn-mac-build-machine
+  macos:
+    xcode: "8.3.3"
+  resource_class: large

 build-defaults: &build-defaults
  docker:
@ -389,27 +476,21 @@ jobs:
      DISPLAY: ':99.0'
      GN_CONFIG: //electron/build/args/debug.gn
      RUN_TESTS: false
-    docker:
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
+    <<: *gn-linux-build-machine
    <<: *gn-build-steps

  electron-gn-linux-x64-testing-fyi:
    environment:
      DISPLAY: ':99.0'
      GN_CONFIG: //electron/build/args/testing.gn
-    docker:
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
+    <<: *gn-linux-build-machine
    <<: *gn-build-steps

  electron-gn-linux-x64-release-fyi:
    environment:
      DISPLAY: ':99.0'
      GN_CONFIG: //electron/build/args/release.gn
-    docker:
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
+    <<: *gn-linux-build-machine
    <<: *gn-build-steps

  electron-gn-linux-ia32-debug-fyi:
@ -419,9 +500,7 @@ jobs:
      GN_EXTRA_ARGS: 'target_cpu = "x86"'
      NPM_CONFIG_ARCH: ia32
      RUN_TESTS: false
-    docker:
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
+    <<: *gn-linux-build-machine
    <<: *gn-build-steps

  electron-gn-linux-ia32-testing-fyi:
@ -430,9 +509,7 @@ jobs:
      GN_CONFIG: //electron/build/args/testing.gn
      GN_EXTRA_ARGS: 'target_cpu = "x86"'
      NPM_CONFIG_ARCH: ia32
-    docker:
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
+    <<: *gn-linux-build-machine
    <<: *gn-build-steps

  electron-gn-linux-ia32-release-fyi:
@ -441,9 +518,7 @@ jobs:
      GN_CONFIG: //electron/build/args/release.gn
      GN_EXTRA_ARGS: 'target_cpu = "x86"'
      NPM_CONFIG_ARCH: ia32
-    docker:
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
+    <<: *gn-linux-build-machine
    <<: *gn-build-steps

  electron-gn-linux-arm-debug-fyi:
@ -452,9 +527,7 @@ jobs:
      GN_EXTRA_ARGS: 'target_cpu = "arm"'
      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True'
      RUN_TESTS: false
-    docker:
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
+    <<: *gn-linux-build-machine
    <<: *gn-build-steps

  electron-gn-linux-arm-testing-fyi:
@ -463,9 +536,7 @@ jobs:
      GN_EXTRA_ARGS: 'target_cpu = "arm"'
      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True'
      RUN_TESTS: false
-    docker:
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
+    <<: *gn-linux-build-machine
    <<: *gn-build-steps

  electron-gn-linux-arm-release-fyi:
@ -474,9 +545,7 @@ jobs:
      GN_EXTRA_ARGS: 'target_cpu = "arm"'
      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True'
      RUN_TESTS: false
-    docker:
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
+    <<: *gn-linux-build-machine
    <<: *gn-build-steps

  electron-gn-linux-arm64-debug-fyi:
@ -485,9 +554,7 @@ jobs:
      GN_EXTRA_ARGS: 'target_cpu = "arm64" fatal_linker_warnings = false enable_linux_installer = false'
      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm64=True'
      RUN_TESTS: false
-    docker:
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
+    <<: *gn-linux-build-machine
    <<: *gn-build-steps

  electron-gn-linux-arm64-testing-fyi:
@ -496,9 +563,7 @@ jobs:
      GN_EXTRA_ARGS: 'target_cpu = "arm64" fatal_linker_warnings = false enable_linux_installer = false'
      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm64=True'
      RUN_TESTS: false
-    docker:
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
+    <<: *gn-linux-build-machine
    <<: *gn-build-steps

  electron-gn-linux-arm64-release-fyi:
@ -507,11 +572,54 @@ jobs:
      GN_EXTRA_ARGS: 'target_cpu = "arm64" fatal_linker_warnings = false enable_linux_installer = false'
      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm64=True'
      RUN_TESTS: false
-    docker:
-      - image: electronbuilds/electron:0.0.8
-    resource_class: 2xlarge
+    <<: *gn-linux-build-machine
    <<: *gn-build-steps

+  electron-gn-osx-release-fyi:
+    environment:
+      GN_CONFIG: //electron/build/args/release.gn
+      RUN_TESTS: true
+    <<: *gn-mac-build-machine
+    <<: *gn-mac-build-steps
+
+  electron-gn-osx-testing-fyi:
+    environment:
+      GN_CONFIG: //electron/build/args/testing.gn
+      RUN_TESTS: true
+    <<: *gn-mac-build-machine
+    <<: *gn-mac-build-steps
+
+  electron-gn-osx-debug-fyi:
+    environment:
+      GN_CONFIG: //electron/build/args/debug.gn
+      RUN_TESTS: false
+    <<: *gn-mac-build-machine
+    <<: *gn-mac-build-steps
+
+  electron-gn-mas-release-fyi:
+    environment:
+      GN_CONFIG: //electron/build/args/release.gn
+      RUN_TESTS: true
+      GN_EXTRA_ARGS: 'is_mas_build = true'
+    <<: *gn-mac-build-machine
+    <<: *gn-mac-build-steps
+
+  electron-gn-mas-testing-fyi:
+    environment:
+      GN_CONFIG: //electron/build/args/testing.gn
+      RUN_TESTS: true
+      GN_EXTRA_ARGS: 'is_mas_build = true'
+    <<: *gn-mac-build-machine
+    <<: *gn-mac-build-steps
+
+  electron-gn-mas-debug-fyi:
+    environment:
+      GN_CONFIG: //electron/build/args/debug.gn
+      RUN_TESTS: false
+      GN_EXTRA_ARGS: 'is_mas_build = true'
+    <<: *gn-mac-build-machine
+    <<: *gn-mac-build-steps
+
 workflows:
  version: 2
  build-arm:
@ -548,6 +656,12 @@ workflows:
      - electron-gn-linux-arm-testing-fyi
      - electron-gn-linux-arm64-debug-fyi
      - electron-gn-linux-arm64-testing-fyi
+  build-gn-mac:
+    jobs:
+      - electron-gn-mas-debug-fyi
+      - electron-gn-mas-testing-fyi
+      - electron-gn-osx-debug-fyi
+      - electron-gn-osx-testing-fyi

  nightly-release-test:
    triggers:
@ -580,3 +694,5 @@ workflows:
      - electron-gn-linux-ia32-release-fyi
      - electron-gn-linux-arm-release-fyi
      - electron-gn-linux-arm64-release-fyi
+      - electron-gn-mas-release-fyi
+      - electron-gn-osx-release-fyi
--- a/BUILD.gn
+++ b/BUILD.gn
@ -183,12 +183,6 @@ asar("app2asar") {
  root = "default_app"
 }

-group("electron") {
-  deps = [
-    ":electron_lib",
-  ]
-}
-
 static_library("electron_lib") {
  configs += [ "//v8:external_startup_data" ]
  configs += [ "//third_party/electron_node:node_internals" ]
@ -631,6 +625,21 @@ if (is_mac) {
      "//ui/strings",
    ]

+    data = []
+
+    data += [ "$root_out_dir/resources.pak" ]
+    data += [ "$root_out_dir/chrome_100_percent.pak" ]
+    if (enable_hidpi) {
+      data += [ "$root_out_dir/chrome_200_percent.pak" ]
+    }
+    foreach(locale, locales) {
+      data += [ "$root_out_dir/locales/$locale.pak" ]
+    }
+
+    if (!is_mac) {
+      data += [ "$root_out_dir/resources" ]
+    }
+
    public_deps = [
      "//tools/v8_context_snapshot:v8_context_snapshot",
    ]
@ -753,3 +762,55 @@ group("chromium_browsertests") {
    "//content/test:content_browsertests",
  ]
 }
+
+template("dist_zip") {
+  _runtime_deps_target = "${target_name}__deps"
+  _runtime_deps_file =
+      "$root_out_dir/gen.runtime/" +
+      get_label_info(target_name, "dir") + "/" +
+      get_label_info(target_name, "name") + ".runtime_deps"
+
+  group(_runtime_deps_target) {
+    forward_variables_from(invoker, [ "deps", "data_deps", "data" ])
+    write_runtime_deps = _runtime_deps_file
+  }
+
+  action(target_name) {
+    script = "//electron/build/zip.py"
+    deps = [ ":$_runtime_deps_target" ]
+    forward_variables_from(invoker, [ "outputs" ])
+    args =  rebase_path(outputs + [ _runtime_deps_file ], root_build_dir) + [target_cpu, target_os]
+  }
+}
+
+
+copy("electron_license") {
+  sources = [ "LICENSE" ]
+  outputs = [ "$root_build_dir/{{source_file_part}}" ]
+}
+copy("chromium_licenses") {
+  deps = [ "//components/resources:about_credits" ]
+  sources = [ "$root_gen_dir/components/resources/about_credits.html" ]
+  outputs = [ "$root_build_dir/LICENSES.chromium.html" ]
+}
+
+group("licenses") {
+  data_deps = [ ":electron_license", ":chromium_licenses" ]
+}
+
+action("electron_version") {
+  script = "build/write_version.py"
+  outputs = [ "$root_build_dir/version" ]
+  args = rebase_path(outputs, root_build_dir) + [ electron_version ]
+}
+
+
+dist_zip("electron_dist_zip") {
+  data_deps = [ ":electron_app", ":licenses", ":electron_version" ]
+  outputs = [ "$root_build_dir/dist.zip" ]
+}
+
+
+group("electron") {
+  deps = [ ":electron_app" ]
+}
--- a/appveyor-gn.yml
+++ b/appveyor-gn.yml
@ -21,6 +21,7 @@ build_script:
  - md src
  - ps: Move-Item $env:APPVEYOR_BUILD_FOLDER -Destination src\electron
  - ps: $env:CHROMIUM_BUILDTOOLS_PATH="$pwd\src\buildtools"
+  - ps: $env:SCCACHE_PATH="$pwd\src\libchromiumcontent\tools\sccache\aad2120\windows\sccache.exe"
  - >-
      gclient config
      --name "src\electron"
@ -29,8 +30,13 @@ build_script:
      "https://github.com/electron/electron"
  - gclient sync --with_branch_heads --with_tags
  - cd src
-  - gn gen out/Default "--args=import(\"//electron/build/args/%gn_args%.gn\")"
+  - >-
+      %SCCACHE_PATH% -s
+  - gn gen out/Default "--args=import(\"//electron/build/args/%gn_args%.gn\") cc_wrapper=\"%CC_WRAPPER%\" "
  - ninja -C out/Default electron:electron_app
+  - ninja -C out/Default electron:electron_dist_zip
+  - >-
+      %SCCACHE_PATH% -s
 test_script:
  - ninja -C out/Default electron/build/node:headers
  - ps: $env:npm_config_nodedir="$pwd/out/Default/gen/node_headers"
@ -43,3 +49,5 @@ test_script:
 artifacts:
 - path: test-results.xml
  name: test-results.xml
+- path: src/out/Default/dist.zip
+  name: dist.zip
--- a/build/args/release.gn
+++ b/build/args/release.gn
@ -2,6 +2,7 @@ import("all.gn")
 is_component_build = false
 is_component_ffmpeg = true
 is_official_build = true
+strip_debug_info = true

 # TODO(nornagon): linking non-CFI code (nodejs) with CFI code fails at runtime.
 # Once we can build nodejs with CFI flags matching Electron's, remove this.
--- a/build/write_version.py
+++ b/build/write_version.py
@ -0,0 +1,11 @@
+import sys
+
+
+def main(argv):
+  out_file, version = argv
+  with open(out_file, 'w') as f:
+    f.write(version)
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))
--- a/build/zip.py
+++ b/build/zip.py
@ -0,0 +1,59 @@
+#!/usr/bin/env python
+import os
+import subprocess
+import sys
+import zipfile
+
+LINUX_BINARIES_TO_STRIP = [
+  'electron',
+  'libffmpeg.so',
+  'libnode.so'
+]
+
+def strip_binaries(target_cpu, dep):
+  for binary in LINUX_BINARIES_TO_STRIP:
+    if dep.endswith(binary):
+     strip_binary(dep, target_cpu)
+
+def strip_binary(binary_path, target_cpu):
+  if target_cpu == 'arm':
+    strip = 'arm-linux-gnueabihf-strip'
+  elif target_cpu == 'arm64':
+    strip = 'aarch64-linux-gnu-strip'
+  elif target_cpu == 'mips64el':
+    strip = 'mips64el-redhat-linux-strip'
+  else:
+    strip = 'strip'
+  execute([strip, binary_path])
+
+def execute(argv):
+  try:
+    output = subprocess.check_output(argv, stderr=subprocess.STDOUT)
+    return output
+  except subprocess.CalledProcessError as e:
+    print e.output
+    raise e
+
+def main(argv):
+  dist_zip, runtime_deps, target_cpu, target_os = argv
+  dist_files = []
+  with open(runtime_deps) as f:
+    for dep in f.readlines():
+      dep = dep.strip()
+      dist_files += [dep]
+  if sys.platform == 'darwin':
+    mac_zip_results = execute(['zip', '-r', '-y', dist_zip] + dist_files)
+  else:
+    with zipfile.ZipFile(dist_zip, 'w', zipfile.ZIP_DEFLATED) as z:
+      for dep in dist_files:
+        if target_os == 'linux':
+            strip_binaries(target_cpu, dep)
+        if os.path.isdir(dep):
+          for root, dirs, files in os.walk(dep):
+            for file in files:
+              z.write(os.path.join(root, file))
+        else:
+          z.write(dep)
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))
--- a/vsts-gn.yml
+++ b/vsts-gn.yml
@ -33,8 +33,8 @@ phases:
      cd src
      export CHROMIUM_BUILDTOOLS_PATH=`pwd`/buildtools
      export SCCACHE_WRAPPER="`pwd`/libchromiumcontent/tools/sccache/aad2120/mac/sccache"
-      "$SCCACHE_WRAPPER" --start-server --azure_container "$(SCCACHE_AZURE_BLOB_CONTAINER)" --azure_connection "$(SCCACHE_AZURE_CONNECTION_STRING)"
-      "$SCCACHE_WRAPPER" -s
+      export SCCACHE_HELPER="`pwd`/libchromiumcontent/script/sccache"
+      "$SCCACHE_HELPER" --start-server --azure_container "$(SCCACHE_AZURE_BLOB_CONTAINER)" --azure_connection "$(SCCACHE_AZURE_CONNECTION_STRING)"
      echo "##vso[task.setvariable variable=SCCACHE_WRAPPER]$SCCACHE_WRAPPER"
      echo "##vso[task.setvariable variable=CHROMIUM_BUILDTOOLS_PATH]`pwd`/buildtools"
      echo "GN gen for: $GN_CONFIG"
@ -59,12 +59,26 @@ phases:
      (cd electron/spec && npm install)
      ./out/Default/Electron.app/Contents/MacOS/Electron electron/spec --ci --enable-logging
    name: Test
+    condition: and(succeeded(), ne(variables['ELECTRON_RELEASE'], '1'))
+
+  - bash: |
+      cd src
+      ninja -C out/Default electron:electron_dist_zip
+    name: Build_dist_zip
+    condition: and(succeeded(), eq(variables['ELECTRON_RELEASE'], '1'))

  - task: PublishTestResults@2
    displayName: Publish Test Results
    inputs:
      testResultsFiles: '**/test-*.xml'
-    condition: and(always(), eq(variables['MOCHA_FILE'], 'junit/test-results.xml'))
+    condition: and(always(), eq(variables['MOCHA_FILE'], 'junit/test-results.xml'), ne(variables['ELECTRON_RELEASE'], '1'))
+
+  - task: PublishBuildArtifacts@1
+    displayName: Publish Build Artifacts
+    inputs:
+      PathtoPublish: '$(Build.SourcesDirectory)/out/Default/dist.zip'
+      ArtifactName: dist.zip
+    condition: and(succeeded(), eq(variables['ELECTRON_RELEASE'], '1'))

  - bash: |
      export BUILD_URL="${SYSTEM_TEAMFOUNDATIONCOLLECTIONURI}${SYSTEM_TEAMPROJECT}/_build/results?buildId=${BUILD_BUILDID}"