Bug 1380893 - Add CI for plain builds; r=mshal

This commit adds CI tasks to perform "plain" builds. These builds use
the same toolchains used by official builds. But that's about it. The
mozconfig changes are minimal and only set up paths to toolchain
artifacts. sccache is not used.

The main goal of these builds is to have a "reference" build that
matches an out-of-the-box build environment as much as possible. We want
this mainly so we have timing and behavior information that matches what
developers are doing.

The Windows/generic Taskcluster worker doesn't like it when you specify
an artifact directory that doesn't exist. So we needed to add a
mozharness step to ensure UPLOAD_PATH exists to prevent those tasks from
erroring.

I'm not super thrilled about using mozharness here. We definitely don't
really need mozharness. But the main thing it is providing that we care
about is the Perfherder metrics data. I don't feel like scope bloating
to move that out of mozharness at this time.

I only implemented Linux64 and Windows64 because I'm not convinced
coverage on 32-bit build variations is useful. Tasks only run on trunk
because they are informational only and we don't need to waste resources
running these on release repos and on Try. They are tier 2 because they
aren't critical to shipping Firefox.

MozReview-Commit-ID: Gl6hGYbFX9b

--HG--
extra : rebase_source : 05360d2f6e5dbfed5543726a1be4b0e5d00e0b3d

browser/config/mozconfigs/linux64/plain-debug

@@ -0,0 +1,3 @@
+. "$topsrcdir/browser/config/mozconfigs/linux64/plain-opt"
+
+ac_add_options --enable-debug

browser/config/mozconfigs/linux64/plain-opt

@@ -0,0 +1,11 @@
+export TOOLTOOL_DIR="$topsrcdir"
+export LLVM_CONFIG="${TOOLTOOL_DIR}/clang/bin/llvm-config"
+
+CARGO="${TOOLTOOL_DIR}/rustc/bin/cargo"
+RUSTC="${TOOLTOOL_DIR}/rustc/bin/rustc"
+
+CC="${TOOLTOOL_DIR}/gcc/bin/gcc"
+CXX="${TOOLTOOL_DIR}/gcc/bin/g++"
+
+mk_add_options "export PATH=${TOOLTOOL_DIR}/gcc/bin:${PATH}"
+mk_add_options "export LD_LIBRARY_PATH=${TOOLTOOL_DIR}/gcc/lib64:${TOOLTOOL_DIR}/gcc/lib32:${TOOLTOOL_DIR}/gcc/lib"

browser/config/mozconfigs/win64/plain-debug

@@ -0,0 +1,3 @@
+. $topsrcdir/browser/config/mozconfigs/win64/plain-opt
+
+ac_add_options --enable-debug

browser/config/mozconfigs/win64/plain-opt

@@ -0,0 +1,11 @@
+TOOLTOOL_DIR=${topsrcdir}
+export MAKECAB=$TOOLTOOL_DIR/makecab.exe
+export LLVM_CONFIG="${TOOLTOOL_DIR}/clang/bin/llvm-config"
+
+RUSTC="${TOOLTOOL_DIR}/rustc/bin/rustc"
+CARGO="${TOOLTOOL_DIR}/rustc/bin/cargo"
+
+ac_add_options --target=x86_64-pc-mingw32
+ac_add_options --host=x86_64-pc-mingw32
+
+. $topsrcdir/build/win64/mozconfig.vs-latest

taskcluster/ci/build/linux.yml

@@ -26,6 +26,34 @@ linux64/opt:
         - linux64-rust
         - linux64-sccache
 
+linux64-plain/opt:
+    description: "Linux64 Opt Plain"
+    index:
+        product: firefox
+        job-name: linux64-plain-opt
+    treeherder:
+        platform: linux64/opt
+        symbol: Bp
+        tier: 2
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        max-run-time: 3600
+        env:
+            PERFHERDER_EXTRA_OPTIONS: plain
+    run:
+        using: mozharness
+        config:
+            - builds/releng_base_firefox.py
+            - builds/releng_base_linux_64_builds.py
+            - builds/releng_sub_linux_configs/64_plain_opt.py
+        script: mozharness/scripts/fx_desktop_build.py
+        tooltool-downloads: public
+    run-on-projects: [trunk]
+    toolchains:
+        - linux64-clang
+        - linux64-gcc
+        - linux64-rust
+
 linux64-dmd/opt:
     description: "Linux64 DMD Opt"
     index:
@@ -144,6 +172,34 @@ linux64/debug:
         - linux64-sccache
         - linux64-rust
 
+linux64-plain/debug:
+    description: "Linux64 Debug Plain"
+    index:
+        product: firefox
+        job-name: linux64-plain-debug
+    treeherder:
+        platform: linux64/debug
+        symbol: Bp
+        tier: 2
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        max-run-time: 3600
+        env:
+            PERFHERDER_EXTRA_OPTIONS: plain
+    run:
+        using: mozharness
+        config:
+            - builds/releng_base_firefox.py
+            - builds/releng_base_linux_64_builds.py
+            - builds/releng_sub_linux_configs/64_plain_debug.py
+        script: mozharness/scripts/fx_desktop_build.py
+        tooltool-downloads: public
+    run-on-projects: [trunk]
+    toolchains:
+        - linux64-clang
+        - linux64-gcc
+        - linux64-rust
+
 linux64-devedition-nightly/opt:
     description: "Linux64 devedition Nightly"
     attributes:

taskcluster/ci/build/windows.yml

@@ -141,6 +141,35 @@ win64/debug:
         - win64-rust
         - win64-sccache
 
+win64-plain/debug:
+    description: "Win64 Debug Plain"
+    index:
+        product: firefox
+        job-name: win64-plain-debug
+    treeherder:
+        platform: windows2012-64/debug
+        symbol: Bp
+        tier: 2
+    worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+    worker:
+        max-run-time: 7200
+        env:
+            PERFHERDER_EXTRA_OPTIONS: plain
+            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/releng.manifest"
+    run:
+        using: mozharness
+        options: [append-env-variables-from-configs]
+        script: mozharness/scripts/fx_desktop_build.py
+        config:
+            - builds/releng_base_firefox.py
+            - builds/taskcluster_base_windows.py
+            - builds/taskcluster_base_win64.py
+            - builds/taskcluster_sub_win64/plain_debug.py
+    run-on-projects: [trunk]
+    toolchains:
+        - win64-clang-cl
+        - win64-rust
+
 win64/opt:
     description: "Win64 Opt"
     index:
@@ -169,6 +198,35 @@ win64/opt:
         - win64-rust
         - win64-sccache
 
+win64-plain/opt:
+    description: "Win64 Opt Plain"
+    index:
+        product: firefox
+        job-name: win64-plain-opt
+    treeherder:
+        platform: windows2012-64/opt
+        symbol: Bp
+        tier: 2
+    worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+    worker:
+        max-run-time: 7200
+        env:
+            PERFHERDER_EXTRA_OPTIONS: plain
+            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/releng.manifest"
+    run:
+        using: mozharness
+        options: [append-env-variables-from-configs]
+        script: mozharness/scripts/fx_desktop_build.py
+        config:
+            - builds/releng_base_firefox.py
+            - builds/taskcluster_base_windows.py
+            - builds/taskcluster_base_win64.py
+            - builds/taskcluster_sub_win64/plain_opt.py
+    run-on-projects: [trunk]
+    toolchains:
+        - win64-clang-cl
+        - win64-rust
+
 win64-dmd/opt:
     description: "Win64 DMD Opt"
     index:

testing/mozharness/configs/builds/releng_sub_linux_configs/64_plain_debug.py

@@ -0,0 +1,7 @@
+config = {
+    'default_actions': [
+        'build',
+    ],
+    'disable_package_metrics': True,
+    'mozconfig_variant': 'plain-debug',
+}

testing/mozharness/configs/builds/releng_sub_linux_configs/64_plain_opt.py

@@ -0,0 +1,7 @@
+config = {
+    'default_actions': [
+        'build',
+    ],
+    'disable_package_metrics': True,
+    'mozconfig_variant': 'plain-opt',
+}

testing/mozharness/configs/builds/taskcluster_sub_win64/plain_debug.py

@@ -0,0 +1,9 @@
+config = {
+    'default_actions': [
+        'build',
+        'ensure-upload-path',
+    ],
+    'disable_package_metrics': True,
+    'mozconfig_variant': 'plain-debug',
+    'stage_platform': 'win64',
+}

testing/mozharness/configs/builds/taskcluster_sub_win64/plain_opt.py

@@ -0,0 +1,9 @@
+config = {
+    'default_actions': [
+        'build',
+        'ensure-upload-path',
+    ],
+    'disable_package_metrics': True,
+    'mozconfig_variant': 'plain-opt',
+    'stage_platform': 'win64',
+}

testing/mozharness/mozharness/mozilla/building/buildbase.py

@@ -1780,7 +1780,14 @@ or run without that action (ie: --no-{action})"
             self.fatal("'mach valgrind-test' did not run successfully. Please check "
                        "log for errors.")
 
+    def ensure_upload_path(self):
+        env = self.query_mach_build_env()
 
+        # Some Taskcluster workers don't like it if an artifacts directory
+        # is defined but no artifacts are uploaded. Guard against this by always
+        # ensuring the artifacts directory exists.
+        if 'UPLOAD_PATH' in env and not os.path.exists(env['UPLOAD_PATH']):
+            os.makedirs(env['UPLOAD_PATH'])
 
     def _post_fatal(self, message=None, exit_code=None):
         if not self.return_code:  # only overwrite return_code if it's 0

testing/mozharness/scripts/fx_desktop_build.py

@@ -41,6 +41,7 @@ class FxDesktopBuild(BuildScript, TryToolsMixin, object):
                 'multi-l10n',
                 'package-source',
                 'update',
+                'ensure-upload-path',
             ],
             'require_config_file': True,
             # Default configuration