Bug 1573435 - Use toolchain fetches for openh264 tasks. r=nalexander

Differential Revision: https://phabricator.services.mozilla.com/D41737
2019-08-13 17:23:26 +09:00 · 2019-08-13 17:23:26 +09:00 · 8cd0a0666a
--- a/taskcluster/ci/openh264-plugin/kind.yml
+++ b/taskcluster/ci/openh264-plugin/kind.yml
@ -9,7 +9,6 @@ kind-dependencies:

 transforms:
    - taskgraph.transforms.openh264:transforms
-    - taskgraph.transforms.use_toolchains:transforms
    - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms

@ -45,10 +44,11 @@ jobs:
            config:
                - openh264/linux32.py
            tooltool-downloads: public
-        toolchains:
-            - linux64-binutils
-            - linux64-clang
-            - linux64-nasm
+        fetches:
+            toolchain:
+                - linux64-binutils
+                - linux64-clang
+                - linux64-nasm
    linux64/opt:
        attributes:
            build_platform: linux64
@ -68,10 +68,11 @@ jobs:
            config:
                - openh264/linux64.py
            tooltool-downloads: public
-        toolchains:
-            - linux64-binutils
-            - linux64-clang
-            - linux64-nasm
+        fetches:
+            toolchain:
+                - linux64-binutils
+                - linux64-clang
+                - linux64-nasm
    macosx64/opt:
        attributes:
            build_platform: macosx64
@ -91,9 +92,10 @@ jobs:
            config:
                - openh264/macosx64.py
            tooltool-downloads: internal
-        toolchains:
-            - linux64-cctools-port
-            - linux64-clang
+        fetches:
+            toolchain:
+                - linux64-cctools-port
+                - linux64-clang
    win32/opt:
        attributes:
            build_platform: win32
@ -112,8 +114,9 @@ jobs:
            script: mozharness/scripts/openh264_build.py
            config:
                - openh264/win32.py
-        toolchains:
-            - win64-clang-cl
+        fetches:
+            toolchain:
+                - win64-clang-cl
    win64/opt:
        attributes:
            build_platform: win64
@ -132,8 +135,9 @@ jobs:
            script: mozharness/scripts/openh264_build.py
            config:
                - openh264/win64.py
-        toolchains:
-            - win64-clang-cl
+        fetches:
+            toolchain:
+                - win64-clang-cl
    win64-aarch64/opt:
        attributes:
            build_platform: win64-aarch64
@ -152,8 +156,9 @@ jobs:
            script: mozharness/scripts/openh264_build.py
            config:
                - openh264/win64-aarch64.py
-        toolchains:
-            - win64-clang-cl
+        fetches:
+            toolchain:
+                - win64-clang-cl
    android-api-16/opt:
        attributes:
            build_platform: android-arm
@ -177,11 +182,12 @@ jobs:
            config:
                - openh264/android-arm.py
            tooltool-downloads: internal
-        toolchains:
-            - android-ndk-linux
-            - android-sdk-linux
-            - linux64-clang
-            - linux64-nasm
+        fetches:
+            toolchain:
+                - android-ndk-linux
+                - android-sdk-linux
+                - linux64-clang
+                - linux64-nasm
    android-aarch64/opt:
        attributes:
            build_platform: android-aarch64
@ -205,11 +211,12 @@ jobs:
            config:
                - openh264/android-aarch64.py
            tooltool-downloads: internal
-        toolchains:
-            - android-ndk-linux
-            - android-sdk-linux
-            - linux64-clang
-            - linux64-nasm
+        fetches:
+            toolchain:
+                - android-ndk-linux
+                - android-sdk-linux
+                - linux64-clang
+                - linux64-nasm
    android-x86/opt:
        attributes:
            build_platform: android-x86
@ -233,11 +240,12 @@ jobs:
            config:
                - openh264/android-x86.py
            tooltool-downloads: internal
-        toolchains:
-            - android-ndk-linux
-            - android-sdk-linux
-            - linux64-clang
-            - linux64-nasm
+        fetches:
+            toolchain:
+                - android-ndk-linux
+                - android-sdk-linux
+                - linux64-clang
+                - linux64-nasm
    android-x86_64/opt:
        attributes:
            build_platform: android-x86_64
@ -261,8 +269,9 @@ jobs:
            config:
                - openh264/android-x86_64.py
            tooltool-downloads: internal
-        toolchains:
-            - android-ndk-linux
-            - android-sdk-linux
-            - linux64-clang
-            - linux64-nasm
+        fetches:
+            toolchain:
+                - android-ndk-linux
+                - android-sdk-linux
+                - linux64-clang
+                - linux64-nasm
--- a/testing/mozharness/configs/openh264/android-aarch64.py
+++ b/testing/mozharness/configs/openh264/android-aarch64.py
@ -30,8 +30,9 @@ config = {
    'partial_env': {
        'CXXFLAGS': '-stdlib=libstdc++',
        'LDFLAGS': '-stdlib=libstdc++',
-        'PATH': ('%(abs_work_dir)s/src/android-sdk-linux/tools:'
-                 '%(abs_work_dir)s/src/clang/bin:'
-                 '%(abs_work_dir)s/src/nasm:%(PATH)s'),
+        'PATH': ('{MOZ_FETCHES_DIR}/android-sdk-linux/tools:'
+                 '{MOZ_FETCHES_DIR}/clang/bin:'
+                 '{MOZ_FETCHES_DIR}/nasm:%(PATH)s'
+                 .format(MOZ_FETCHES_DIR=os.environ['MOZ_FETCHES_DIR'])),
    },
 }
--- a/testing/mozharness/configs/openh264/android-arm.py
+++ b/testing/mozharness/configs/openh264/android-arm.py
@ -30,8 +30,9 @@ config = {
    'partial_env': {
        'CXXFLAGS': '-stdlib=libstdc++',
        'LDFLAGS': '-stdlib=libstdc++',
-        'PATH': ('%(abs_work_dir)s/src/android-sdk-linux/tools:'
-                 '%(abs_work_dir)s/src/clang/bin:'
-                 '%(abs_work_dir)s/src/nasm:%(PATH)s'),
+        'PATH': ('{MOZ_FETCHES_DIR}/android-sdk-linux/tools:'
+                 '{MOZ_FETCHES_DIR}/clang/bin:'
+                 '{MOZ_FETCHES_DIR}/nasm:%(PATH)s'
+                 .format(MOZ_FETCHES_DIR=os.environ['MOZ_FETCHES_DIR'])),
    },
 }
--- a/testing/mozharness/configs/openh264/android-x86.py
+++ b/testing/mozharness/configs/openh264/android-x86.py
@ -31,8 +31,9 @@ config = {
    'partial_env': {
        'CXXFLAGS': '-stdlib=libstdc++',
        'LDFLAGS': '-stdlib=libstdc++',
-        'PATH': ('%(abs_work_dir)s/src/android-sdk-linux/tools:'
-                 '%(abs_work_dir)s/src/clang/bin:'
-                 '%(abs_work_dir)s/src/nasm:%(PATH)s'),
+        'PATH': ('{MOZ_FETCHES_DIR}/android-sdk-linux/tools:'
+                 '{MOZ_FETCHES_DIR}/clang/bin:'
+                 '{MOZ_FETCHES_DIR}/nasm:%(PATH)s'
+                 .format(MOZ_FETCHES_DIR=os.environ['MOZ_FETCHES_DIR'])),
    },
 }
--- a/testing/mozharness/configs/openh264/android-x86_64.py
+++ b/testing/mozharness/configs/openh264/android-x86_64.py
@ -31,8 +31,9 @@ config = {
    'partial_env': {
        'CXXFLAGS': '-stdlib=libstdc++',
        'LDFLAGS': '-stdlib=libstdc++',
-        'PATH': ('%(abs_work_dir)s/src/android-sdk-linux/tools:'
-                 '%(abs_work_dir)s/src/clang/bin:'
-                 '%(abs_work_dir)s/src/nasm:%(PATH)s'),
+        'PATH': ('{MOZ_FETCHES_DIR}/android-sdk-linux/tools:'
+                 '{MOZ_FETCHES_DIR}/clang/bin:'
+                 '{MOZ_FETCHES_DIR}/nasm:%(PATH)s'
+                 .format(MOZ_FETCHES_DIR=os.environ['MOZ_FETCHES_DIR'])),
    },
 }
--- a/testing/mozharness/configs/openh264/linux32.py
+++ b/testing/mozharness/configs/openh264/linux32.py
@ -19,8 +19,9 @@ config = {
    'avoid_avx2': True,
    'operating_system': 'linux',
    'partial_env': {
-        'PATH': ('%(abs_work_dir)s/src/clang/bin:'
-                 '%(abs_work_dir)s/src/binutils/bin:'
-                 '%(abs_work_dir)s/src/nasm:%(PATH)s'),
+        'PATH': ('{MOZ_FETCHES_DIR}/clang/bin:'
+                 '{MOZ_FETCHES_DIR}/binutils/bin:'
+                 '{MOZ_FETCHES_DIR}/nasm:%(PATH)s'
+                 .format(MOZ_FETCHES_DIR=os.environ['MOZ_FETCHES_DIR'])),
    },
 }
--- a/testing/mozharness/configs/openh264/linux64.py
+++ b/testing/mozharness/configs/openh264/linux64.py
@ -19,8 +19,9 @@ config = {
    'avoid_avx2': True,
    'operating_system': 'linux',
    'partial_env': {
-        'PATH': ('%(abs_work_dir)s/src/clang/bin:'
-                 '%(abs_work_dir)s/src/binutils/bin:'
-                 '%(abs_work_dir)s/src/nasm:%(PATH)s'),
+        'PATH': ('{MOZ_FETCHES_DIR}/clang/bin:'
+                 '{MOZ_FETCHES_DIR}/binutils/bin:'
+                 '{MOZ_FETCHES_DIR}/nasm:%(PATH)s'
+                 .format(MOZ_FETCHES_DIR=os.environ['MOZ_FETCHES_DIR'])),
    },
 }
--- a/testing/mozharness/configs/openh264/macosx64.py
+++ b/testing/mozharness/configs/openh264/macosx64.py
@ -20,14 +20,17 @@ config = {
    'operating_system': 'darwin',
    'partial_env': {
        'CXXFLAGS': ('-target x86_64-apple-darwin '
-                     '-B %(abs_work_dir)s/src/cctools/bin '
+                     '-B {MOZ_FETCHES_DIR}/cctools/bin '
                     '-isysroot %(abs_work_dir)s/src/MacOSX10.11.sdk '
-                     '-mmacosx-version-min=10.11'),
+                     '-mmacosx-version-min=10.11'
+                     .format(MOZ_FETCHES_DIR=os.environ['MOZ_FETCHES_DIR'])),
        'LDFLAGS': ('-target x86_64-apple-darwin '
-                    '-B %(abs_work_dir)s/src/cctools/bin '
+                    '-B {MOZ_FETCHES_DIR}/cctools/bin '
                    '-isysroot %(abs_work_dir)s/src/MacOSX10.11.sdk '
-                    '-mmacosx-version-min=10.11'),
-        'PATH': '%(abs_work_dir)s/src/clang/bin/:%(PATH)s',
+                    '-mmacosx-version-min=10.11'
+                     .format(MOZ_FETCHES_DIR=os.environ['MOZ_FETCHES_DIR'])),
+        'PATH': ('{MOZ_FETCHES_DIR}/clang/bin/:%(PATH)s'
+                 .format(MOZ_FETCHES_DIR=os.environ['MOZ_FETCHES_DIR'])),
    },
    "tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/'],
    "tooltool_url": 'http://taskcluster/tooltool.mozilla-releng.net/',
--- a/testing/mozharness/configs/openh264/win32.py
+++ b/testing/mozharness/configs/openh264/win32.py
@ -19,13 +19,13 @@ config = {
    'arch': 'x86',
    'use_yasm': True,
    'partial_env': {
-        'PATH': ('%(abs_work_dir)s\\src\\clang\\bin\\;'
+        'PATH': ('{MOZ_FETCHES_DIR}\\clang\\bin\\;'
                 '{_VSPATH}\\VC\\bin\\Hostx64\\x64;%(PATH)s'
                 # 32-bit redist here for our dump_syms.exe
                 '{_VSPATH}/VC/redist/x86/Microsoft.VC141.CRT;'
                 '{_VSPATH}/SDK/Redist/ucrt/DLLs/x86;'
                 '{_VSPATH}/DIA SDK/bin'
-        ).format(_VSPATH=VSPATH),
+        ).format(_VSPATH=VSPATH, MOZ_FETCHES_DIR=os.environ['MOZ_FETCHES_DIR']),
        'INCLUDES': (
            '-I{_VSPATH}\\VC\\include '
            '-I{_VSPATH}\\VC\\atlmfc\\include '
--- a/testing/mozharness/configs/openh264/win64-aarch64.py
+++ b/testing/mozharness/configs/openh264/win64-aarch64.py
@ -20,14 +20,14 @@ config = {
    'use_yasm': False,
    'partial_env': {
        'PATH': ('%(abs_work_dir)s\\openh264;'
-                 '%(abs_work_dir)s\\src\\clang\\bin\\;'
+                 '{MOZ_FETCHES_DIR}\\clang\\bin\\;'
                 '{_VSPATH}\\VC\\bin\\Hostx64\\arm64;'
                 '{_VSPATH}\\VC\\bin\\Hostx64\\x64;'
                 # 32-bit redist here for our dump_syms.exe
                 '{_VSPATH}/VC/redist/x86/Microsoft.VC141.CRT;'
                 '{_VSPATH}/SDK/Redist/ucrt/DLLs/x86;'
                 '{_VSPATH}/DIA SDK/bin;%(PATH)s;'
-        ).format(_VSPATH=VSPATH),
+        ).format(_VSPATH=VSPATH, MOZ_FETCHES_DIR=os.environ['MOZ_FETCHES_DIR']),
        'INCLUDES': (
            '-I{_VSPATH}\\VC\\include '
            '-I{_VSPATH}\\VC\\atlmfc\\include '
--- a/testing/mozharness/configs/openh264/win64.py
+++ b/testing/mozharness/configs/openh264/win64.py
@ -19,13 +19,13 @@ config = {
    'arch': 'x64',
    'use_yasm': True,
    'partial_env': {
-        'PATH': ('%(abs_work_dir)s\\src\\clang\\bin\\;'
+        'PATH': ('{MOZ_FETCHES_DIR}\\clang\\bin\\;'
                 '{_VSPATH}\\VC\\bin\\Hostx64\\x64;%(PATH)s;'
                 # 32-bit redist here for our dump_syms.exe
                 '{_VSPATH}/VC/redist/x86/Microsoft.VC141.CRT;'
                 '{_VSPATH}/SDK/Redist/ucrt/DLLs/x86;'
                 '{_VSPATH}/DIA SDK/bin'
-        ).format(_VSPATH=VSPATH),
+        ).format(_VSPATH=VSPATH, MOZ_FETCHES_DIR=os.environ['MOZ_FETCHES_DIR']),
        'INCLUDES': (
            '-I{_VSPATH}\\VC\\include '
            '-I{_VSPATH}\\VC\\atlmfc\\include '
--- a/testing/mozharness/scripts/openh264_build.py
+++ b/testing/mozharness/scripts/openh264_build.py
@ -170,7 +170,6 @@ class OpenH264Build(TransferMixin, VCSScript, TooltoolMixin):
        self.fatal("can't determine platform")

    def query_make_params(self):
-        dirs = self.query_abs_dirs()
        retval = []
        if self.config['debug_build']:
            retval.append('BUILDTYPE=Debug')
@ -194,7 +193,7 @@ class OpenH264Build(TransferMixin, VCSScript, TooltoolMixin):
                    retval.append("ARCH=arm")
                retval.append('TARGET=invalid')
                retval.append('NDKLEVEL=%s' % self.config['min_sdk'])
-                retval.append('NDKROOT=%s/src/android-ndk' % dirs['abs_work_dir'])
+                retval.append('NDKROOT=%s/android-ndk' % os.environ['MOZ_FETCHES_DIR'])
                retval.append('NDK_TOOLCHAIN_VERSION=clang')
            if self.config["operating_system"] == "darwin":
                retval.append('OS=darwin')
@ -281,7 +280,7 @@ class OpenH264Build(TransferMixin, VCSScript, TooltoolMixin):
                # os.symlink is not available on Windows until we switch to
                # Python 3.
                os.system('ln -s %s %s' % (
-                          os.path.join(dirs['abs_work_dir'], 'src', 'clang',
+                          os.path.join(os.environ['MOZ_FETCHES_DIR'], 'clang',
                                       'bin', 'clang.exe'),
                          os.path.join(openh264_dir, 'cpp')))
            return 0