Fix #3608: Remove code refs to TaskCluster

.github/workflows/macOS-amd64.yml

@@ -133,8 +133,8 @@ jobs:
           echo "Moving ${bits} to LDC93S1.wav"
           mv data/smoke_test/LDC93S1_pcms16le_1_${bits}.wav data/smoke_test/LDC93S1.wav
 
-          ./bin/run-tc-ldc93s1_new.sh 249 ${bits}
-          ./bin/run-tc-ldc93s1_tflite.sh ${bits}
+          ./bin/run-ci-ldc93s1_new.sh 249 ${bits}
+          ./bin/run-ci-ldc93s1_tflite.sh ${bits}
       - run: |
           curl -vsSL https://github.com/mozilla/DeepSpeech/releases/download/v0.9.3/macOS.amd64.convert_graphdef_memmapped_format.xz | xz -d > /tmp/convert_graphdef_memmapped_format
           chmod +x /tmp/convert_graphdef_memmapped_format

.readthedocs.yml

@@ -14,4 +14,4 @@ sphinx:
 python:
   version: 3.7
   install:
-    - requirements: taskcluster/docs-requirements.txt
+    - requirements: ci_scripts/docs-requirements.txt

README.rst

@@ -7,9 +7,17 @@ Project DeepSpeech
    :alt: Documentation
 
 
-.. image:: https://community-tc.services.mozilla.com/api/github/v1/repository/mozilla/DeepSpeech/master/badge.svg
-   :target: https://community-tc.services.mozilla.com/api/github/v1/repository/mozilla/DeepSpeech/master/latest
-   :alt: Task Status
+.. image:: https://github.com/mozilla/DeepSpeech/actions/workflows/macOS-amd64.yml/badge.svg
+   :target: https://github.com/mozilla/DeepSpeech/actions/workflows/macOS-amd64.yml
+   :alt: macOS builds
+
+.. image:: https://github.com/mozilla/DeepSpeech/actions/workflows/lint.yml/badge.svg
+   :target: https://github.com/mozilla/DeepSpeech/actions/workflows/lint.yml
+   :alt: Linters
+
+.. image:: https://github.com/mozilla/DeepSpeech/actions/workflows/docker.yml/badge.svg
+   :target: https://github.com/mozilla/DeepSpeech/actions/workflows/docker.yml
+   :alt: Docker Images
 
 
 DeepSpeech is an open-source Speech-To-Text engine, using a model trained by machine learning techniques based on `Baidu's Deep Speech research paper <https://arxiv.org/abs/1412.5567>`_. Project DeepSpeech uses Google's `TensorFlow <https://www.tensorflow.org/>`_ to make the implementation easier.

bin/run-ci-transfer.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 # This bash script is for running minimum working examples
 # of transfer learning for continuous integration tests
-# to be run on Taskcluster.
+# to be run on CI.
 set -xe
 
 ru_dir="./data/smoke_test/russian_sample_data"

doc/Makefile

@@ -17,7 +17,7 @@ help:
 .PHONY: help pip3 Makefile
 
 pip3:
-	$(PIP_INSTALL) -r ../taskcluster/docs-requirements.txt
+	$(PIP_INSTALL) -r ../ci_scripts/docs-requirements.txt
 
 submodule:
 	git submodule update --init --remote -- ../doc/examples

doc/conf.py

@@ -27,8 +27,7 @@ sys.path.insert(0, os.path.abspath('../'))
 autodoc_mock_imports = ['deepspeech']
 
 # This is in fact only relevant on ReadTheDocs, but we want to run the same way
-# on our CI as in RTD to avoid regressions on RTD that we would not catch on
-# TaskCluster
+# on our CI as in RTD to avoid regressions on RTD that we would not catch on CI
 import subprocess
 parent = subprocess.check_output("cd ../ && pwd", shell=True).decode().strip()
 os.environ["PATH"] = os.path.join(parent, 'node_modules', '.bin') + ':' + os.environ["PATH"]

native_client/definitions.mk

@@ -209,11 +209,11 @@ endef
 SWIG_DIST_URL ?=
 ifeq ($(SWIG_DIST_URL),)
 ifeq ($(findstring Linux,$(OS)),Linux)
-SWIG_DIST_URL := "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.linux.amd64.fec7d5d3179833e37759ffc6532f86344982e26a.0/artifacts/public/ds-swig.tar.gz"
+SWIG_DIST_URL := "https://github.com/mozilla/DeepSpeech/releases/download/v0.9.3/ds-swig.linux.amd64.tar.gz"
 else ifeq ($(findstring Darwin,$(OS)),Darwin)
-SWIG_DIST_URL := "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.darwin.amd64.fec7d5d3179833e37759ffc6532f86344982e26a.0/artifacts/public/ds-swig.tar.gz"
+SWIG_DIST_URL := "https://github.com/mozilla/DeepSpeech/releases/download/v0.9.3/ds-swig.darwin.amd64.tar.gz"
 else ifeq ($(findstring _NT,$(OS)),_NT)
-SWIG_DIST_URL := "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.win.amd64.fec7d5d3179833e37759ffc6532f86344982e26a.0/artifacts/public/ds-swig.tar.gz"
+SWIG_DIST_URL := "https://github.com/mozilla/DeepSpeech/releases/download/v0.9.3/ds-swig.win.amd64.tar.gz"
 else
 $(error There is no prebuilt SWIG available for your platform. Please produce one and set SWIG_DIST_URL.)
 endif # findstring()

native_client/javascript/package.json.in

@@ -29,7 +29,7 @@
         "module_path" : "./lib/binding/v{version}/{platform}-{arch}/{node_abi}/",
         "remote_path" : "./v{version}/{configuration}/",
         "package_name": "{module_name}-v{version}-{node_abi}-{platform}-{arch}.tar.gz",
-        "host"        : "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.v1.0.0-warpctc.arm/artifacts/public/"
+        "host"        : "https://github.com/mozilla/DeepSpeech/releases"
     },
     "dependencies"  : {
       "node-pre-gyp": "0.15.x",

native_client/swift/deepspeech-ios.podspec

@@ -12,7 +12,7 @@ Pod::Spec.new do |s|
   s.platforms    = { :ios => "9.0" }
   s.source       = { :git => "https://github.com/mozilla/DeepSpeech.git", :tag => "v#{s.version}" }
 
-  # Assuming taskcluster build location. Depending on your Xcode setup, this might be in
+  # Assuming CI build location. Depending on your Xcode setup, this might be in
   # build/Release-iphoneos/deepspeech_ios.framework instead.
   s.vendored_frameworks = "native_client/swift/DerivedData/Build/Products/Release-iphoneos/deepspeech_ios.framework"
   s.source_files = "native_client/swift/deepspeech_ios/**/*.{h,m,mm,swift}"

setup.py

@@ -7,43 +7,6 @@ from pkg_resources import parse_version
 from setuptools import find_packages, setup
 
 
-def get_tc_decoder_pkg_url(version, artifacts_root):
-    assert artifacts_root
-
-    ds_version = parse_version(version)
-    branch = "v{}".format(version)
-
-    plat = platform.system().lower()
-    arch = platform.machine().lower()
-
-    if plat == 'linux' and arch == 'x86_64':
-        plat = 'manylinux1'
-
-    if plat == 'darwin':
-        plat = 'macosx_10_10'
-
-    if plat == 'windows':
-        plat = 'win'
-
-    # ABI does not contain m / mu anymore after Python 3.8
-    if sys.version_info.major == 3 and sys.version_info.minor >= 8:
-        m_or_mu = ''
-    else:
-        is_ucs2 = sys.maxunicode < 0x10ffff
-        m_or_mu = 'mu' if is_ucs2 else 'm'
-
-    pyver = ''.join(str(i) for i in sys.version_info[0:2])
-
-    return 'ds_ctcdecoder @ {artifacts_root}/ds_ctcdecoder-{ds_version}-cp{pyver}-cp{pyver}{m_or_mu}-{platform}_{arch}.whl'.format(
-        artifacts_root=artifacts_root,
-        ds_version=ds_version,
-        pyver=pyver,
-        m_or_mu=m_or_mu,
-        platform=plat,
-        arch=arch,
-    )
-
-
 def main():
     version_file = Path(__file__).parent / 'VERSION'
     with open(str(version_file)) as fin:
@@ -80,15 +43,19 @@ def main():
         'horovod[tensorflow] == 0.21.3'
     ]
 
+    # TODO: FIXME: This is likely not needed anymore given the way TC and
+    # GitHub Actions artifacts differs in how we can download them.
+    """
     # Due to pip craziness environment variables are the only consistent way to
     # get options into this script when doing `pip install`.
-    tc_decoder_artifacts_root = os.environ.get('DECODER_ARTIFACTS_ROOT', '')
-    if tc_decoder_artifacts_root:
-        # We're running inside the TaskCluster environment, override the decoder
+    ci_decoder_artifacts_root = os.environ.get('DECODER_ARTIFACTS_ROOT', '')
+    if ci_decoder_artifacts_root:
+        # We're running inside the CI environment, override the decoder
         # package URL with the one we just built.
-        decoder_pkg_url = get_tc_decoder_pkg_url(version, tc_decoder_artifacts_root)
+        decoder_pkg_url = get_ci_decoder_pkg_url(version, ci_decoder_artifacts_root)
         install_requires = install_requires_base + [decoder_pkg_url]
-    elif os.environ.get('DS_NODECODER', ''):
+    """
+    if os.environ.get('DS_NODECODER', ''):
         install_requires = install_requires_base
     else:
         install_requires = install_requires_base + decoder_pypi_dep

training/deepspeech_training/util/taskcluster.py

@@ -1,139 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-from __future__ import print_function, absolute_import, division
-
-import argparse
-import errno
-import gzip
-import os
-import platform
-import six.moves.urllib as urllib
-import stat
-import subprocess
-import sys
-
-from pkg_resources import parse_version
-
-
-DEFAULT_SCHEMES = {
-    'deepspeech': 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.deepspeech.native_client.%(branch_name)s.%(arch_string)s/artifacts/public/%(artifact_name)s',
-    'tensorflow': 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.%(branch_name)s.%(arch_string)s/artifacts/public/%(artifact_name)s'
-}
-
-TASKCLUSTER_SCHEME = os.getenv('TASKCLUSTER_SCHEME', DEFAULT_SCHEMES['deepspeech'])
-
-def get_tc_url(arch_string, artifact_name='native_client.tar.xz', branch_name='master'):
-    assert arch_string is not None
-    assert artifact_name is not None
-    assert artifact_name
-    assert branch_name is not None
-    assert branch_name
-
-    return TASKCLUSTER_SCHEME % {'arch_string': arch_string, 'artifact_name': artifact_name, 'branch_name': branch_name}
-
-def maybe_download_tc(target_dir, tc_url, progress=True):
-    def report_progress(count, block_size, total_size):
-        percent = (count * block_size * 100) // total_size
-        sys.stdout.write("\rDownloading: %d%%" % percent)
-        sys.stdout.flush()
-
-        if percent >= 100:
-            print('\n')
-
-    assert target_dir is not None
-
-    target_dir = os.path.abspath(target_dir)
-    try:
-        os.makedirs(target_dir)
-    except OSError as e:
-        if e.errno != errno.EEXIST:
-            raise e
-    assert os.path.isdir(os.path.dirname(target_dir))
-
-    tc_filename = os.path.basename(tc_url)
-    target_file = os.path.join(target_dir, tc_filename)
-    is_gzip = False
-    if not os.path.isfile(target_file):
-        print('Downloading %s ...' % tc_url)
-        _, headers = urllib.request.urlretrieve(tc_url, target_file, reporthook=(report_progress if progress else None))
-        is_gzip = headers.get('Content-Encoding') == 'gzip'
-    else:
-        print('File already exists: %s' % target_file)
-
-    if is_gzip:
-        with open(target_file, "r+b") as frw:
-            decompressed = gzip.decompress(frw.read())
-            frw.seek(0)
-            frw.write(decompressed)
-            frw.truncate()
-
-    return target_file
-
-def maybe_download_tc_bin(**kwargs):
-    final_file = maybe_download_tc(kwargs['target_dir'], kwargs['tc_url'], kwargs['progress'])
-    final_stat = os.stat(final_file)
-    os.chmod(final_file, final_stat.st_mode | stat.S_IEXEC)
-
-def read(fname):
-    return open(os.path.join(os.path.dirname(__file__), fname)).read()
-
-def main():
-    parser = argparse.ArgumentParser(description='Tooling to ease downloading of components from TaskCluster.')
-    parser.add_argument('--target', required=False,
-                        help='Where to put the native client binary files')
-    parser.add_argument('--arch', required=False,
-                        help='Which architecture to download binaries for. "arm" for ARM 7 (32-bit), "arm64" for ARM64, "gpu" for CUDA enabled x86_64 binaries, "cpu" for CPU-only x86_64 binaries, "osx" for CPU-only x86_64 OSX binaries. Optional ("cpu" by default)')
-    parser.add_argument('--artifact', required=False,
-                        default='native_client.tar.xz',
-                        help='Name of the artifact to download. Defaults to "native_client.tar.xz"')
-    parser.add_argument('--source', required=False, default=None,
-                        help='Name of the TaskCluster scheme to use.')
-    parser.add_argument('--branch', required=False,
-                        help='Branch name to use. Defaulting to current content of VERSION file.')
-
-    args = parser.parse_args()
-
-    if not args.target and not args.decoder:
-        print('Pass either --target or --decoder.')
-        sys.exit(1)
-
-    is_arm = 'arm' in platform.machine()
-    is_mac = 'darwin' in sys.platform
-    is_64bit = sys.maxsize > (2**31 - 1)
-    is_ucs2 = sys.maxunicode < 0x10ffff
-
-    if not args.arch:
-        if is_arm:
-            args.arch = 'arm64' if is_64bit else 'arm'
-        elif is_mac:
-            args.arch = 'osx'
-        else:
-            args.arch = 'cpu'
-
-    if not args.branch:
-        version_string = read('../VERSION').strip()
-        ds_version = parse_version(version_string)
-        args.branch = "v{}".format(version_string)
-    else:
-        ds_version = parse_version(args.branch)
-
-    if args.source is not None:
-        if args.source in DEFAULT_SCHEMES:
-            global TASKCLUSTER_SCHEME
-            TASKCLUSTER_SCHEME = DEFAULT_SCHEMES[args.source]
-        else:
-            print('No such scheme: %s' % args.source)
-            sys.exit(1)
-
-    maybe_download_tc(target_dir=args.target, tc_url=get_tc_url(args.arch, args.artifact, args.branch))
-
-    if args.artifact == "convert_graphdef_memmapped_format":
-        convert_graph_file = os.path.join(args.target, args.artifact)
-        final_stat = os.stat(convert_graph_file)
-        os.chmod(convert_graph_file, final_stat.st_mode | stat.S_IEXEC)
-
-    if '.tar.' in args.artifact:
-        subprocess.check_call(['tar', 'xvf', os.path.join(args.target, args.artifact), '-C', args.target])
-
-if __name__ == '__main__':
-    main()

util/taskcluster.py

@@ -1,12 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-from __future__ import absolute_import, division, print_function
-
-if __name__ == '__main__':
-    try:
-        from deepspeech_training.util import taskcluster as dsu_taskcluster
-    except ImportError:
-        print('Training package is not installed. See training documentation.')
-        raise
-
-    dsu_taskcluster.main()