Merge pull request #3237 from lissyx/rename-training-package

Rename deepspeech_training package

DeepSpeech.py

@@ -4,7 +4,7 @@ from __future__ import absolute_import, division, print_function
 
 if __name__ == '__main__':
     try:
-        from deepspeech_training import train as ds_train
+        from mozilla_voice_stt_training import train as ds_train
     except ImportError:
         print('Training package is not installed. See training documentation.')
         raise

GRAPH_VERSION

@@ -1 +1 @@
-training/deepspeech_training/GRAPH_VERSION
+training/mozilla_voice_stt_training/GRAPH_VERSION

VERSION

@@ -1 +1 @@
-training/deepspeech_training/VERSION
+training/mozilla_voice_stt_training/VERSION

bin/compare_samples.py

@@ -5,8 +5,8 @@ Tool for comparing two wav samples
 import sys
 import argparse
 
-from deepspeech_training.util.audio import AUDIO_TYPE_NP, mean_dbfs
-from deepspeech_training.util.sample_collections import load_sample
+from mozilla_voice_stt_training.util.audio import AUDIO_TYPE_NP, mean_dbfs
+from mozilla_voice_stt_training.util.sample_collections import load_sample
 
 
 def fail(message):

bin/data_set_tool.py

@@ -8,19 +8,19 @@ import argparse
 import progressbar
 from pathlib import Path
 
-from deepspeech_training.util.audio import (
+from mozilla_voice_stt_training.util.audio import (
     AUDIO_TYPE_PCM,
     AUDIO_TYPE_OPUS,
     AUDIO_TYPE_WAV,
     change_audio_types,
 )
-from deepspeech_training.util.downloader import SIMPLE_BAR
-from deepspeech_training.util.sample_collections import (
+from mozilla_voice_stt_training.util.downloader import SIMPLE_BAR
+from mozilla_voice_stt_training.util.sample_collections import (
     CSVWriter,
     DirectSDBWriter,
     samples_from_sources,
 )
-from deepspeech_training.util.augmentations import (
+from mozilla_voice_stt_training.util.augmentations import (
     parse_augmentations,
     apply_sample_augmentations,
     SampleAugmentation

bin/import_aidatatang.py

@@ -5,7 +5,7 @@ import tarfile
 
 import pandas
 
-from deepspeech_training.util.importers import get_importers_parser
+from mozilla_voice_stt_training.util.importers import get_importers_parser
 
 COLUMN_NAMES = ["wav_filename", "wav_filesize", "transcript"]
 

bin/import_aishell.py

@@ -5,7 +5,7 @@ import tarfile
 
 import pandas
 
-from deepspeech_training.util.importers import get_importers_parser
+from mozilla_voice_stt_training.util.importers import get_importers_parser
 
 COLUMNNAMES = ["wav_filename", "wav_filesize", "transcript"]
 

bin/import_cv.py

@@ -9,13 +9,13 @@ from multiprocessing import Pool
 import progressbar
 import sox
 
-from deepspeech_training.util.downloader import SIMPLE_BAR, maybe_download
-from deepspeech_training.util.importers import (
+from mozilla_voice_stt_training.util.downloader import SIMPLE_BAR, maybe_download
+from mozilla_voice_stt_training.util.importers import (
     get_counter,
     get_imported_samples,
     print_import_report,
 )
-from deepspeech_training.util.importers import validate_label_eng as validate_label
+from mozilla_voice_stt_training.util.importers import validate_label_eng as validate_label
 
 FIELDNAMES = ["wav_filename", "wav_filesize", "transcript"]
 SAMPLE_RATE = 16000

bin/import_cv2.py

@@ -15,8 +15,8 @@ from multiprocessing import Pool
 import progressbar
 import sox
 
-from deepspeech_training.util.downloader import SIMPLE_BAR
-from deepspeech_training.util.importers import (
+from mozilla_voice_stt_training.util.downloader import SIMPLE_BAR
+from mozilla_voice_stt_training.util.importers import (
     get_counter,
     get_imported_samples,
     get_importers_parser,

bin/import_fisher.py

@@ -10,7 +10,7 @@ import librosa
 import pandas
 import soundfile  # <= Has an external dependency on libsndfile
 
-from deepspeech_training.util.importers import validate_label_eng as validate_label
+from mozilla_voice_stt_training.util.importers import validate_label_eng as validate_label
 
 # Prerequisite: Having the sph2pipe tool in your PATH:
 # https://www.ldc.upenn.edu/language-resources/tools/sphere-conversion-tools

bin/import_freestmandarin.py

@@ -6,7 +6,7 @@ import tarfile
 import numpy as np
 import pandas
 
-from deepspeech_training.util.importers import get_importers_parser
+from mozilla_voice_stt_training.util.importers import get_importers_parser
 
 COLUMN_NAMES = ["wav_filename", "wav_filesize", "transcript"]
 

bin/import_gram_vaani.py

@@ -12,7 +12,7 @@ import pandas as pd
 from sox import Transformer
 
 import swifter
-from deepspeech_training.util.importers import get_importers_parser, get_validate_label
+from mozilla_voice_stt_training.util.importers import get_importers_parser, get_validate_label
 
 __version__ = "0.1.0"
 _logger = logging.getLogger(__name__)

bin/import_ldc93s1.py

@@ -4,7 +4,7 @@ import sys
 
 import pandas
 
-from deepspeech_training.util.downloader import maybe_download
+from mozilla_voice_stt_training.util.downloader import maybe_download
 
 
 def _download_and_preprocess_data(data_dir):

bin/import_librivox.py

@@ -12,7 +12,7 @@ import progressbar
 from sox import Transformer
 from tensorflow.python.platform import gfile
 
-from deepspeech_training.util.downloader import maybe_download
+from mozilla_voice_stt_training.util.downloader import maybe_download
 
 SAMPLE_RATE = 16000
 

bin/import_lingua_libre.py

@@ -12,8 +12,8 @@ from multiprocessing import Pool
 import progressbar
 import sox
 
-from deepspeech_training.util.downloader import SIMPLE_BAR, maybe_download
-from deepspeech_training.util.importers import (
+from mozilla_voice_stt_training.util.downloader import SIMPLE_BAR, maybe_download
+from mozilla_voice_stt_training.util.importers import (
     get_counter,
     get_imported_samples,
     get_importers_parser,

bin/import_m-ailabs.py

@@ -10,8 +10,8 @@ from multiprocessing import Pool
 
 import progressbar
 
-from deepspeech_training.util.downloader import SIMPLE_BAR, maybe_download
-from deepspeech_training.util.importers import (
+from mozilla_voice_stt_training.util.downloader import SIMPLE_BAR, maybe_download
+from mozilla_voice_stt_training.util.importers import (
     get_counter,
     get_imported_samples,
     get_importers_parser,

bin/import_magicdata.py

@@ -6,7 +6,7 @@ import wave
 
 import pandas
 
-from deepspeech_training.util.importers import get_importers_parser
+from mozilla_voice_stt_training.util.importers import get_importers_parser
 
 COLUMN_NAMES = ["wav_filename", "wav_filesize", "transcript"]
 

bin/import_primewords.py

@@ -7,7 +7,7 @@ import tarfile
 import numpy as np
 import pandas
 
-from deepspeech_training.util.importers import get_importers_parser
+from mozilla_voice_stt_training.util.importers import get_importers_parser
 
 COLUMN_NAMES = ["wav_filename", "wav_filesize", "transcript"]
 

bin/import_slr57.py

@@ -9,8 +9,8 @@ from multiprocessing import Pool
 
 import progressbar
 
-from deepspeech_training.util.downloader import SIMPLE_BAR, maybe_download
-from deepspeech_training.util.importers import (
+from mozilla_voice_stt_training.util.downloader import SIMPLE_BAR, maybe_download
+from mozilla_voice_stt_training.util.importers import (
     get_counter,
     get_imported_samples,
     get_importers_parser,

bin/import_swb.py

@@ -16,7 +16,7 @@ import pandas
 import requests
 import soundfile  # <= Has an external dependency on libsndfile
 
-from deepspeech_training.util.importers import validate_label_eng as validate_label
+from mozilla_voice_stt_training.util.importers import validate_label_eng as validate_label
 
 # ARCHIVE_NAME refers to ISIP alignments from 01/29/03
 ARCHIVE_NAME = "switchboard_word_alignments.tar.gz"

bin/import_swc.py

@@ -22,8 +22,8 @@ from multiprocessing.pool import ThreadPool
 import progressbar
 import sox
 
-from deepspeech_training.util.downloader import SIMPLE_BAR, maybe_download
-from deepspeech_training.util.importers import validate_label_eng as validate_label
+from mozilla_voice_stt_training.util.downloader import SIMPLE_BAR, maybe_download
+from mozilla_voice_stt_training.util.importers import validate_label_eng as validate_label
 from ds_ctcdecoder import Alphabet
 
 SWC_URL = "https://www2.informatik.uni-hamburg.de/nats/pub/SWC/SWC_{language}.tar"

bin/import_ted.py

@@ -10,8 +10,8 @@ import pandas
 from sox import Transformer
 from tensorflow.python.platform import gfile
 
-from deepspeech_training.util.downloader import maybe_download
-from deepspeech_training.util.stm import parse_stm_file
+from mozilla_voice_stt_training.util.downloader import maybe_download
+from mozilla_voice_stt_training.util.stm import parse_stm_file
 
 
 def _download_and_preprocess_data(data_dir):

bin/import_ts.py

@@ -10,8 +10,8 @@ import progressbar
 import sox
 
 import unidecode
-from deepspeech_training.util.downloader import SIMPLE_BAR, maybe_download
-from deepspeech_training.util.importers import (
+from mozilla_voice_stt_training.util.downloader import SIMPLE_BAR, maybe_download
+from mozilla_voice_stt_training.util.importers import (
     get_counter,
     get_imported_samples,
     get_importers_parser,

bin/import_tuda.py

@@ -14,8 +14,8 @@ from collections import Counter
 
 import progressbar
 
-from deepspeech_training.util.downloader import SIMPLE_BAR, maybe_download
-from deepspeech_training.util.importers import validate_label_eng as validate_label
+from mozilla_voice_stt_training.util.downloader import SIMPLE_BAR, maybe_download
+from mozilla_voice_stt_training.util.importers import validate_label_eng as validate_label
 from ds_ctcdecoder import Alphabet
 
 TUDA_VERSION = "v2"

bin/import_vctk.py

@@ -11,8 +11,8 @@ from zipfile import ZipFile
 import librosa
 import progressbar
 
-from deepspeech_training.util.downloader import SIMPLE_BAR, maybe_download
-from deepspeech_training.util.importers import (
+from mozilla_voice_stt_training.util.downloader import SIMPLE_BAR, maybe_download
+from mozilla_voice_stt_training.util.importers import (
     get_counter,
     get_imported_samples,
     print_import_report,

bin/import_voxforge.py

@@ -13,7 +13,7 @@ from os import makedirs, path
 import pandas
 from bs4 import BeautifulSoup
 from tensorflow.python.platform import gfile
-from deepspeech_training.util.downloader import maybe_download
+from mozilla_voice_stt_training.util.downloader import maybe_download
 
 """The number of jobs to run in parallel"""
 NUM_PARALLEL = 8

bin/play.py

@@ -9,9 +9,9 @@ import sys
 import random
 import argparse
 
-from deepspeech_training.util.audio import LOADABLE_AUDIO_EXTENSIONS, AUDIO_TYPE_PCM, AUDIO_TYPE_WAV
-from deepspeech_training.util.sample_collections import SampleList, LabeledSample, samples_from_source
-from deepspeech_training.util.augmentations import parse_augmentations, apply_sample_augmentations, SampleAugmentation
+from mozilla_voice_stt_training.util.audio import LOADABLE_AUDIO_EXTENSIONS, AUDIO_TYPE_PCM, AUDIO_TYPE_WAV
+from mozilla_voice_stt_training.util.sample_collections import SampleList, LabeledSample, samples_from_source
+from mozilla_voice_stt_training.util.augmentations import parse_augmentations, apply_sample_augmentations, SampleAugmentation
 
 
 def get_samples_in_play_order():

data/README.rst

@@ -3,7 +3,7 @@ Language-Specific Data
 
 This directory contains language-specific data files. Most importantly, you will find here:
 
-1. A list of unique characters for the target language (e.g. English) in ``data/alphabet.txt``. After installing the training code, you can check ``python -m deepspeech_training.util.check_characters --help`` for a tool that creates an alphabet file from a list of training CSV files.
+1. A list of unique characters for the target language (e.g. English) in ``data/alphabet.txt``. After installing the training code, you can check ``python -m mozilla_voice_stt_training.util.check_characters --help`` for a tool that creates an alphabet file from a list of training CSV files.
 
 2. A script used to generate a binary n-gram language model: ``data/lm/generate_lm.py``.
 

doc/Flags.rst

@@ -8,7 +8,7 @@ Below you can find the definition of all command-line flags supported by the tra
 Flags
 -----
 
-.. literalinclude:: ../training/deepspeech_training/util/flags.py
+.. literalinclude:: ../training/mozilla_voice_stt_training/util/flags.py
    :language: python
    :linenos:
    :lineno-match:

doc/examples

@@ -1 +1 @@
-Subproject commit 6f5f501fa62743f1b78fe162eb1a579a450bd38f
+Subproject commit 93914db805408b4a130e4489cb5d4ff88cd95159

evaluate.py

@@ -4,7 +4,7 @@ from __future__ import absolute_import, division, print_function
 
 if __name__ == '__main__':
     try:
-        from deepspeech_training import evaluate as ds_evaluate
+        from mozilla_voice_stt_training import evaluate as ds_evaluate
     except ImportError:
         print('Training package is not installed. See training documentation.')
         raise

evaluate_tflite.py

@@ -11,8 +11,8 @@ import os
 import sys
 
 from mozilla_voice_stt import Model
-from deepspeech_training.util.evaluate_tools import calculate_and_print_report
-from deepspeech_training.util.flags import create_flags
+from mozilla_voice_stt_training.util.evaluate_tools import calculate_and_print_report
+from mozilla_voice_stt_training.util.flags import create_flags
 from functools import partial
 from multiprocessing import JoinableQueue, Process, cpu_count, Manager
 from six.moves import zip, range

lm_optimizer.py

@@ -7,12 +7,12 @@ import optuna
 import sys
 import tensorflow.compat.v1 as tfv1
 
-from deepspeech_training.evaluate import evaluate
-from deepspeech_training.train import create_model
-from deepspeech_training.util.config import Config, initialize_globals
-from deepspeech_training.util.flags import create_flags, FLAGS
-from deepspeech_training.util.logging import log_error
-from deepspeech_training.util.evaluate_tools import wer_cer_batch
+from mozilla_voice_stt_training.evaluate import evaluate
+from mozilla_voice_stt_training.train import create_model
+from mozilla_voice_stt_training.util.config import Config, initialize_globals
+from mozilla_voice_stt_training.util.flags import create_flags, FLAGS
+from mozilla_voice_stt_training.util.logging import log_error
+from mozilla_voice_stt_training.util.evaluate_tools import wer_cer_batch
 from ds_ctcdecoder import Scorer
 
 

native_client/bazel_workspace_status_cmd.sh

@@ -22,8 +22,8 @@ echo "STABLE_TF_GIT_VERSION ${tf_git_rev}"
 pushd $(dirname "$0")
 ds_git_rev=$(git describe --long --tags)
 echo "STABLE_DS_GIT_VERSION ${ds_git_rev}"
-ds_version=$(cat ../training/deepspeech_training/VERSION)
+ds_version=$(cat ../training/mozilla_voice_stt_training/VERSION)
 echo "STABLE_DS_VERSION ${ds_version}"
-ds_graph_version=$(cat ../training/deepspeech_training/GRAPH_VERSION)
+ds_graph_version=$(cat ../training/mozilla_voice_stt_training/GRAPH_VERSION)
 echo "STABLE_DS_GRAPH_VERSION ${ds_graph_version}"
 popd

native_client/ctcdecode/setup.py

@@ -51,7 +51,7 @@ def maybe_rebuild(srcs, out_name, build_dir):
                      num_parallel=known_args.num_processes,
                      debug=debug)
 
-project_version = read('../../training/deepspeech_training/VERSION').strip()
+project_version = read('../../training/mozilla_voice_stt_training/VERSION').strip()
 
 build_dir = 'temp_build/temp_build'
 

native_client/javascript/Makefile

@@ -3,7 +3,7 @@ NODE_ABI_TARGET    ?=
 NODE_BUILD_VERBOSE ?= --verbose
 NPM_TOOL           ?= npm
 PROJECT_NAME       ?= @mozilla-voice/stt
-PROJECT_VERSION    ?= $(shell cat ../../training/deepspeech_training/VERSION | tr -d '\n')
+PROJECT_VERSION    ?= $(shell cat ../../training/mozilla_voice_stt_training/VERSION | tr -d '\n')
 NPM_ROOT           ?= $(shell npm root)
 
 NODE_MODULES_BIN   ?= $(NPM_ROOT)/.bin/

native_client/python/setup.py

@@ -31,7 +31,7 @@ def main():
         sys.argv.remove('--project_name')
         sys.argv.pop(project_name_idx)
 
-    with open('../../training/deepspeech_training/VERSION', 'r') as ver:
+    with open('../../training/mozilla_voice_stt_training/VERSION', 'r') as ver:
         project_version = ver.read().strip()
 
     class BuildExtFirst(build):

setup.py

@@ -96,7 +96,7 @@ def main():
         install_requires = install_requires + tensorflow_pypi_dep
 
     setup(
-        name='deepspeech_training',
+        name='mozilla_voice_stt_training',
         version=version,
         description='Training code for mozilla DeepSpeech',
         url='https://github.com/mozilla/DeepSpeech',
@@ -119,7 +119,7 @@ def main():
         # If there are data files included in your packages that need to be
         # installed, specify them here.
         package_data={
-            'deepspeech_training': [
+            'mozilla_voice_stt_training': [
                 'VERSION',
                 'GRAPH_VERSION',
             ],

stats.py

@@ -3,7 +3,7 @@ import argparse
 import functools
 import pandas
 
-from deepspeech_training.util.helpers import secs_to_hours
+from mozilla_voice_stt_training.util.helpers import secs_to_hours
 from pathlib import Path
 
 

taskcluster/tc-all-vars.sh

@@ -53,7 +53,7 @@ export DS_TFDIR=${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow
 export DS_DSDIR=${DS_ROOT_TASK}/DeepSpeech/ds
 export DS_EXAMPLEDIR=${DS_ROOT_TASK}/DeepSpeech/examples
 
-export DS_VERSION="$(cat ${DS_DSDIR}/training/deepspeech_training/VERSION)"
+export DS_VERSION="$(cat ${DS_DSDIR}/training/mozilla_voice_stt_training/VERSION)"
 
 export GRADLE_USER_HOME=${DS_ROOT_TASK}/gradle-cache
 export ANDROID_SDK_HOME=${DS_ROOT_TASK}/DeepSpeech/Android/SDK/

tests/test_importers.py

@@ -1,7 +1,7 @@
 import unittest
 
 from argparse import Namespace
-from deepspeech_training.util.importers import validate_label_eng, get_validate_label
+from mozilla_voice_stt_training.util.importers import validate_label_eng, get_validate_label
 from pathlib import Path
 
 def from_here(path):

tests/test_value_range.py

@@ -2,7 +2,7 @@ import unittest
 
 import numpy as np
 import tensorflow as tf
-from deepspeech_training.util.helpers import ValueRange, get_value_range, pick_value_from_range, tf_pick_value_from_range
+from mozilla_voice_stt_training.util.helpers import ValueRange, get_value_range, pick_value_from_range, tf_pick_value_from_range
 
 
 class TestValueRange(unittest.TestCase):

transcribe.py

@@ -13,11 +13,11 @@ import logging
 logging.getLogger('sox').setLevel(logging.ERROR)
 import glob
 
-from deepspeech_training.util.audio import AudioFile
-from deepspeech_training.util.config import Config, initialize_globals
-from deepspeech_training.util.feeding import split_audio_file
-from deepspeech_training.util.flags import create_flags, FLAGS
-from deepspeech_training.util.logging import log_error, log_info, log_progress, create_progressbar
+from mozilla_voice_stt_training.util.audio import AudioFile
+from mozilla_voice_stt_training.util.config import Config, initialize_globals
+from mozilla_voice_stt_training.util.feeding import split_audio_file
+from mozilla_voice_stt_training.util.flags import create_flags, FLAGS
+from mozilla_voice_stt_training.util.logging import log_error, log_info, log_progress, create_progressbar
 from ds_ctcdecoder import ctc_beam_search_decoder_batch, Scorer
 from multiprocessing import Process, cpu_count
 
@@ -28,8 +28,8 @@ def fail(message, code=1):
 
 
 def transcribe_file(audio_path, tlog_path):
-    from deepspeech_training.train import create_model  # pylint: disable=cyclic-import,import-outside-toplevel
-    from deepspeech_training.util.checkpoints import load_graph_for_evaluation
+    from mozilla_voice_stt_training.train import create_model  # pylint: disable=cyclic-import,import-outside-toplevel
+    from mozilla_voice_stt_training.util.checkpoints import load_graph_for_evaluation
     initialize_globals()
     scorer = Scorer(FLAGS.lm_alpha, FLAGS.lm_beta, FLAGS.scorer_path, Config.alphabet)
     try:

util/taskcluster.py

@@ -4,7 +4,7 @@ from __future__ import absolute_import, division, print_function
 
 if __name__ == '__main__':
     try:
-        from deepspeech_training.util import taskcluster as dsu_taskcluster
+        from mozilla_voice_stt_training.util import taskcluster as dsu_taskcluster
     except ImportError:
         print('Training package is not installed. See training documentation.')
         raise