mozilla
/
DeepSpeech
зеркало из https://github.com/mozilla/DeepSpeech.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Support MemmappedFileSystem 

Fixes #1231
This commit is contained in:
Alexandre Lissy 2018-02-14 18:41:21 +01:00
Родитель bd8d463dc1
Коммит 0c323e8c67
21 изменённых файлов: 106 добавлений и 41 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

1
native_client/BUILD
Просмотреть файл

@ -67,6 +67,7 @@ tf_cc_shared_object(
### Obtained by trial/error process ...
### CPU only build libdeepspeech.so from 63M to 36M
"//tensorflow/core/kernels:constant_op", # Const
"//tensorflow/core/kernels:immutable_constant_op", # ImmutableConst
"//tensorflow/core/kernels:identity_op", # Identity
"//tensorflow/core/kernels:transpose_op", # Transpose
"//tensorflow/core/kernels:reshape_op", # Reshape

34
native_client/deepspeech.cc
Просмотреть файл

@ -16,6 +16,7 @@
#include "tensorflow/core/public/session.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/util/memmapped_file_system.h"
#define BATCH_SIZE 1
@ -27,6 +28,7 @@ namespace DeepSpeech {
class Private {
public:
MemmappedEnv* mmap_env;
Session* session;
GraphDef graph_def;
int ncep;
@ -42,6 +44,7 @@ Model::Model(const char* aModelPath, int aNCep, int aNContext,
const char* aAlphabetConfigPath, int aBeamWidth)
{
mPriv = new Private;
mPriv->mmap_env = new MemmappedEnv(Env::Default());
mPriv->session = NULL;
mPriv->scorer = NULL;
mPriv->ncep = aNCep;
@ -56,13 +59,39 @@ Model::Model(const char* aModelPath, int aNCep, int aNContext,
return;
}
Status status = NewSession(SessionOptions(), &mPriv->session);
Status status;
SessionOptions options;
bool is_mmap = std::string(aModelPath).find(".pbmm") != std::string::npos;
if (!is_mmap) {
std::cerr << "Warning: reading entire model file into memory. Transform model file into an mmapped graph to reduce heap usage." << std::endl;
}
if (is_mmap) {
status = mPriv->mmap_env->InitializeFromFile(aModelPath);
if (!status.ok()) {
std::cerr << status.ToString() << std::endl;
return;
}
options.config.mutable_graph_options()
->mutable_optimizer_options()
->set_opt_level(::OptimizerOptions::L0);
options.env = mPriv->mmap_env;
}
status = NewSession(options, &mPriv->session);
if (!status.ok()) {
std::cerr << status.ToString() << std::endl;
return;
}
status = ReadBinaryProto(Env::Default(), aModelPath, &mPriv->graph_def);
if (is_mmap) {
status = ReadBinaryProto(mPriv->mmap_env,
MemmappedFileSystem::kMemmappedPackageDefaultGraphDef,
&mPriv->graph_def);
} else {
status = ReadBinaryProto(Env::Default(), aModelPath, &mPriv->graph_def);
}
if (!status.ok()) {
mPriv->session->Close();
mPriv->session = NULL;
@ -105,6 +134,7 @@ Model::~Model()
mPriv->session->Close();
}
delete mPriv->mmap_env;
delete mPriv->alphabet;
delete mPriv->scorer;

1
taskcluster/.build.yml
Просмотреть файл

@ -30,3 +30,4 @@ build:
args:
tests_cmdline: ''
deepspeech_pkg_name: ''
convert_graphdef: ''

4
taskcluster/darwin-amd64-cpu-aot_prod-opt.yml
Просмотреть файл

@ -4,8 +4,8 @@ build:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.osx_aot"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.${event.head.sha}.osx_aot"
- "index.project.deepspeech.deepspeech.native_client.osx_aot.${event.head.sha}"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.723fb3984980c994579c055736688f693e38ee27.osx/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.723fb3984980c994579c055736688f693e38ee27.osx/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.ad8f785459e80823a2ff4456eeb9d7220c33b9c6.osx/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.ad8f785459e80823a2ff4456eeb9d7220c33b9c6.osx/artifacts/public/summarize_graph"
scripts:
build: "taskcluster/host-build.sh --aot"
package: "taskcluster/package.sh"

4
taskcluster/darwin-amd64-cpu-opt.yml
Просмотреть файл

@ -6,8 +6,8 @@ build:
- "index.project.deepspeech.deepspeech.native_client.osx.${event.head.sha}"
- "notify.irc-channel.${notifications.irc}.on-exception"
- "notify.irc-channel.${notifications.irc}.on-failed"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.723fb3984980c994579c055736688f693e38ee27.osx/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.723fb3984980c994579c055736688f693e38ee27.osx/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.ad8f785459e80823a2ff4456eeb9d7220c33b9c6.osx/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.ad8f785459e80823a2ff4456eeb9d7220c33b9c6.osx/artifacts/public/summarize_graph"
scripts:
build: "taskcluster/host-build.sh"
package: "taskcluster/package.sh"

4
taskcluster/linux-amd64-cpu-aot_prod-opt.yml
Просмотреть файл

@ -4,8 +4,8 @@ build:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.cpu_aot"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.${event.head.sha}.cpu_aot"
- "index.project.deepspeech.deepspeech.native_client.cpu_aot.${event.head.sha}"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.723fb3984980c994579c055736688f693e38ee27.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.723fb3984980c994579c055736688f693e38ee27.cpu/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.ad8f785459e80823a2ff4456eeb9d7220c33b9c6.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.ad8f785459e80823a2ff4456eeb9d7220c33b9c6.cpu/artifacts/public/summarize_graph"
system_setup:
>
${nodejs.packages.prep_6} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml &&

4
taskcluster/linux-amd64-cpu-aot_test-opt.yml
Просмотреть файл

@ -6,8 +6,8 @@ build:
template_file: linux-opt-base.tyml
dependencies:
- "test-training_upstream-linux-amd64-py27-opt"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.723fb3984980c994579c055736688f693e38ee27.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.723fb3984980c994579c055736688f693e38ee27.cpu/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.ad8f785459e80823a2ff4456eeb9d7220c33b9c6.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.ad8f785459e80823a2ff4456eeb9d7220c33b9c6.cpu/artifacts/public/summarize_graph"
system_setup:
>
${nodejs.packages.prep_6} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml &&

4
taskcluster/linux-amd64-cpu-opt.yml
Просмотреть файл

@ -13,8 +13,8 @@ build:
system_config:
>
${swig.patch_nodejs.linux}
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.723fb3984980c994579c055736688f693e38ee27.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.723fb3984980c994579c055736688f693e38ee27.cpu/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.ad8f785459e80823a2ff4456eeb9d7220c33b9c6.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.ad8f785459e80823a2ff4456eeb9d7220c33b9c6.cpu/artifacts/public/summarize_graph"
scripts:
build: "taskcluster/host-build.sh"
package: "taskcluster/package.sh"

4
taskcluster/linux-amd64-ctc-opt.yml
Просмотреть файл

@ -4,8 +4,8 @@ build:
- "pull_request.synchronize"
- "pull_request.reopened"
template_file: linux-opt-base.tyml
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.723fb3984980c994579c055736688f693e38ee27.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.723fb3984980c994579c055736688f693e38ee27.cpu/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.ad8f785459e80823a2ff4456eeb9d7220c33b9c6.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.ad8f785459e80823a2ff4456eeb9d7220c33b9c6.cpu/artifacts/public/summarize_graph"
scripts:
build: 'taskcluster/decoder-build.sh'
package: 'taskcluster/decoder-package.sh'

4
taskcluster/linux-amd64-gpu-opt.yml
Просмотреть файл

@ -11,8 +11,8 @@ build:
system_config:
>
${swig.patch_nodejs.linux}
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.723fb3984980c994579c055736688f693e38ee27.gpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.723fb3984980c994579c055736688f693e38ee27.gpu/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.ad8f785459e80823a2ff4456eeb9d7220c33b9c6.gpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.ad8f785459e80823a2ff4456eeb9d7220c33b9c6.gpu/artifacts/public/summarize_graph"
maxRunTime: 14400
scripts:
build: "taskcluster/cuda-build.sh"

4
taskcluster/linux-rpi3-cpu-aot_prod-opt.yml
Просмотреть файл

@ -4,8 +4,8 @@ build:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.arm_aot"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.${event.head.sha}.arm_aot"
- "index.project.deepspeech.deepspeech.native_client.arm_aot.${event.head.sha}"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.723fb3984980c994579c055736688f693e38ee27.arm/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.723fb3984980c994579c055736688f693e38ee27.cpu/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.ad8f785459e80823a2ff4456eeb9d7220c33b9c6.arm/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.ad8f785459e80823a2ff4456eeb9d7220c33b9c6.cpu/artifacts/public/summarize_graph"
## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787
system_setup:
>

4
taskcluster/linux-rpi3-cpu-opt.yml
Просмотреть файл

@ -4,8 +4,8 @@ build:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.arm"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branch}.${event.head.sha}.arm"
- "index.project.deepspeech.deepspeech.native_client.arm.${event.head.sha}"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.723fb3984980c994579c055736688f693e38ee27.arm/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.723fb3984980c994579c055736688f693e38ee27.cpu/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.ad8f785459e80823a2ff4456eeb9d7220c33b9c6.arm/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.ad8f785459e80823a2ff4456eeb9d7220c33b9c6.cpu/artifacts/public/summarize_graph"
## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787
system_setup:
>

4
taskcluster/node-package.yml
Просмотреть файл

@ -14,8 +14,8 @@ build:
system_config:
>
${swig.patch_nodejs.linux}
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.723fb3984980c994579c055736688f693e38ee27.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.723fb3984980c994579c055736688f693e38ee27.cpu/artifacts/public/summarize_graph"
tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.ad8f785459e80823a2ff4456eeb9d7220c33b9c6.cpu/artifacts/public/home.tar.xz"
summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.ad8f785459e80823a2ff4456eeb9d7220c33b9c6.cpu/artifacts/public/summarize_graph"
scripts:
build: "taskcluster/node-build.sh"
package: "taskcluster/node-package.sh"

3
taskcluster/test-darwin-opt-base.tyml
Просмотреть файл

@ -37,7 +37,8 @@ then:
DEEPSPEECH_ARTIFACTS_ROOT: https://queue.taskcluster.net/v1/task/${darwin_amd64_build}/artifacts/public
DEEPSPEECH_NODEJS: https://queue.taskcluster.net/v1/task/${node_package}/artifacts/public
DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/artifacts/public/output_graph.pb
DEEPSPEECH_PROD_MODEL: https://s3.amazonaws.com/deep-speech/output_graph.pb
DEEPSPEECH_PROD_MODEL: https://s3-us-west-2.amazonaws.com/deepspeech/mmap/output_graph.pb
DEEPSPEECH_PROD_MODEL_MMAP: https://s3-us-west-2.amazonaws.com/deepspeech/mmap/output_graph.pbmm
command:
- - "/bin/bash"

4
taskcluster/test-linux-opt-base.tyml
Просмотреть файл

@ -37,13 +37,15 @@ then:
linux_amd64_ctc: { $eval: as_slugid("linux-amd64-ctc-opt") }
node_package: { $eval: as_slugid("node-package") }
in:
CONVERT_GRAPHDEF_MEMMAPPED: ${build.convert_graphdef}
DEEPSPEECH_ARTIFACTS_ROOT: https://queue.taskcluster.net/v1/task/${linux_amd64_build}/artifacts/public
DEEPSPEECH_PYTHON_PACKAGE: https://queue.taskcluster.net/v1/task/${linux_amd64_build}/artifacts/public/${build.deepspeech_pkg_name}
DEEPSPEECH_NODEJS: https://queue.taskcluster.net/v1/task/${node_package}/artifacts/public
DEEPSPEECH_AOT_ARTIFACTS_ROOT: https://queue.taskcluster.net/v1/task/${linux_amd64_aot_test}/artifacts/public
DEEPSPEECH_LIBCTC: https://queue.taskcluster.net/v1/task/${linux_amd64_ctc}/artifacts/public/decoder.tar.xz
DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/artifacts/public/output_graph.pb
DEEPSPEECH_PROD_MODEL: https://s3.amazonaws.com/deep-speech/output_graph.pb
DEEPSPEECH_PROD_MODEL: https://s3-us-west-2.amazonaws.com/deepspeech/mmap/output_graph.pb
DEEPSPEECH_PROD_MODEL_MMAP: https://s3-us-west-2.amazonaws.com/deepspeech/mmap/output_graph.pbmm
command:
- "/bin/bash"

1
taskcluster/test-training_upstream-linux-amd64-py27-opt.yml
Просмотреть файл

@ -7,6 +7,7 @@ build:
apt-get -qq -y install ${python.packages.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-train-tests.sh 2.7.13 upstream"
convert_graphdef: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.r1.5.ad8f785459e80823a2ff4456eeb9d7220c33b9c6.cpu/artifacts/public/convert_graphdef_memmapped_format"
metadata:
name: "DeepSpeech Linux AMD64 CPU upstream training Py2.7"
description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 using upstream TensorFlow Python 2.7, CPU only, optimized version"

3
tc-cpp-ds-tests-prod.sh
Просмотреть файл

@ -7,6 +7,9 @@ source $(dirname "$0")/tc-tests-utils.sh
model_source=${DEEPSPEECH_PROD_MODEL}
model_name=$(basename "${model_source}")
model_source_mmap=${DEEPSPEECH_PROD_MODEL_MMAP}
model_name_mmap=$(basename "${model_source_mmap}")
download_material "${TASKCLUSTER_TMP_DIR}/ds"
export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH

3
tc-node-tests-prod.sh
Просмотреть файл

@ -14,6 +14,9 @@ fi;
model_source=${DEEPSPEECH_PROD_MODEL}
model_name=$(basename "${model_source}")
model_source_mmap=${DEEPSPEECH_PROD_MODEL_MMAP}
model_name_mmap=$(basename "${model_source_mmap}")
download_data
node --version

3
tc-python-tests-prod.sh
Просмотреть файл

@ -31,6 +31,9 @@ mkdir -p ${PYENV_ROOT} || true
model_source=${DEEPSPEECH_PROD_MODEL}
model_name=$(basename "${model_source}")
model_source_mmap=${DEEPSPEECH_PROD_MODEL_MMAP}
model_name_mmap=$(basename "${model_source_mmap}")
download_data
install_pyenv "${PYENV_ROOT}"

46
tc-tests-utils.sh
Просмотреть файл

@ -37,8 +37,10 @@ export BAZEL_AOT_TARGETS="
//native_client:libdeepspeech_model.so
"
model_source=${DEEPSPEECH_TEST_MODEL}
model_name=$(basename "${model_source}")
model_source="${DEEPSPEECH_TEST_MODEL}"
model_name="$(basename "${model_source}")"
model_name_mmap="$(basename -s ".pb" "${model_source}").pbmm"
model_source_mmap="$(dirname "${model_source}")/${model_name_mmap}"
SUPPORTED_PYTHON_VERSIONS=${SUPPORTED_PYTHON_VERSIONS:-2.7.13 3.4.6 3.5.3 3.6.2}
SUPPORTED_NODEJS_VERSIONS=${SUPPORTED_NODEJS_VERSIONS:-4.8.6 5.12.0 6.12.0 7.10.1 8.9.1 9.2.0}
@ -147,14 +149,19 @@ assert_correct_multi_ldc93s1()
# assert_shows_something "$1" "/LDC93S1_pcms16le_1_8000.wav%she hayorasryrtl lyreasy asr watal w water all year%"
}
assert_correct_ldc93s1_prodmodel()
assert_correct_ldc93s1_prodmodel_v1()
{
assert_correct_inference "$1" "she had the duck so ingrecywachworallyear"
}
assert_correct_ldc93s1_prodmodel_v2()
{
assert_correct_inference "$1" "she had a ducsuotangresywathorerall year"
}
assert_working_ldc93s1_prodmodel()
{
assert_working_inference "$1" "she had the duck so"
assert_working_inference "$1" "she had"
}
assert_correct_ldc93s1_somodel()
@ -198,19 +205,22 @@ run_all_inference_tests()
phrase_pbmodel_nolm=$(deepspeech ${TASKCLUSTER_TMP_DIR}/${model_name} ${TASKCLUSTER_TMP_DIR}/alphabet.txt ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav)
assert_correct_ldc93s1 "${phrase_pbmodel_nolm}"
phrase_pbmodel_withlm=$(deepspeech ${TASKCLUSTER_TMP_DIR}/${model_name} ${TASKCLUSTER_TMP_DIR}/alphabet.txt ${TASKCLUSTER_TMP_DIR}/lm.binary ${TASKCLUSTER_TMP_DIR}/trie ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav)
phrase_pbmodel_nolm=$(deepspeech ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} ${TASKCLUSTER_TMP_DIR}/alphabet.txt ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav)
assert_correct_ldc93s1 "${phrase_pbmodel_nolm}"
phrase_pbmodel_withlm=$(deepspeech ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} ${TASKCLUSTER_TMP_DIR}/alphabet.txt ${TASKCLUSTER_TMP_DIR}/lm.binary ${TASKCLUSTER_TMP_DIR}/trie ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav)
assert_correct_ldc93s1 "${phrase_pbmodel_withlm}"
phrase_pbmodel_nolm_stereo_44k=$(deepspeech ${TASKCLUSTER_TMP_DIR}/${model_name} ${TASKCLUSTER_TMP_DIR}/alphabet.txt ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav)
phrase_pbmodel_nolm_stereo_44k=$(deepspeech ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} ${TASKCLUSTER_TMP_DIR}/alphabet.txt ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav)
assert_correct_ldc93s1 "${phrase_pbmodel_nolm_stereo_44k}"
phrase_pbmodel_withlm_stereo_44k=$(deepspeech ${TASKCLUSTER_TMP_DIR}/${model_name} ${TASKCLUSTER_TMP_DIR}/alphabet.txt ${TASKCLUSTER_TMP_DIR}/lm.binary ${TASKCLUSTER_TMP_DIR}/trie ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav)
phrase_pbmodel_withlm_stereo_44k=$(deepspeech ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} ${TASKCLUSTER_TMP_DIR}/alphabet.txt ${TASKCLUSTER_TMP_DIR}/lm.binary ${TASKCLUSTER_TMP_DIR}/trie ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav)
assert_correct_ldc93s1 "${phrase_pbmodel_withlm_stereo_44k}"
phrase_pbmodel_nolm_mono_8k=$(deepspeech ${TASKCLUSTER_TMP_DIR}/${model_name} ${TASKCLUSTER_TMP_DIR}/alphabet.txt ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null)
phrase_pbmodel_nolm_mono_8k=$(deepspeech ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} ${TASKCLUSTER_TMP_DIR}/alphabet.txt ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null)
assert_correct_warning_upsampling "${phrase_pbmodel_nolm_mono_8k}"
phrase_pbmodel_withlm_mono_8k=$(deepspeech ${TASKCLUSTER_TMP_DIR}/${model_name} ${TASKCLUSTER_TMP_DIR}/alphabet.txt ${TASKCLUSTER_TMP_DIR}/lm.binary ${TASKCLUSTER_TMP_DIR}/trie ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null)
phrase_pbmodel_withlm_mono_8k=$(deepspeech ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} ${TASKCLUSTER_TMP_DIR}/alphabet.txt ${TASKCLUSTER_TMP_DIR}/lm.binary ${TASKCLUSTER_TMP_DIR}/trie ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null)
assert_correct_warning_upsampling "${phrase_pbmodel_withlm_mono_8k}"
if [ "${aot_model}" = "--aot" ]; then
@ -234,12 +244,15 @@ run_all_inference_tests()
run_prod_inference_tests()
{
phrase_pbmodel_withlm=$(deepspeech ${TASKCLUSTER_TMP_DIR}/${model_name} ${TASKCLUSTER_TMP_DIR}/alphabet.txt ${TASKCLUSTER_TMP_DIR}/lm.binary ${TASKCLUSTER_TMP_DIR}/trie ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav)
assert_correct_ldc93s1_prodmodel "${phrase_pbmodel_withlm}"
assert_correct_ldc93s1_prodmodel_v1 "${phrase_pbmodel_withlm}"
phrase_pbmodel_withlm_stereo_44k=$(deepspeech ${TASKCLUSTER_TMP_DIR}/${model_name} ${TASKCLUSTER_TMP_DIR}/alphabet.txt ${TASKCLUSTER_TMP_DIR}/lm.binary ${TASKCLUSTER_TMP_DIR}/trie ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav)
phrase_pbmodel_withlm=$(deepspeech ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} ${TASKCLUSTER_TMP_DIR}/alphabet.txt ${TASKCLUSTER_TMP_DIR}/lm.binary ${TASKCLUSTER_TMP_DIR}/trie ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav)
assert_correct_ldc93s1_prodmodel_v2 "${phrase_pbmodel_withlm}"
phrase_pbmodel_withlm_stereo_44k=$(deepspeech ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} ${TASKCLUSTER_TMP_DIR}/alphabet.txt ${TASKCLUSTER_TMP_DIR}/lm.binary ${TASKCLUSTER_TMP_DIR}/trie ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav)
assert_working_ldc93s1_prodmodel "${phrase_pbmodel_withlm_stereo_44k}"
phrase_pbmodel_withlm_mono_8k=$(deepspeech ${TASKCLUSTER_TMP_DIR}/${model_name} ${TASKCLUSTER_TMP_DIR}/alphabet.txt ${TASKCLUSTER_TMP_DIR}/lm.binary ${TASKCLUSTER_TMP_DIR}/trie ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null)
phrase_pbmodel_withlm_mono_8k=$(deepspeech ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} ${TASKCLUSTER_TMP_DIR}/alphabet.txt ${TASKCLUSTER_TMP_DIR}/lm.binary ${TASKCLUSTER_TMP_DIR}/trie ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null)
assert_correct_warning_upsampling "${phrase_pbmodel_withlm_mono_8k}"
}
@ -286,7 +299,8 @@ download_ctc_kenlm()
download_data()
{
wget ${model_source} -O ${TASKCLUSTER_TMP_DIR}/${model_name}
wget -P "${TASKCLUSTER_TMP_DIR}" "${model_source}"
wget -P "${TASKCLUSTER_TMP_DIR}" "${model_source_mmap}"
cp ${DS_ROOT_TASK}/DeepSpeech/ds/data/smoke_test/*.wav ${TASKCLUSTER_TMP_DIR}/
cp ${DS_ROOT_TASK}/DeepSpeech/ds/data/alphabet.txt ${TASKCLUSTER_TMP_DIR}/alphabet.txt
cp ${DS_ROOT_TASK}/DeepSpeech/ds/data/lm/lm.binary ${TASKCLUSTER_TMP_DIR}/lm.binary
@ -306,7 +320,7 @@ download_material()
download_data
ls -hal ${TASKCLUSTER_TMP_DIR}/${model_name} ${TASKCLUSTER_TMP_DIR}/LDC93S1*.wav ${TASKCLUSTER_TMP_DIR}/alphabet.txt
ls -hal ${TASKCLUSTER_TMP_DIR}/${model_name} ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} ${TASKCLUSTER_TMP_DIR}/LDC93S1*.wav ${TASKCLUSTER_TMP_DIR}/alphabet.txt
}
install_pyenv()
@ -351,9 +365,7 @@ do_get_model_parameters()
fi;
wget "${model_url}" -O "${model_file}"
wget "${SUMMARIZE_GRAPH_BINARY}" -O "/tmp/summarize_graph"
chmod +x /tmp/summarize_graph
wget -P "/tmp/" "${SUMMARIZE_GRAPH_BINARY}" && chmod +x /tmp/summarize_graph
if [ ! -f "${model_file}" ]; then
echo "No such model: ${model_file}"

8
tc-train-tests.sh
Просмотреть файл

@ -60,3 +60,11 @@ deactivate
pyenv uninstall --force ${PYENV_NAME}
cp /tmp/train/output_graph.pb ${TASKCLUSTER_ARTIFACTS}
if [ ! -z "${CONVERT_GRAPHDEF_MEMMAPPED}" ]; then
convert_graphdef=$(basename "${CONVERT_GRAPHDEF_MEMMAPPED}")
wget -P "/tmp/" "${CONVERT_GRAPHDEF_MEMMAPPED}" && chmod +x "/tmp/${convert_graphdef}"
/tmp/${convert_graphdef} --in_graph=/tmp/train/output_graph.pb --out_graph=/tmp/train/output_graph.pbmm
cp /tmp/train/output_graph.pbmm ${TASKCLUSTER_ARTIFACTS}
fi;