Merge pull request #3621 from mozilla/windows-ga-ci

Base Windows CI
This commit is contained in:
Reuben Morais 2021-04-16 20:52:55 +02:00 коммит произвёл GitHub
Родитель 2f2240e18a 4df81c2144
Коммит 0b007c9fb6
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
25 изменённых файлов: 3892 добавлений и 777 удалений

1
.gitattributes поставляемый
Просмотреть файл

@ -1 +1,2 @@
data/lm/kenlm.scorer filter=lfs diff=lfs merge=lfs -text
.github/actions/check_artifact_exists/dist/index.js binary

Просмотреть файл

@ -35,3 +35,9 @@ Hacking:
For hacking into the action, please follow the [GitHub JavaScript
Actions](https://docs.github.com/en/actions/creating-actions/creating-a-javascript-action#commit-tag-and-push-your-action-to-github)
and specifically the usage of `ncc`.
```
$ npm install
$ npx ncc build main.js --license licenses.txt
$ git add dist/
```

Разница между файлами не показана из-за своего большого размера Загрузить разницу

Просмотреть файл

@ -160,6 +160,31 @@ The above copyright notice and this permission notice (including the next paragr
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@octokit/plugin-throttling
MIT
The MIT License
Copyright (c) 2018 Octokit contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
@octokit/request
MIT
The MIT License
@ -450,6 +475,30 @@ Apache-2.0
limitations under the License.
bottleneck
MIT
The MIT License (MIT)
Copyright (c) 2014 Simon Grondin
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
deprecation
ISC
The ISC License

34
.github/actions/check_artifact_exists/main.js поставляемый
Просмотреть файл

@ -4,6 +4,8 @@ const AdmZip = require('adm-zip');
const filesize = require('filesize');
const pathname = require('path');
const fs = require('fs');
const { throttling } = require('@octokit/plugin-throttling');
const { GitHub } = require('@actions/github/lib/utils');
async function getGoodArtifacts(client, owner, repo, name) {
const goodWorkflowArtifacts = await client.paginate(
@ -57,8 +59,29 @@ async function main() {
const path = core.getInput("path", { required: true });
const name = core.getInput("name");
const download = core.getInput("download");
const client = github.getOctokit(token)
const OctokitWithThrottling = GitHub.plugin(throttling);
const client = new OctokitWithThrottling({
auth: token,
throttle: {
onRateLimit: (retryAfter, options) => {
console.log(
`Request quota exhausted for request ${options.method} ${options.url}`
);
// Retry twice after hitting a rate limit error, then give up
if (options.request.retryCount <= 2) {
console.log(`Retrying after ${retryAfter} seconds!`);
return true;
}
},
onAbuseLimit: (retryAfter, options) => {
// does not retry, only logs a warning
console.log(
`Abuse detected for request ${options.method} ${options.url}`
);
},
},
});
console.log("==> Repo:", owner + "/" + repo);
const goodArtifacts = await getGoodArtifacts(client, owner, repo, name);
@ -116,4 +139,11 @@ async function main() {
return;
}
main();
// We have to manually wrap the main function with a try-catch here because
// GitHub will ignore uncatched exceptions and continue running the workflow,
// leading to harder to diagnose errors downstream from this action.
try {
main();
} catch (error) {
core.setFailed(error.message);
}

220
.github/actions/check_artifact_exists/package-lock.json сгенерированный поставляемый Normal file
Просмотреть файл

@ -0,0 +1,220 @@
{
"name": "check_artifact_exists",
"requires": true,
"lockfileVersion": 1,
"dependencies": {
"@actions/core": {
"version": "1.2.6",
"resolved": "https://registry.npmjs.org/@actions/core/-/core-1.2.6.tgz",
"integrity": "sha512-ZQYitnqiyBc3D+k7LsgSBmMDVkOVidaagDG7j3fOym77jNunWRuYx7VSHa9GNfFZh+zh61xsCjRj4JxMZlDqTA==",
"dev": true
},
"@actions/github": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/@actions/github/-/github-4.0.0.tgz",
"integrity": "sha512-Ej/Y2E+VV6sR9X7pWL5F3VgEWrABaT292DRqRU6R4hnQjPtC/zD3nagxVdXWiRQvYDh8kHXo7IDmG42eJ/dOMA==",
"dev": true,
"requires": {
"@actions/http-client": "^1.0.8",
"@octokit/core": "^3.0.0",
"@octokit/plugin-paginate-rest": "^2.2.3",
"@octokit/plugin-rest-endpoint-methods": "^4.0.0"
}
},
"@actions/http-client": {
"version": "1.0.11",
"resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-1.0.11.tgz",
"integrity": "sha512-VRYHGQV1rqnROJqdMvGUbY/Kn8vriQe/F9HR2AlYHzmKuM/p3kjNuXhmdBfcVgsvRWTz5C5XW5xvndZrVBuAYg==",
"dev": true,
"requires": {
"tunnel": "0.0.6"
}
},
"@octokit/auth-token": {
"version": "2.4.5",
"resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.4.5.tgz",
"integrity": "sha512-BpGYsPgJt05M7/L/5FoE1PiAbdxXFZkX/3kDYcsvd1v6UhlnE5e96dTDr0ezX/EFwciQxf3cNV0loipsURU+WA==",
"dev": true,
"requires": {
"@octokit/types": "^6.0.3"
}
},
"@octokit/core": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/@octokit/core/-/core-3.4.0.tgz",
"integrity": "sha512-6/vlKPP8NF17cgYXqucdshWqmMZGXkuvtcrWCgU5NOI0Pl2GjlmZyWgBMrU8zJ3v2MJlM6++CiB45VKYmhiWWg==",
"dev": true,
"requires": {
"@octokit/auth-token": "^2.4.4",
"@octokit/graphql": "^4.5.8",
"@octokit/request": "^5.4.12",
"@octokit/request-error": "^2.0.5",
"@octokit/types": "^6.0.3",
"before-after-hook": "^2.2.0",
"universal-user-agent": "^6.0.0"
}
},
"@octokit/endpoint": {
"version": "6.0.11",
"resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-6.0.11.tgz",
"integrity": "sha512-fUIPpx+pZyoLW4GCs3yMnlj2LfoXTWDUVPTC4V3MUEKZm48W+XYpeWSZCv+vYF1ZABUm2CqnDVf1sFtIYrj7KQ==",
"dev": true,
"requires": {
"@octokit/types": "^6.0.3",
"is-plain-object": "^5.0.0",
"universal-user-agent": "^6.0.0"
}
},
"@octokit/graphql": {
"version": "4.6.1",
"resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.6.1.tgz",
"integrity": "sha512-2lYlvf4YTDgZCTXTW4+OX+9WTLFtEUc6hGm4qM1nlZjzxj+arizM4aHWzBVBCxY9glh7GIs0WEuiSgbVzv8cmA==",
"dev": true,
"requires": {
"@octokit/request": "^5.3.0",
"@octokit/types": "^6.0.3",
"universal-user-agent": "^6.0.0"
}
},
"@octokit/openapi-types": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-6.0.0.tgz",
"integrity": "sha512-CnDdK7ivHkBtJYzWzZm7gEkanA7gKH6a09Eguz7flHw//GacPJLmkHA3f3N++MJmlxD1Fl+mB7B32EEpSCwztQ=="
},
"@octokit/plugin-paginate-rest": {
"version": "2.13.3",
"resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.13.3.tgz",
"integrity": "sha512-46lptzM9lTeSmIBt/sVP/FLSTPGx6DCzAdSX3PfeJ3mTf4h9sGC26WpaQzMEq/Z44cOcmx8VsOhO+uEgE3cjYg==",
"dev": true,
"requires": {
"@octokit/types": "^6.11.0"
}
},
"@octokit/plugin-rest-endpoint-methods": {
"version": "4.15.0",
"resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-4.15.0.tgz",
"integrity": "sha512-1AF9GM/Ywk8ukUM5seDRj286GdFpdfsHeOrOPBV2rVtRN7MQNzRIcw8W5sb4JPerjQ0WcRRwAwQyufg64BxJkA==",
"dev": true,
"requires": {
"@octokit/types": "^6.13.0",
"deprecation": "^2.3.1"
}
},
"@octokit/plugin-throttling": {
"version": "3.4.1",
"resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-3.4.1.tgz",
"integrity": "sha512-qCQ+Z4AnL9OrXvV59EH3GzPxsB+WyqufoCjiCJXJxTbnt3W+leXbXw5vHrMp4NG9ltw00McFWIxIxNQAzLNoTA==",
"requires": {
"@octokit/types": "^6.0.1",
"bottleneck": "^2.15.3"
}
},
"@octokit/request": {
"version": "5.4.15",
"resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.4.15.tgz",
"integrity": "sha512-6UnZfZzLwNhdLRreOtTkT9n57ZwulCve8q3IT/Z477vThu6snfdkBuhxnChpOKNGxcQ71ow561Qoa6uqLdPtag==",
"dev": true,
"requires": {
"@octokit/endpoint": "^6.0.1",
"@octokit/request-error": "^2.0.0",
"@octokit/types": "^6.7.1",
"is-plain-object": "^5.0.0",
"node-fetch": "^2.6.1",
"universal-user-agent": "^6.0.0"
}
},
"@octokit/request-error": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-2.0.5.tgz",
"integrity": "sha512-T/2wcCFyM7SkXzNoyVNWjyVlUwBvW3igM3Btr/eKYiPmucXTtkxt2RBsf6gn3LTzaLSLTQtNmvg+dGsOxQrjZg==",
"dev": true,
"requires": {
"@octokit/types": "^6.0.3",
"deprecation": "^2.0.0",
"once": "^1.4.0"
}
},
"@octokit/types": {
"version": "6.13.0",
"resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.13.0.tgz",
"integrity": "sha512-W2J9qlVIU11jMwKHUp5/rbVUeErqelCsO5vW5PKNb7wAXQVUz87Rc+imjlEvpvbH8yUb+KHmv8NEjVZdsdpyxA==",
"requires": {
"@octokit/openapi-types": "^6.0.0"
}
},
"@vercel/ncc": {
"version": "0.27.0",
"resolved": "https://registry.npmjs.org/@vercel/ncc/-/ncc-0.27.0.tgz",
"integrity": "sha512-DllIJQapnU2YwewIhh/4dYesmMQw3h2cFtabECc/zSJHqUbNa0eJuEkRa6DXbZvh1YPWBtYQoPV17NlDpBw1Vw==",
"dev": true
},
"adm-zip": {
"version": "0.5.5",
"resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.5.tgz",
"integrity": "sha512-IWwXKnCbirdbyXSfUDvCCrmYrOHANRZcc8NcRrvTlIApdl7PwE9oGcsYvNeJPAVY1M+70b4PxXGKIf8AEuiQ6w==",
"dev": true
},
"before-after-hook": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.1.tgz",
"integrity": "sha512-/6FKxSTWoJdbsLDF8tdIjaRiFXiE6UHsEHE3OPI/cwPURCVi1ukP0gmLn7XWEiFk5TcwQjjY5PWsU+j+tgXgmw==",
"dev": true
},
"bottleneck": {
"version": "2.19.5",
"resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz",
"integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw=="
},
"deprecation": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz",
"integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==",
"dev": true
},
"filesize": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/filesize/-/filesize-6.1.0.tgz",
"integrity": "sha512-LpCHtPQ3sFx67z+uh2HnSyWSLLu5Jxo21795uRDuar/EOuYWXib5EmPaGIBuSnRqH2IODiKA2k5re/K9OnN/Yg==",
"dev": true
},
"is-plain-object": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz",
"integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==",
"dev": true
},
"node-fetch": {
"version": "2.6.1",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
"integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==",
"dev": true
},
"once": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
"dev": true,
"requires": {
"wrappy": "1"
}
},
"tunnel": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",
"integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==",
"dev": true
},
"universal-user-agent": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz",
"integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==",
"dev": true
},
"wrappy": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
"dev": true
}
}
}

Просмотреть файл

@ -4,6 +4,7 @@
"devDependencies": {
"@actions/core": "^1.2.6",
"@actions/github": "^4.0.0",
"@octokit/plugin-throttling": "^3.4.1",
"@vercel/ncc": "^0.27.0",
"adm-zip": "^0.5.2",
"filesize": "^6.1.0"

Просмотреть файл

@ -23,3 +23,8 @@ runs:
which python3
python3 --version
python3 -c "import sysconfig; print(sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET'))"
- shell: bash
name: Set up venv with upstream Python
run: |
python3 -m venv /tmp/venv
echo "/tmp/venv/bin" >> $GITHUB_PATH

4
.github/actions/numpy_vers/action.yml поставляемый
Просмотреть файл

@ -16,6 +16,7 @@ runs:
steps:
- id: numpy
run: |
set -ex
NUMPY_BUILD_VERSION="==1.7.0"
NUMPY_DEP_VERSION=">=1.7.0"
@ -61,8 +62,7 @@ runs:
esac
;;
# TODO: 'Windows*' might not be good
Windows*:x86_64)
${CI_MSYS_VERSION}:x86_64)
case "${{ inputs.pyver }}" in
3.5*)
NUMPY_BUILD_VERSION="==1.11.0"

12
.github/actions/win-install-sox/action.yml поставляемый Normal file
Просмотреть файл

@ -0,0 +1,12 @@
name: "Install SoX and add to PATH"
description: "Install SoX and add to PATH"
runs:
using: "composite"
steps:
- run: |
set -ex
wget https://sourceforge.net/projects/sox/files/sox/14.4.2/sox-14.4.2-win32.zip/download -O sox-14.4.2-win32.zip
"C:/Program Files/7-Zip/7z.exe" x -o`pwd`/bin/ -tzip -aoa sox-14.4.2-win32.zip
rm sox-*zip
echo "`pwd`/bin/sox-14.4.2/" >> $GITHUB_PATH
shell: bash

1440
.github/workflows/build-and-test.yml поставляемый Normal file

Разница между файлами не показана из-за своего большого размера Загрузить разницу

681
.github/workflows/macOS-amd64.yml поставляемый
Просмотреть файл

@ -1,681 +0,0 @@
name: "macOS amd64"
on:
pull_request:
push:
branches:
- master
env:
CI_TASK_DIR: ${{ github.workspace }}
CI_ARTIFACTS_DIR: ${{ github.workspace }}/artifacts
CI_NODE_MODULES_NTH: 1
MACOSX_DEPLOYMENT_TARGET: "10.10"
jobs:
swig_macOS:
name: "Build SWIG for macOS"
runs-on: macos-10.15
env:
swig_hash: "90cdbee6a69d13b39d734083b9f91069533b0d7b"
steps:
- uses: actions/checkout@v2
with:
repository: "swig/swig"
ref: ${{ env.swig_hash }}
- run: |
mkdir -p build-static/
- uses: actions/cache@v2
id: swig-build-cache
with:
path: build-static/
key: swig-${{ runner.os }}-${{ env.swig_hash }}
- run: |
brew install automake
if: steps.swig-build-cache.outputs.cache-hit != 'true'
- run: |
curl -sSL https://ftp.pcre.org/pub/pcre/pcre-8.43.tar.gz > pcre-8.43.tar.gz
if: steps.swig-build-cache.outputs.cache-hit != 'true'
- run: |
./Tools/pcre-build.sh
if: steps.swig-build-cache.outputs.cache-hit != 'true'
- run: |
sh autogen.sh
./configure \
--prefix=${{ github.workspace }}/build-static/ \
--program-prefix=ds-
if: steps.swig-build-cache.outputs.cache-hit != 'true'
- run: |
make -j
if: steps.swig-build-cache.outputs.cache-hit != 'true'
- run: |
make install
if: steps.swig-build-cache.outputs.cache-hit != 'true'
- uses: actions/upload-artifact@v2
with:
name: ${{ github.job }}
path: ${{ github.workspace }}/build-static/
build-ctc-decoder:
name: "Build CTC decoder Python package for testing"
needs: [ swig_macOS ]
runs-on: macos-10.15
strategy:
matrix:
python-version: [3.6.8]
if: ${{ github.event_name == 'pull_request' }}
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- uses: ./.github/actions/install-python-upstream
with:
version: ${{ matrix.python-version }}
- run: |
python --version
pip --version
- uses: actions/download-artifact@v2
with:
name: "swig_macOS"
path: ${{ github.workspace }}/native_client/ds-swig/
- run: |
ls -hal ${{ github.workspace }}/native_client/ds-swig/bin
ln -s ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
chmod +x ${{ github.workspace }}/native_client/ds-swig/bin/ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
- id: get_numpy
uses: ./.github/actions/numpy_vers
with:
pyver: ${{ matrix.python-version }}
- run: |
NUMPY_BUILD_VERSION=${{ steps.get_numpy.outputs.build_version }} \
NUMPY_DEP_VERSION=${{ steps.get_numpy.outputs.dep_version }} \
make -C native_client/ctcdecode/ \
NUM_PROCESSES=$(sysctl hw.ncpu |cut -d' ' -f2) \
bindings
- uses: actions/upload-artifact@v2
with:
name: "ds_ctcdecoder-test.whl"
path: ${{ github.workspace }}/native_client/ctcdecode/dist/*.whl
- run: |
make -C native_client/ctcdecode clean-keep-third-party
train-test-model:
name: "Train a test model"
needs: [ "build-ctc-decoder" ]
runs-on: macos-10.15
if: ${{ github.event_name == 'pull_request' }}
strategy:
matrix:
bitrate: ["8k", "16k"]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 1
- uses: actions/setup-python@v2
with:
python-version: 3.6
- uses: actions/download-artifact@v2
with:
name: "ds_ctcdecoder-test.whl"
- run: |
python --version
pip --version
- run: |
pip install --upgrade pip==19.3.1 setuptools==45.0.0 wheel==0.33.6
- run: |
pip install ds_ctcdecoder-*-cp36-cp36m-*_x86_64.whl
DS_NODECODER=y pip install --upgrade .
- run: |
bits=""
if [ "${{ matrix.bitrate }}" = "8k" ]; then
bits=8000
fi
if [ "${{ matrix.bitrate }}" = "16k" ]; then
bits=16000
fi
# Easier to rename to that we can exercize the LDC93S1 importer code to
# generate the CSV file.
echo "Moving ${bits} to LDC93S1.wav"
mv data/smoke_test/LDC93S1_pcms16le_1_${bits}.wav data/smoke_test/LDC93S1.wav
./bin/run-ci-ldc93s1_new.sh 249 ${bits}
./bin/run-ci-ldc93s1_tflite.sh ${bits}
- run: |
curl -vsSL https://github.com/mozilla/DeepSpeech/releases/download/v0.9.3/macOS.amd64.convert_graphdef_memmapped_format.xz | xz -d > /tmp/convert_graphdef_memmapped_format
chmod +x /tmp/convert_graphdef_memmapped_format
/tmp/convert_graphdef_memmapped_format --in_graph=/tmp/train/output_graph.pb --out_graph=/tmp/train/output_graph.pbmm
- run: |
tar -cf - \
-C /tmp/ckpt/ . \
| xz -9 -T0 > /tmp/checkpoint.tar.xz
- run: |
mkdir -p ${{ github.workspace }}/tmp/
cp /tmp/train*/output_graph.* /tmp/checkpoint.tar.xz ${{ github.workspace }}/tmp/
- run: |
ls -hal /tmp/ ${{ github.workspace }}/tmp/
- uses: actions/upload-artifact@v2
with:
name: "test-model.tf-${{ matrix.bitrate }}.zip"
path: ${{ github.workspace }}/tmp/output_graph.pb*
- uses: actions/upload-artifact@v2
with:
name: "test-model.tflite-${{ matrix.bitrate }}.zip"
path: ${{ github.workspace }}/tmp/output_graph.tflite
- uses: actions/upload-artifact@v2
with:
name: "test-checkpoint.${{ matrix.bitrate }}.zip"
path: ${{ github.workspace }}/tmp/checkpoint.tar.xz
tensorflow_opt-macOS:
name: "Check cache for TensorFlow"
runs-on: ubuntu-20.04
outputs:
status: ${{ steps.check_artifact_exists.outputs.status }}
cache_key: ${{ steps.get_cache_key.outputs.key }}
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 1
- id: get_cache_key
uses: ./.github/actions/get_cache_key
with:
extras: "2"
- id: check_artifact_exists
uses: ./.github/actions/check_artifact_exists
with:
name: ${{ steps.get_cache_key.outputs.key }}
build-tensorflow-macOS:
name: "Build TensorFlow (opt) for macOS"
needs: tensorflow_opt-macOS
runs-on: macos-10.15
steps:
- run: true
if: needs.tensorflow_opt-macOS.outputs.status == 'found'
- uses: actions/checkout@v2
with:
fetch-depth: 0
submodules: 'recursive'
if: needs.tensorflow_opt-macOS.outputs.status == 'missing'
- uses: ./.github/actions/select-xcode
with:
version: "12.1.1"
if: needs.tensorflow_opt-macOS.outputs.status == 'missing'
- uses: ./.github/actions/setup-tensorflow
if: needs.tensorflow_opt-macOS.outputs.status == 'missing'
- uses: ./.github/actions/build-tensorflow
with:
flavor: "--darwin-cpu"
if: needs.tensorflow_opt-macOS.outputs.status == 'missing'
- uses: ./.github/actions/package-tensorflow
if: needs.tensorflow_opt-macOS.outputs.status == 'missing'
- uses: actions/upload-artifact@v2
with:
name: ${{ needs.tensorflow_opt-macOS.outputs.cache_key }}
path: ${{ github.workspace }}/artifacts/home.tar.xz
if: needs.tensorflow_opt-macOS.outputs.status == 'missing'
build-lib_macOS:
name: "Build libdeepspeech.so and deepspeech C++ binary"
runs-on: macos-10.15
needs: [ build-tensorflow-macOS, tensorflow_opt-macOS ]
strategy:
matrix:
build-flavor: ["tf", "tflite"]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- uses: actions/download-artifact@v2
with:
name: ${{ needs.tensorflow_opt-macOS.outputs.cache_key }}
path: ${{ github.workspace }}/
if: needs.tensorflow_opt-macOS.outputs.status == 'missing'
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-macOS.outputs.cache_key }}
path: ${{ github.workspace }}/
download: true
if: needs.tensorflow_opt-macOS.outputs.status == 'found'
- run: |
tar xkf ${{ github.workspace }}/home.tar.xz
rm ${{ github.workspace }}/home.tar.xz
- run: |
git status
- uses: ./.github/actions/select-xcode
with:
version: "12.1.1"
- uses: ./.github/actions/host-build
with:
flavor: ${{ matrix.build-flavor }}
- uses: ./.github/actions/package
- uses: actions/upload-artifact@v2
with:
name: "native_client.${{ matrix.build-flavor }}.tar.xz"
path: ${{ github.workspace }}/artifacts/native_client.tar.xz
- uses: actions/upload-artifact@v2
with:
name: "libdeepspeech.${{ matrix.build-flavor }}.zip"
path: ${{ github.workspace }}/artifacts/libdeepspeech.zip
build-python-macOS:
name: "Build python bindings for macOS"
runs-on: macos-10.15
needs: [ build-lib_macOS, swig_macOS ]
strategy:
matrix:
build-flavor: ["tf", "tflite"]
python-version: [3.6.8, 3.7.9, 3.8.8, 3.9.2]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 1
- uses: actions/download-artifact@v2
with:
name: "native_client.${{ matrix.build-flavor }}.tar.xz"
path: ${{ github.workspace }}/tensorflow/bazel-bin/native_client/
- run: |
cd ${{ github.workspace }}/tensorflow/bazel-bin/native_client/
tar xf native_client.tar.xz
ls -hal
cd ${{ github.workspace }}/
- uses: actions/download-artifact@v2
with:
name: "swig_macOS"
path: ${{ github.workspace }}/native_client/ds-swig/
- run: |
ls -hal ${{ github.workspace }}/native_client/ds-swig/bin
ln -s ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
chmod +x ${{ github.workspace }}/native_client/ds-swig/bin/ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
- uses: ./.github/actions/install-python-upstream
with:
version: ${{ matrix.python-version }}
# GitHub packaged version are limited to macOS deployment target 10.14
#- uses: actions/setup-python@v2
# with:
# python-version: ${{ matrix.python-version }}
- id: get_numpy
uses: ./.github/actions/numpy_vers
with:
pyver: ${{ matrix.python-version }}
- uses: ./.github/actions/python-build
with:
build_flavor: ${{ matrix.build-flavor }}
numpy_build: "${{ steps.get_numpy.outputs.build_version }}"
numpy_dep: "${{ steps.get_numpy.outputs.dep_version }}"
- uses: actions/upload-artifact@v2
with:
name: "deepspeech-${{ matrix.build-flavor }}-${{ matrix.python-version }}.whl"
path: ${{ github.workspace }}/wheels/*.whl
build-nodejs-macOS:
name: "Build NodeJS and ElectronJS for macOS"
runs-on: macos-10.15
needs: [ build-lib_macOS, swig_macOS ]
strategy:
matrix:
build-flavor: ["tf", "tflite"]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 1
- uses: actions/download-artifact@v2
with:
name: "native_client.${{ matrix.build-flavor }}.tar.xz"
path: ${{ github.workspace }}/tensorflow/bazel-bin/native_client/
- run: |
cd ${{ github.workspace }}/tensorflow/bazel-bin/native_client/
tar xf native_client.tar.xz
ls -hal
cd ${{ github.workspace }}/
- uses: actions/download-artifact@v2
with:
name: "swig_macOS"
path: ${{ github.workspace }}/native_client/ds-swig/
- run: |
ls -hal ${{ github.workspace }}/native_client/ds-swig/bin
ln -s ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
chmod +x ${{ github.workspace }}/native_client/ds-swig/bin/ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
- uses: actions/setup-node@v2
with:
node-version: 12
- uses: actions/cache@v2
id: node-headers-cache
with:
path: native_client/javascript/headers/nodejs/
key: node-headers-10.0.0_15.0.0
- uses: actions/cache@v2
id: electron-headers-cache
with:
path: native_client/javascript/headers/electronjs/
key: electron-headers-5.0.13_12.0.0
- uses: ./.github/actions/node-build
with:
nodejs_versions: "10.0.0 11.0.0 12.7.0 13.0.0 14.0.0 15.0.0"
electronjs_versions: "5.0.13 6.0.12 6.1.7 7.0.1 7.1.8 8.0.1 9.0.1 9.1.0 9.2.0 10.0.0 10.1.0 11.0.0 12.0.0"
- uses: actions/upload-artifact@v2
with:
name: "nodewrapper-${{ matrix.build-flavor }}-macOS_amd64.tar.gz"
path: ${{ github.workspace }}/native_client/javascript/wrapper.tar.gz
- uses: actions/upload-artifact@v2
with:
name: "deepspeech_intermediate-${{ matrix.build-flavor }}.tgz"
path: ${{ github.workspace }}/native_client/javascript/deepspeech-*.tgz
test-cpp-macOS:
name: "Test C++ binary on macOS"
runs-on: macos-10.15
needs: [ build-lib_macOS, train-test-model ]
if: ${{ github.event_name == 'pull_request' }}
strategy:
matrix:
build-flavor: ["tf", "tflite"]
models: ["test", "prod"]
bitrate: ["8k", "16k"]
env:
CI_TMP_DIR: ${{ github.workspace }}/tmp/
DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb
DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm
DEEPSPEECH_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 1
- uses: actions/download-artifact@v2
with:
name: "native_client.${{ matrix.build-flavor }}.tar.xz"
path: ${{ env.CI_TMP_DIR }}
- run: |
cd ${{ env.CI_TMP_DIR }}
mkdir ds && cd ds && tar xf ../native_client.tar.xz
- uses: actions/download-artifact@v2
with:
name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip"
path: ${{ env.CI_TMP_DIR }}
if: matrix.models == 'test'
- run: |
ls -hal ${{ env.CI_TMP_DIR }}/
if: matrix.models == 'test'
- uses: ./.github/actions/run-tests
with:
runtime: "cpp"
build-flavor: ${{ matrix.build-flavor }}
bitrate: ${{ matrix.bitrate }}
model-kind: ${{ matrix.models }}
test-py-macOS:
name: "Test Python bindings on macOS"
runs-on: macos-10.15
needs: [ build-python-macOS, train-test-model ]
if: ${{ github.event_name == 'pull_request' }}
strategy:
matrix:
python-version: [3.6.8, 3.7.9, 3.8.8, 3.9.2]
build-flavor: ["tf", "tflite"]
models: ["test", "prod"]
bitrate: ["8k", "16k"]
env:
CI_TMP_DIR: ${{ github.workspace }}/tmp/
DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb
DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm
DEEPSPEECH_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 1
- uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- uses: actions/download-artifact@v2
with:
name: "deepspeech-${{ matrix.build-flavor }}-${{ matrix.python-version }}.whl"
path: ${{ env.CI_TMP_DIR }}
- uses: actions/download-artifact@v2
with:
name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip"
path: ${{ env.CI_TMP_DIR }}
if: matrix.models == 'test'
- run: |
ls -hal ${{ env.CI_TMP_DIR }}/
if: matrix.models == 'test'
- run: |
ls -hal ${{ env.CI_TMP_DIR }}/
pip3 install --only-binary :all: --upgrade ${{ env.CI_TMP_DIR }}/deepspeech*.whl
- uses: ./.github/actions/run-tests
with:
runtime: "python"
build-flavor: ${{ matrix.build-flavor }}
bitrate: ${{ matrix.bitrate }}
model-kind: ${{ matrix.models }}
repackage-nodejs-allplatforms:
name: "Repackage NodeJS / ElectronJS for multiplatforms"
runs-on: ubuntu-20.04
needs: [ build-nodejs-macOS ]
strategy:
matrix:
build-flavor: ["tf", "tflite"]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 1
- run: |
mkdir -p /tmp/nodewrapper-${{ matrix.build-flavor }}-macOS_amd64/
- uses: actions/download-artifact@v2
with:
name: "nodewrapper-${{ matrix.build-flavor }}-macOS_amd64.tar.gz"
path: /tmp/nodewrapper-macOS_amd64/
- run: |
tar -C ${{ github.workspace }}/native_client/javascript -xzvf /tmp/nodewrapper-macOS_amd64/wrapper.tar.gz
- run: |
PROJECT_NAME="deepspeech"
if [ "${{ matrix.build-flavor }}" = "tflite" ]; then
PROJECT_NAME="deepspeech-tflite"
fi
make -C native_client/javascript clean npm-pack PROJECT_NAME=$PROJECT_NAME
- uses: actions/upload-artifact@v2
with:
name: "deepspeech-${{ matrix.build-flavor }}.tgz"
path: ${{ github.workspace }}/native_client/javascript/deepspeech-*.tgz
test-nodejs-macOS:
name: "Test NodeJS bindings on macOS"
runs-on: macos-10.15
needs: [ build-nodejs-macOS, train-test-model ]
if: ${{ github.event_name == 'pull_request' }}
strategy:
matrix:
# https://nodejs.org/en/about/releases/
nodejs-version: [10, 12, 14, 15]
build-flavor: ["tf", "tflite"]
models: ["test"]
bitrate: ["16k"]
env:
CI_TMP_DIR: ${{ github.workspace }}/tmp/
DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb
DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm
DEEPSPEECH_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 1
- uses: actions/setup-node@v2
with:
node-version: ${{ matrix.nodejs-version }}
- uses: actions/download-artifact@v2
with:
name: "deepspeech_intermediate-${{ matrix.build-flavor }}.tgz"
path: ${{ env.CI_TMP_DIR }}
- uses: actions/download-artifact@v2
with:
name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip"
path: ${{ env.CI_TMP_DIR }}
if: matrix.models == 'test'
- run: |
ls -hal ${{ env.CI_TMP_DIR }}/
if: matrix.models == 'test'
- uses: actions/cache@v2
id: node-modules-cache
with:
path: ~/.npm/
key: node-modules-${{ matrix.build-flavor }}-${{ runner.os }}-${{ env.CI_NODE_MODULES_NTH }}
- run: |
ls -hal ${{ env.CI_TMP_DIR }}/
npm install --verbose ${{ env.CI_TMP_DIR }}/deepspeech*.tgz
- run: |
ls -hal node_modules/deepspeech* node_modules/.bin/
- uses: ./.github/actions/run-tests
with:
runtime: "node"
build-flavor: ${{ matrix.build-flavor }}
bitrate: ${{ matrix.bitrate }}
model-kind: ${{ matrix.models }}
test-electronjs-macOS:
name: "Test ElectronJS bindings on macOS"
runs-on: macos-10.15
needs: [ build-nodejs-macOS, train-test-model ]
if: ${{ github.event_name == 'pull_request' }}
strategy:
matrix:
electronjs-version: [5.0.13, 6.1.7, 7.1.8, 8.0.1, 9.2.0, 10.1.0, 11.0.0, 12.0.0]
build-flavor: ["tf", "tflite"]
models: ["test"]
bitrate: ["16k"]
env:
CI_TMP_DIR: ${{ github.workspace }}/tmp/
DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb
DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm
DEEPSPEECH_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 1
- uses: actions/setup-node@v2
with:
node-version: 12
- uses: actions/download-artifact@v2
with:
name: "deepspeech_intermediate-${{ matrix.build-flavor }}.tgz"
path: ${{ env.CI_TMP_DIR }}
- uses: actions/download-artifact@v2
with:
name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip"
path: ${{ env.CI_TMP_DIR }}
if: matrix.models == 'test'
- run: |
ls -hal ${{ env.CI_TMP_DIR }}/
if: matrix.models == 'test'
- uses: actions/cache@v2
id: electron-modules-cache
with:
path: ~/.npm/
key: electron-modules-${{ matrix.build-flavor }}-${{ runner.os }}-${{ env.CI_NODE_MODULES_NTH }}
- run: |
ls -hal ${{ env.CI_TMP_DIR }}/
npm install ${{ env.CI_TMP_DIR }}/deepspeech*.tgz
- run: |
npm install electron@${{ matrix.electronjs-version }}
- uses: ./.github/actions/run-tests
with:
runtime: "electronjs"
build-flavor: ${{ matrix.build-flavor }}
bitrate: ${{ matrix.bitrate }}
model-kind: ${{ matrix.models }}
timeout-minutes: 5
test-nodejs_all-macOS:
name: "Test MultiArchPlatform NodeJS bindings on macOS"
runs-on: macos-10.15
needs: [ repackage-nodejs-allplatforms, train-test-model ]
if: ${{ github.event_name == 'pull_request' }}
strategy:
matrix:
# https://nodejs.org/en/about/releases/
nodejs-version: [10, 15]
build-flavor: ["tf", "tflite"]
models: ["test", "prod"]
bitrate: ["8k", "16k"]
env:
CI_TMP_DIR: ${{ github.workspace }}/tmp/
DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb
DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm
DEEPSPEECH_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 1
- uses: actions/setup-node@v2
with:
node-version: ${{ matrix.nodejs-version }}
- uses: actions/download-artifact@v2
with:
name: "deepspeech-${{ matrix.build-flavor }}.tgz"
path: ${{ env.CI_TMP_DIR }}
- uses: actions/download-artifact@v2
with:
name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip"
path: ${{ env.CI_TMP_DIR }}
if: matrix.models == 'test'
- run: |
ls -hal ${{ env.CI_TMP_DIR }}/
if: matrix.models == 'test'
- uses: actions/cache@v2
id: node-modules-cache
with:
path: ~/.npm/
key: node-modules-${{ matrix.build-flavor }}-${{ runner.os }}-${{ env.CI_NODE_MODULES_NTH }}
- run: |
ls -hal ${{ env.CI_TMP_DIR }}/
npm install --verbose ${{ env.CI_TMP_DIR }}/deepspeech*.tgz
- run: |
ls -hal node_modules/deepspeech* node_modules/.bin/
- uses: ./.github/actions/run-tests
with:
runtime: "node"
build-flavor: ${{ matrix.build-flavor }}
bitrate: ${{ matrix.bitrate }}
model-kind: ${{ matrix.models }}
test-electronjs_all-macOS:
name: "Test MultiArchPlatform ElectronJS bindings on macOS"
runs-on: macos-10.15
needs: [ repackage-nodejs-allplatforms, train-test-model ]
if: ${{ github.event_name == 'pull_request' }}
strategy:
matrix:
electronjs-version: [5.0.13, 12.0.0]
build-flavor: ["tf", "tflite"]
models: ["test", "prod"]
bitrate: ["8k", "16k"]
env:
CI_TMP_DIR: ${{ github.workspace }}/tmp/
DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb
DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm
DEEPSPEECH_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 1
- uses: actions/setup-node@v2
with:
node-version: 12
- uses: actions/download-artifact@v2
with:
name: "deepspeech-${{ matrix.build-flavor }}.tgz"
path: ${{ env.CI_TMP_DIR }}
- uses: actions/download-artifact@v2
with:
name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip"
path: ${{ env.CI_TMP_DIR }}
if: matrix.models == 'test'
- run: |
ls -hal ${{ env.CI_TMP_DIR }}/
if: matrix.models == 'test'
- uses: actions/cache@v2
id: electron-modules-cache
with:
path: ~/.npm/
key: electron-modules-${{ matrix.build-flavor }}-${{ runner.os }}-${{ env.CI_NODE_MODULES_NTH }}
- run: |
ls -hal ${{ env.CI_TMP_DIR }}/
npm install ${{ env.CI_TMP_DIR }}/deepspeech*.tgz
- run: |
npm install electron@${{ matrix.electronjs-version }}
- uses: ./.github/actions/run-tests
with:
runtime: "electronjs"
build-flavor: ${{ matrix.build-flavor }}
bitrate: ${{ matrix.bitrate }}
model-kind: ${{ matrix.models }}
timeout-minutes: 5

1
.gitignore поставляемый
Просмотреть файл

@ -40,4 +40,3 @@ doc/xml-dotnet
convert_graphdef_memmapped_format
native_client/swift/deepspeech_ios.framework/deepspeech_ios
.github/actions/check_artifact_exists/node_modules/
.github/actions/check_artifact_exists/package-lock.json

Просмотреть файл

@ -109,7 +109,15 @@ verify_bazel_rebuild()
symlink_electron()
{
ln -s Electron.app/Contents/MacOS/Electron node_modules/electron/dist/node
if [ "${OS}" = "Darwin" ]; then
ln -s Electron.app/Contents/MacOS/Electron node_modules/electron/dist/node
else
ln -s electron "node_modules/electron/dist/node"
if [ "${OS}" = "Linux" -a -f "node_modules/electron/dist/chrome-sandbox" ]; then
export ELECTRON_DISABLE_SANDBOX=1
fi
fi
}
export_node_bin_path()

Просмотреть файл

@ -10,6 +10,7 @@ if [ "${OS}" = "Linux" ]; then
fi;
if [ "${OS}" = "${CI_MSYS_VERSION}" ]; then
export CI_TASK_DIR="$(cygpath ${CI_TASK_DIR})"
export DS_ROOT_TASK=${CI_TASK_DIR}
export PYENV_ROOT="${CI_TASK_DIR}/pyenv-root"
export PLATFORM_EXE_SUFFIX=.exe

Просмотреть файл

@ -3,7 +3,10 @@
set -xe
strip() {
echo "$(echo $1 | sed -e 's/^[[:space:]]+//' -e 's/[[:space:]]+$//')"
# We strip leading carriage return due to ElectronJS on Windows producing stray
# characters before its output intermittently.
# Then we strip leading and trailing whitespace.
echo "$(echo $1 | tr -d $'\r' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
}
# This verify exact inference result
@ -220,6 +223,17 @@ assert_correct_ldc93s1_prodmodel()
fi;
}
assert_working_ldc93s1_prodmodel()
{
if [ -z "$3" -o "$3" = "16k" ]; then
assert_working_inference "$1" "she had your dark suit in greasy wash water all year" "$2"
fi
if [ "$3" = "8k" ]; then
assert_working_inference "$1" "she had to do suit in greasy wash water all year" "$2"
fi
}
assert_correct_ldc93s1_prodtflitemodel()
{
if [ -z "$3" -o "$3" = "16k" ]; then
@ -231,16 +245,37 @@ assert_correct_ldc93s1_prodtflitemodel()
fi;
}
assert_working_ldc93s1_prodtflitemodel()
{
if [ -z "$3" -o "$3" = "16k" ]; then
assert_working_inference "$1" "she had her dark suit in greasy wash water all year" "$2"
fi;
if [ "$3" = "8k" ]; then
assert_working_inference "$1" "she had to do so and greasy wash water all year" "$2"
fi;
}
assert_correct_ldc93s1_prodmodel_stereo_44k()
{
assert_correct_inference "$1" "she had your dark suit in greasy wash water all year" "$2"
}
assert_working_ldc93s1_prodmodel_stereo_44k()
{
assert_working_inference "$1" "she had your dark suit in greasy wash water all year" "$2"
}
assert_correct_ldc93s1_prodtflitemodel_stereo_44k()
{
assert_correct_inference "$1" "she had her dark suit in greasy wash water all year" "$2"
}
assert_working_ldc93s1_prodtflitemodel_stereo_44k()
{
assert_working_inference "$1" "she had her dark suit in greasy wash water all year" "$2"
}
assert_correct_warning_upsampling()
{
assert_shows_something "$1" "erratic speech recognition"
@ -478,6 +513,31 @@ run_prod_inference_tests()
fi;
}
# Equivalent to run_prod_inference_tests but we use assert_working* instead of assert_correct
# ElectronJS mixes stdout and stderr and exact matching is broken
run_electronjs_prod_inference_tests()
{
local _bitrate=$1
set +e
phrase_pbmodel_withlm=$(deepspeech --model ${CI_TMP_DIR}/${model_name} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} 2>${CI_TMP_DIR}/stderr)
status=$?
set -e
assert_working_ldc93s1_prodmodel "${phrase_pbmodel_withlm}" "$status" "${_bitrate}"
set +e
phrase_pbmodel_withlm=$(deepspeech --model ${CI_TMP_DIR}/${model_name_mmap} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} 2>${CI_TMP_DIR}/stderr)
status=$?
set -e
assert_working_ldc93s1_prodmodel "${phrase_pbmodel_withlm}" "$status" "${_bitrate}"
set +e
phrase_pbmodel_withlm_stereo_44k=$(deepspeech --model ${CI_TMP_DIR}/${model_name_mmap} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${CI_TMP_DIR}/stderr)
status=$?
set -e
assert_working_ldc93s1_prodmodel_stereo_44k "${phrase_pbmodel_withlm_stereo_44k}" "$status"
}
run_prodtflite_inference_tests()
{
local _bitrate=$1
@ -509,6 +569,31 @@ run_prodtflite_inference_tests()
fi;
}
# Equivalent to run_prodtflite_inference_tests but we use assert_working* instead of assert_correct
# ElectronJS mixes stdout and stderr and exact matching is broken
run_electronjs_prodtflite_inference_tests()
{
local _bitrate=$1
set +e
phrase_pbmodel_withlm=$(deepspeech --model ${CI_TMP_DIR}/${model_name} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} 2>${CI_TMP_DIR}/stderr)
status=$?
set -e
assert_working_ldc93s1_prodtflitemodel "${phrase_pbmodel_withlm}" "$status" "${_bitrate}"
set +e
phrase_pbmodel_withlm=$(deepspeech --model ${CI_TMP_DIR}/${model_name_mmap} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} 2>${CI_TMP_DIR}/stderr)
status=$?
set -e
assert_working_ldc93s1_prodtflitemodel "${phrase_pbmodel_withlm}" "$status" "${_bitrate}"
set +e
phrase_pbmodel_withlm_stereo_44k=$(deepspeech --model ${CI_TMP_DIR}/${model_name_mmap} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${CI_TMP_DIR}/stderr)
status=$?
set -e
assert_working_ldc93s1_prodtflitemodel_stereo_44k "${phrase_pbmodel_withlm_stereo_44k}" "$status"
}
run_multi_inference_tests()
{
set +e -o pipefail

Просмотреть файл

@ -0,0 +1,23 @@
#!/bin/bash
set -xe
source $(dirname "$0")/all-vars.sh
source $(dirname "$0")/all-utils.sh
source $(dirname "$0")/asserts.sh
bitrate=$1
set_ldc_sample_filename "${bitrate}"
model_source=${DEEPSPEECH_TEST_MODEL//.pb/.tflite}
model_name=$(basename "${model_source}")
model_name_mmap=$(basename "${model_source}")
export DATA_TMP_DIR=${CI_TMP_DIR}
download_material "${CI_TMP_DIR}/ds"
export PATH=${CI_TMP_DIR}/ds/:$PATH
check_versions
run_basic_inference_tests

Просмотреть файл

@ -34,4 +34,4 @@ deepspeech --version
check_runtime_electronjs
run_prod_inference_tests "${bitrate}"
run_electronjs_prod_inference_tests "${bitrate}"

Просмотреть файл

@ -34,4 +34,4 @@ deepspeech --version
check_runtime_electronjs
run_prodtflite_inference_tests "${bitrate}"
run_electronjs_prodtflite_inference_tests "${bitrate}"

Просмотреть файл

@ -26,7 +26,7 @@ package_native_client()
win_lib="-C ${tensorflow_dir}/bazel-bin/native_client/ libdeepspeech.so.if.lib"
fi;
${TAR} -cf - \
${TAR} --verbose -cf - \
-C ${tensorflow_dir}/bazel-bin/native_client/ libdeepspeech.so \
${win_lib} \
-C ${tensorflow_dir}/bazel-bin/native_client/ generate_scorer_package \
@ -60,7 +60,7 @@ package_native_client_ndk()
echo "Please specify arch abi."
fi;
${TAR} -cf - \
${TAR} --verbose -cf - \
-C ${deepspeech_dir}/native_client/libs/${arch_abi}/ deepspeech \
-C ${deepspeech_dir}/native_client/libs/${arch_abi}/ libdeepspeech.so \
-C ${tensorflow_dir}/bazel-bin/native_client/ generate_scorer_package \

Просмотреть файл

@ -1,6 +1,7 @@
#!/bin/bash
set -ex
set -o pipefail
source $(dirname $0)/tf-vars.sh
@ -21,7 +22,10 @@ pushd ${DS_ROOT_TASK}/tensorflow/
fi;
case "$1" in
"--linux-cpu"|"--darwin-cpu"|"--windows-cpu")
"--windows-cpu")
echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LIBDEEPSPEECH} ${BUILD_TARGET_LITE_LIB} --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh"
;;
"--linux-cpu"|"--darwin-cpu")
echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LIB_CPP_API} ${BUILD_TARGET_LITE_LIB}
;;
"--linux-cuda"|"--windows-cuda")

Просмотреть файл

@ -109,4 +109,4 @@ mkdir -p ${CI_ARTIFACTS_DIR} || true
# Taken from https://www.tensorflow.org/install/source
# Only future is needed for our builds, as we don't build the Python package
pip install -U --user future==0.17.1 || true
python -m pip install -U --user future==0.17.1 || true

Просмотреть файл

@ -40,9 +40,8 @@ elif [ "${OS}" = "${CI_MSYS_VERSION}" ]; then
export CI_ARTIFACTS_DIR="$(cygpath ${CI_ARTIFACTS_DIR})"
export DS_ROOT_TASK=${CI_TASK_DIR}
export BAZEL_VC='C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC'
export BAZEL_SH='C:\builds\tc-workdir\msys64\usr\bin\bash'
export TC_WIN_BUILD_PATH='C:\builds\tc-workdir\msys64\usr\bin;C:\Python36'
export BAZEL_VC="C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC"
export BAZEL_VC_FULL_VERSION="14.28.29910"
export MSYS2_ARG_CONV_EXCL='//'
mkdir -p ${CI_TASK_DIR}/tmp/
@ -119,16 +118,10 @@ export TF_NEED_ROCM=0
# This should be gcc-5, hopefully. CUDA and TensorFlow might not be happy, otherwise.
export GCC_HOST_COMPILER_PATH=/usr/bin/gcc
if [ "${OS}" = "${CI_MSYS_VERSION}" ]; then
export PYTHON_BIN_PATH=C:/Python36/python.exe
else
if [ "${OS}" = "Linux" ]; then
source /etc/os-release
if [ "${ID}" = "ubuntu" -a "${VERSION_ID}" = "20.04" ]; then
export PYTHON_BIN_PATH=/usr/bin/python3
else
export PYTHON_BIN_PATH=/usr/bin/python2.7
fi
if [ "${OS}" = "Linux" ]; then
source /etc/os-release
if [ "${ID}" = "ubuntu" -a "${VERSION_ID}" = "20.04" ]; then
export PYTHON_BIN_PATH=/usr/bin/python3
else
export PYTHON_BIN_PATH=/usr/bin/python2.7
fi
@ -186,15 +179,7 @@ fi
BAZEL_IOS_ARM64_FLAGS="--config=ios_arm64 --define=runtime=tflite --copt=-DTFLITE_WITH_RUY_GEMV"
BAZEL_IOS_X86_64_FLAGS="--config=ios_x86_64 --define=runtime=tflite --copt=-DTFLITE_WITH_RUY_GEMV"
if [ "${OS}" = "${CI_MSYS_VERSION}" ]; then
# Somehow, even with Python being in the PATH, Bazel on windows struggles
# with '/usr/bin/env python' ...
#
# We also force TMP/TEMP otherwise Bazel will pick default Windows one
# under %USERPROFILE%\AppData\Local\Temp and with 8.3 file format convention
# it messes with cxx_builtin_include_directory
BAZEL_EXTRA_FLAGS="--action_env=PATH=${TC_WIN_BUILD_PATH} --action_env=TEMP=${TEMP} --action_env=TMP=${TMP}"
else
if [ "${OS}" != "${CI_MSYS_VERSION}" ]; then
BAZEL_EXTRA_FLAGS="--config=noaws --config=nogcp --config=nohdfs --config=nonccl --copt=-fvisibility=hidden"
fi
@ -211,3 +196,4 @@ BUILD_TARGET_GRAPH_BENCHMARK="//tensorflow/tools/benchmark:benchmark_model"
BUILD_TARGET_TOCO="//tensorflow/lite/toco:toco"
BUILD_TARGET_LITE_BENCHMARK="//tensorflow/lite/tools/benchmark:benchmark_model"
BUILD_TARGET_LITE_LIB="//tensorflow/lite/c:libtensorflowlite_c.so"
BUILD_TARGET_LIBDEEPSPEECH="//native_client:libdeepspeech.so"

Просмотреть файл

@ -45,16 +45,16 @@ workspace_status.cc:
# Enforce PATH here because swig calls from build_ext looses track of some
# variables over several runs
bindings: clean-keep-third-party workspace_status.cc $(DS_SWIG_DEP)
pip3 install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==45.0.0
DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python3 ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
python -m pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==45.0.0
DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
find temp_build -type f -name "*.o" -delete
DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python3 ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
rm -rf temp_build
bindings-debug: clean-keep-third-party workspace_status.cc $(DS_SWIG_DEP)
pip3 install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==45.0.0
DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python3 ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
python -m pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==45.0.0
DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
$(GENERATE_DEBUG_SYMS)
find temp_build -type f -name "*.o" -delete
DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python3 ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
rm -rf temp_build

Просмотреть файл

@ -55,7 +55,7 @@ PYTHON_PLATFORM_NAME ?= --plat-name manylinux1_x86_64
endif
endif
ifeq ($(TARGET),host-win)
ifeq ($(findstring _NT,$(OS)),_NT)
TOOLCHAIN := '$(VCToolsInstallDir)\bin\Hostx64\x64\'
TOOL_CC := cl.exe
TOOL_CXX := cl.exe
@ -181,7 +181,7 @@ define copy_missing_libs
if [ "$(OS)" = "Darwin" ]; then \
new_missing="$$( (for f in $$(otool -L $$lib 2>/dev/null | tail -n +2 | awk '{ print $$1 }' | grep -v '$$lib'); do ls -hal $$f; done;) 2>&1 | grep 'No such' | cut -d':' -f2 | xargs basename -a)"; \
missing_libs="$$missing_libs $$new_missing"; \
elif [ "$(OS)" = "${TC_MSYS_VERSION}" ]; then \
elif [ "$(OS)" = "${CI_MSYS_VERSION}" ]; then \
missing_libs="libdeepspeech.so"; \
else \
missing_libs="$$missing_libs $$($(LDD) $$lib | grep 'not found' | awk '{ print $$1 }')"; \