Merge pull request #3639 from lissyx/ga-linux-armv7-aarch64

CI: Linux ARMv7 / Aarch64
This commit is contained in:
lissyx 2021-04-28 10:51:44 +02:00 коммит произвёл GitHub
Родитель 510d161db5 00684371ff
Коммит 280f272431
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
24 изменённых файлов: 1111 добавлений и 34 удалений

29
.github/actions/chroot-bind-mount/action.yml поставляемый Normal file
Просмотреть файл

@ -0,0 +1,29 @@
name: "chroot bind mount"
description: "Bind mount into chroot"
inputs:
mounts:
description: "Path to consider"
required: true
runs:
using: "composite"
steps:
- id: install_qemu
run: |
sudo apt-get update -y
sudo apt-get install -y --no-install-recommends qemu-user-static
shell: bash
- id: bind_mount_chroot
run: |
set -xe
# Bind-mount so that we have the same tree inside the chroot
for dev in ${{ github.workspace }} ${{ inputs.mounts }};
do
sudo mount -o bind ${dev} ${{ env.SYSTEM_RASPBIAN }}${dev}
done;
for dev in ${{ inputs.mounts }};
do
sudo mount -o bind /${dev} ${{ env.SYSTEM_RASPBIAN }}/${dev}
done;
shell: bash

10
.github/actions/get_cache_key/action.yml поставляемый
Просмотреть файл

@ -4,6 +4,9 @@ inputs:
extras:
description: "Extra cache key value"
required: true
osarch:
description: "Override automatic OSARCH value"
required: false
outputs:
key:
description: "Computed cache key name"
@ -16,7 +19,12 @@ runs:
JOB=${{ github.job }}
SUBMODULE=$(echo $JOB | cut -d'-' -f1 | cut -d'_' -f1)
FLAVOR=$(echo $JOB | cut -d'-' -f1 | cut -d'_' -f2)
OSARCH=$(echo $JOB | cut -d'-' -f2)
if [ -z "${{ inputs.osarch }}" ]; then
OSARCH=$(echo $JOB | cut -d'-' -f2)
else
OSARCH=${{ inputs.osarch }}
fi
SHA=$(git submodule status ${SUBMODULE} | sed -e 's/^-//g' -e 's/^+//g' -e 's/^U//g' | awk '{ print $1 }')

10
.github/actions/host-build/action.yml поставляемый
Просмотреть файл

@ -1,11 +1,15 @@
name: "Host build lib"
description: "Host build of lib"
name: "Run build lib"
description: "Run build of lib"
inputs:
arch:
description: "Target arch for loading script (host/armv7/aarch64)"
required: false
default: "host"
flavor:
description: "Build flavor"
required: true
runs:
using: "composite"
steps:
- run: ./ci_scripts/host-build.sh ${{ inputs.flavor }}
- run: ./ci_scripts/${{ inputs.arch }}-build.sh ${{ inputs.flavor }}
shell: bash

18
.github/actions/install-xldd/action.yml поставляемый Normal file
Просмотреть файл

@ -0,0 +1,18 @@
name: "xldd install"
description: "Install xldd"
inputs:
target:
description: "System target"
required: true
runs:
using: "composite"
steps:
- id: install_xldd
run: |
source ./ci_scripts/all-vars.sh
# -s required to avoid the noisy output like "Entering / Leaving directories"
toolchain=$(make -s -C ${DS_DSDIR}/native_client/ TARGET=${{ inputs.target }} TFDIR=${DS_TFDIR} print-toolchain)
if [ ! -x "${toolchain}ldd" ]; then
cp "${DS_DSDIR}/native_client/xldd" "${toolchain}ldd" && chmod +x "${toolchain}ldd"
fi
shell: bash

67
.github/actions/multistrap/action.yml поставляемый Normal file
Просмотреть файл

@ -0,0 +1,67 @@
name: "multistrap install"
description: "Install a system root using multistrap"
inputs:
arch:
description: "Target arch"
required: true
packages:
description: "Extra packages to install"
required: false
default: ""
runs:
using: "composite"
steps:
- id: install_multistrap
run: |
sudo apt-get update -y
sudo apt-get install -y --no-install-recommends multistrap qemu-user-static
shell: bash
- id: create_chroot
run: |
set -xe
multistrap_conf=""
if [ "${{ inputs.arch }}" = "armv7" ]; then
multistrap_conf=multistrap_raspbian_buster.conf
wget http://archive.raspbian.org/raspbian/pool/main/r/raspbian-archive-keyring/raspbian-archive-keyring_20120528.2_all.deb && sudo dpkg -i raspbian-archive-keyring_20120528.2_all.deb
fi
if [ "${{ inputs.arch }}" = "aarch64" ]; then
multistrap_conf=multistrap_armbian64_buster.conf
fi
multistrap -d ${{ env.SYSTEM_RASPBIAN }} -f ${{ github.workspace }}/native_client/${multistrap_conf}
if [ ! -z "${{ inputs.packages }}" ]; then
TO_MOUNT=${{ github.workspace }}
# Prepare target directory to bind-mount the github tree
mkdir -p ${{ env.SYSTEM_RASPBIAN }}/${{ github.workspace }}
# Bind-mount so that we have the same tree inside the chroot
for dev in ${TO_MOUNT};
do
sudo mount -o bind ${dev} ${{ env.SYSTEM_RASPBIAN }}${dev}
done;
# Copy some host data:
# resolv.conf: for getting DNS working
# passwd, group, shadow: to have user accounts and apt-get install working
for ff in resolv.conf passwd group shadow;
do
sudo cp /etc/${ff} ${{ env.SYSTEM_RASPBIAN }}/etc/
done;
# Perform apt steps.
# Preserving the env is required
sudo --preserve-env chroot ${{ env.SYSTEM_RASPBIAN }}/ apt-get update -y
sudo --preserve-env chroot ${{ env.SYSTEM_RASPBIAN }}/ apt-get install -y --no-install-recommends ${{ inputs.packages }}
# Cleanup apt info to save space
sudo --preserve-env chroot ${{ env.SYSTEM_RASPBIAN }}/ rm -fr /var/cache/apt/* /var/lib/apt/lists/*
# Unmount what has been mounted
for dev in ${TO_MOUNT};
do
sudo umount ${{ env.SYSTEM_RASPBIAN }}${dev}
done;
fi
shell: bash

12
.github/actions/node-build/action.yml поставляемый
Просмотреть файл

@ -19,6 +19,14 @@ inputs:
description: "LIBS for NodeJS package"
required: false
default: ""
target:
description: "TARGET value"
required: false
default: "host"
chroot:
description: "RASPBIAN value"
required: false
default: ""
runs:
using: "composite"
steps:
@ -38,6 +46,8 @@ runs:
EXTRA_LDFLAGS=${{ inputs.local_ldflags }} \
EXTRA_LIBS=${{ inputs.local_libs }} \
make -C native_client/javascript \
TARGET=${{ inputs.target }} \
RASPBIAN=${{ inputs.chroot }} \
NODE_ABI_TARGET=--target=${node} \
NODE_DEVDIR=--devdir=headers/nodejs \
clean node-wrapper
@ -49,6 +59,8 @@ runs:
EXTRA_LDFLAGS=${{ inputs.local_ldflags }} \
EXTRA_LIBS=${{ inputs.local_libs }} \
make -C native_client/javascript \
TARGET=${{ inputs.target }} \
RASPBIAN=${{ inputs.chroot }} \
NODE_ABI_TARGET=--target=${electron} \
NODE_DIST_URL=--disturl=https://electronjs.org/headers \
NODE_RUNTIME=--runtime=electron \

22
.github/actions/node-install/action.yml поставляемый Normal file
Просмотреть файл

@ -0,0 +1,22 @@
name: "nodejs install"
description: "Install nodejs in a chroot"
inputs:
node:
description: "NodeJS version"
required: true
runs:
using: "composite"
steps:
- id: add_apt_source
run: |
set -ex
(echo "Package: nodejs" && echo "Pin: origin deb.nodesource.com" && echo "Pin-Priority: 999") > ${{ env.SYSTEM_RASPBIAN }}/etc/apt/preferences
echo "deb http://deb.nodesource.com/node_${{ inputs.node }}.x buster main" > ${{ env.SYSTEM_RASPBIAN }}/etc/apt/sources.list.d/nodesource.list
wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | sudo --preserve-env chroot ${{ env.SYSTEM_RASPBIAN }}/ apt-key add -
shell: bash
- id: install_nodejs
run: |
set -ex
sudo --preserve-env chroot ${{ env.SYSTEM_RASPBIAN }}/ apt-get update -y
sudo --preserve-env chroot ${{ env.SYSTEM_RASPBIAN }}/ apt-get install -y nodejs
shell: bash

23
.github/actions/python-build/action.yml поставляемый
Просмотреть файл

@ -22,30 +22,53 @@ inputs:
description: "LIBS for Python package"
required: false
default: ""
target:
description: "TARGET value"
required: false
default: "host"
chroot:
description: "RASPBIAN value"
required: false
default: ""
runs:
using: "composite"
steps:
- run: |
python3 --version
pip3 --version
python3 -m pip install virtualenv
python3 -m virtualenv deepspeech-build
shell: bash
- run: |
mkdir -p wheels
shell: bash
- run: |
set -xe
PROJECT_NAME="deepspeech"
if [ "${{ inputs.build_flavor }}" = "tflite" ]; then
PROJECT_NAME="deepspeech-tflite"
fi
OS=$(uname)
if [ "${OS}" = "Linux" ]; then
source deepspeech-build/bin/activate
fi
NUMPY_BUILD_VERSION="${{ inputs.numpy_build }}" \
NUMPY_DEP_VERSION="${{ inputs.numpy_dep }}" \
EXTRA_CFLAGS=${{ inputs.local_cflags }} \
EXTRA_LDFLAGS=${{ inputs.local_ldflags }} \
EXTRA_LIBS=${{ inputs.local_libs }} \
make -C native_client/python/ \
TARGET=${{ inputs.target }} \
RASPBIAN=${{ inputs.chroot }} \
SETUP_FLAGS="--project_name ${PROJECT_NAME}" \
bindings-clean bindings
if [ "${OS}" = "Linux" ]; then
deactivate
fi
shell: bash
- run: |
cp native_client/python/dist/*.whl wheels

12
.github/actions/run-tests/action.yml поставляемый
Просмотреть файл

@ -13,10 +13,15 @@ inputs:
bitrate:
description: "Bitrate for testing"
required: true
chroot:
description: "Run using a chroot"
required: false
runs:
using: "composite"
steps:
- run: |
set -xe
build=""
if [ "${{ inputs.build-flavor }}" = "tflite" ]; then
build="_tflite"
@ -27,5 +32,10 @@ runs:
model_kind="-prod"
fi
./ci_scripts/${{ inputs.runtime }}${build}-tests${model_kind}.sh ${{ inputs.bitrate }}
prefix="."
if [ ! -z "${{ inputs.chroot }}" ]; then
prefix="${{ inputs.chroot }}"
fi
${prefix}/ci_scripts/${{ inputs.runtime }}${build}-tests${model_kind}.sh ${{ inputs.bitrate }}
shell: bash

830
.github/workflows/build-and-test.yml поставляемый
Просмотреть файл

@ -135,7 +135,7 @@ jobs:
- id: get_numpy
uses: ./.github/actions/numpy_vers
with:
pyver: 3.6.8
pyver: 3.6
- name: Make decoder package
run: |
NUMPY_BUILD_VERSION=${{ steps.get_numpy.outputs.build_version }} \
@ -1675,7 +1675,7 @@ jobs:
repackage-nodejs-allplatforms:
name: "Repackage NodeJS / ElectronJS for multiplatforms"
runs-on: ubuntu-20.04
needs: [build-nodejs-macOS, build-nodejs-Windows, build-nodejs-Linux]
needs: [build-nodejs-macOS, build-nodejs-Windows, build-nodejs-Linux, build-nodejs-LinuxArmv7, build-nodejs-LinuxAarch64]
strategy:
matrix:
build-flavor: ["tf", "tflite"]
@ -1698,11 +1698,26 @@ jobs:
with:
name: "nodewrapper-${{ matrix.build-flavor }}-Linux_amd64.tar.gz"
path: /tmp/nodewrapper-Linux_amd64/
- uses: actions/download-artifact@v2
with:
name: "nodewrapper-${{ matrix.build-flavor }}-Linux_armv7.tar.gz"
path: /tmp/nodewrapper-Linux_armv7/
if: matrix.build-flavor == 'tflite'
- uses: actions/download-artifact@v2
with:
name: "nodewrapper-${{ matrix.build-flavor }}-Linux_aarch64.tar.gz"
path: /tmp/nodewrapper-Linux_aarch64/
if: matrix.build-flavor == 'tflite'
- name: Extract nodewrapper archives
run: |
tar -C ${{ github.workspace }}/native_client/javascript -xzvf /tmp/nodewrapper-macOS_amd64/wrapper.tar.gz
tar -C ${{ github.workspace }}/native_client/javascript -xzvf /tmp/nodewrapper-Windows_amd64/wrapper.tar.gz
tar -C ${{ github.workspace }}/native_client/javascript -xzvf /tmp/nodewrapper-Linux_amd64/wrapper.tar.gz
- name: Extract nodewrapper tflite-only archives
run: |
tar -C ${{ github.workspace }}/native_client/javascript -xzvf /tmp/nodewrapper-Linux_armv7/wrapper.tar.gz
tar -C ${{ github.workspace }}/native_client/javascript -xzvf /tmp/nodewrapper-Linux_aarch64/wrapper.tar.gz
if: matrix.build-flavor == 'tflite'
- run: |
PROJECT_NAME="deepspeech"
if [ "${{ matrix.build-flavor }}" = "tflite" ]; then
@ -2067,3 +2082,814 @@ jobs:
bitrate: ${{ matrix.bitrate }}
model-kind: ${{ matrix.models }}
timeout-minutes: 5
# Linux Armv7 and Aarch64 jobs
tensorflow_opt-LinuxArmv7:
name: "LinArmv7|Check TensorFlow cache"
runs-on: ubuntu-20.04
outputs:
status: ${{ steps.check_artifact_exists.outputs.status }}
cache_key: ${{ steps.get_cache_key.outputs.key }}
strategy:
matrix:
arch: [ "armv7" ]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 1
- id: get_cache_key
uses: ./.github/actions/get_cache_key
with:
extras: "0"
- id: check_artifact_exists
uses: ./.github/actions/check_artifact_exists
with:
name: ${{ steps.get_cache_key.outputs.key }}
tensorflow_opt-LinuxAarch64:
name: "LinAarch64|Check TensorFlow cache"
runs-on: ubuntu-20.04
outputs:
status: ${{ steps.check_artifact_exists.outputs.status }}
cache_key: ${{ steps.get_cache_key.outputs.key }}
strategy:
matrix:
arch: [ "aarch64" ]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 1
- id: get_cache_key
uses: ./.github/actions/get_cache_key
with:
extras: "0"
- id: check_artifact_exists
uses: ./.github/actions/check_artifact_exists
with:
name: ${{ steps.get_cache_key.outputs.key }}
build-tensorflow-LinuxArmv7:
name: "LinArmv7|Build TensorFlow (opt)"
needs: tensorflow_opt-LinuxArmv7
runs-on: ubuntu-20.04
strategy:
matrix:
arch: [ "armv7" ]
steps:
- run: true
if: needs.tensorflow_opt-LinuxArmv7.outputs.status == 'found'
- uses: actions/checkout@v2
with:
fetch-depth: 0
submodules: 'recursive'
if: needs.tensorflow_opt-LinuxArmv7.outputs.status == 'missing'
- uses: ./.github/actions/setup-tensorflow
if: needs.tensorflow_opt-LinuxArmv7.outputs.status == 'missing'
- uses: ./.github/actions/build-tensorflow
with:
flavor: "--linux-${{ matrix.arch }}"
if: needs.tensorflow_opt-LinuxArmv7.outputs.status == 'missing'
- uses: ./.github/actions/package-tensorflow
if: needs.tensorflow_opt-LinuxArmv7.outputs.status == 'missing'
- uses: actions/upload-artifact@v2
with:
name: ${{ needs.tensorflow_opt-LinuxArmv7.outputs.cache_key }}
path: ${{ github.workspace }}/artifacts/home.tar.xz
if: needs.tensorflow_opt-LinuxArmv7.outputs.status == 'missing'
build-tensorflow-LinuxAarch64:
name: "LinAarch64|Build TensorFlow (opt)"
needs: tensorflow_opt-LinuxAarch64
runs-on: ubuntu-20.04
strategy:
matrix:
arch: [ "aarch64" ]
steps:
- run: true
if: needs.tensorflow_opt-LinuxAarch64.outputs.status == 'found'
- uses: actions/checkout@v2
with:
fetch-depth: 0
submodules: 'recursive'
if: needs.tensorflow_opt-LinuxAarch64.outputs.status == 'missing'
- uses: ./.github/actions/setup-tensorflow
if: needs.tensorflow_opt-LinuxAarch64.outputs.status == 'missing'
- uses: ./.github/actions/build-tensorflow
with:
flavor: "--linux-${{ matrix.arch }}"
if: needs.tensorflow_opt-LinuxAarch64.outputs.status == 'missing'
- uses: ./.github/actions/package-tensorflow
if: needs.tensorflow_opt-LinuxAarch64.outputs.status == 'missing'
- uses: actions/upload-artifact@v2
with:
name: ${{ needs.tensorflow_opt-LinuxAarch64.outputs.cache_key }}
path: ${{ github.workspace }}/artifacts/home.tar.xz
if: needs.tensorflow_opt-LinuxAarch64.outputs.status == 'missing'
build-lib_LinuxArmv7:
name: "LinArmv7|Build libdeepspeech+client"
runs-on: ubuntu-20.04
strategy:
matrix:
build-flavor: ["tflite"]
arch: [ "armv7" ]
needs: [ build-tensorflow-LinuxArmv7, tensorflow_opt-LinuxArmv7 ]
env:
SYSTEM_TARGET: rpi3
SYSTEM_RASPBIAN: ${{ github.workspace }}/multistrap-raspbian-buster
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- uses: actions/download-artifact@v2
with:
name: ${{ needs.tensorflow_opt-LinuxArmv7.outputs.cache_key }}
path: ${{ github.workspace }}/
if: needs.tensorflow_opt-LinuxArmv7.outputs.status == 'missing'
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-LinuxArmv7.outputs.cache_key }}
path: ${{ github.workspace }}/
download: true
if: needs.tensorflow_opt-LinuxArmv7.outputs.status == 'found'
- run: |
tar -xf ${{ github.workspace }}/home.tar.xz --skip-old-files
rm ${{ github.workspace }}/home.tar.xz
- run: |
git status
- name: "Install chroot"
uses: ./.github/actions/multistrap
with:
arch: ${{ matrix.arch }}
- uses: ./.github/actions/host-build
with:
arch: ${{ matrix.arch }}
flavor: ${{ matrix.build-flavor }}
- uses: ./.github/actions/package
- uses: actions/upload-artifact@v2
with:
name: "native_client.${{ matrix.build-flavor }}.linux.${{ matrix.arch }}.tar.xz"
path: ${{ github.workspace }}/artifacts/native_client.tar.xz
- uses: actions/upload-artifact@v2
with:
name: "libdeepspeech.${{ matrix.build-flavor }}.linux.${{ matrix.arch }}.zip"
path: ${{ github.workspace }}/artifacts/libdeepspeech.zip
build-lib_LinuxAarch64:
name: "LinAarch64|Build libdeepspeech+client"
runs-on: ubuntu-20.04
strategy:
matrix:
build-flavor: ["tflite"]
arch: [ "aarch64" ]
needs: [ build-tensorflow-LinuxAarch64, tensorflow_opt-LinuxAarch64 ]
env:
SYSTEM_TARGET: rpi3-armv8
SYSTEM_RASPBIAN: ${{ github.workspace }}/multistrap-armbian64-buster
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- uses: actions/download-artifact@v2
with:
name: ${{ needs.tensorflow_opt-LinuxAarch64.outputs.cache_key }}
path: ${{ github.workspace }}/
if: needs.tensorflow_opt-LinuxAarch64.outputs.status == 'missing'
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-LinuxAarch64.outputs.cache_key }}
path: ${{ github.workspace }}/
download: true
if: needs.tensorflow_opt-LinuxAarch64.outputs.status == 'found'
- run: |
tar -xf ${{ github.workspace }}/home.tar.xz --skip-old-files
rm ${{ github.workspace }}/home.tar.xz
- run: |
git status
- name: "Install chroot"
uses: ./.github/actions/multistrap
with:
arch: ${{ matrix.arch }}
- uses: ./.github/actions/host-build
with:
arch: ${{ matrix.arch }}
flavor: ${{ matrix.build-flavor }}
- uses: ./.github/actions/package
- uses: actions/upload-artifact@v2
with:
name: "native_client.${{ matrix.build-flavor }}.linux.${{ matrix.arch }}.tar.xz"
path: ${{ github.workspace }}/artifacts/native_client.tar.xz
- uses: actions/upload-artifact@v2
with:
name: "libdeepspeech.${{ matrix.build-flavor }}.linux.${{ matrix.arch }}.zip"
path: ${{ github.workspace }}/artifacts/libdeepspeech.zip
build-python-LinuxArmv7:
name: "LinArmv7|Build python bindings"
runs-on: ubuntu-20.04
needs: [ build-lib_LinuxArmv7, swig_Linux, tensorflow_opt-LinuxArmv7 ]
strategy:
matrix:
build-flavor: ["tflite"]
python-version: [3.7]
arch: [ "armv7" ]
env:
DEBIAN_FRONTEND: "noninteractive"
SYSTEM_TARGET: rpi3
SYSTEM_RASPBIAN: ${{ github.workspace }}/multistrap-raspbian-buster
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 1
- uses: actions/download-artifact@v2
with:
name: "native_client.${{ matrix.build-flavor }}.linux.${{ matrix.arch }}.tar.xz"
path: ${{ github.workspace }}/tensorflow/bazel-bin/native_client/
- run: |
cd ${{ github.workspace }}/tensorflow/bazel-bin/native_client/
tar xf native_client.tar.xz
ls -hal
cd ${{ github.workspace }}/
- uses: actions/download-artifact@v2
with:
name: "swig_Linux"
path: ${{ github.workspace }}/native_client/ds-swig/
- name: Link ds-swig into swig
run: |
ls -hal ${{ github.workspace }}/native_client/ds-swig/bin
ln -s ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
chmod +x ${{ github.workspace }}/native_client/ds-swig/bin/ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
- uses: actions/download-artifact@v2
with:
name: ${{ needs.tensorflow_opt-LinuxArmv7.outputs.cache_key }}
path: ${{ github.workspace }}/
if: needs.tensorflow_opt-LinuxArmv7.outputs.status == 'missing'
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-LinuxArmv7.outputs.cache_key }}
path: ${{ github.workspace }}/
download: true
if: needs.tensorflow_opt-LinuxArmv7.outputs.status == 'found'
- run: |
tar -xf ${{ github.workspace }}/home.tar.xz --skip-old-files
rm ${{ github.workspace }}/home.tar.xz
- uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- uses: ./.github/actions/install-xldd
with:
target: ${{ env.SYSTEM_TARGET }}
- name: "Install chroot"
uses: ./.github/actions/multistrap
with:
arch: ${{ matrix.arch }}
- id: get_numpy
uses: ./.github/actions/numpy_vers
with:
pyver: ${{ matrix.python-version }}
- uses: ./.github/actions/python-build
with:
build_flavor: ${{ matrix.build-flavor }}
numpy_build: "${{ steps.get_numpy.outputs.build_version }}"
numpy_dep: "${{ steps.get_numpy.outputs.dep_version }}"
target: ${{ env.SYSTEM_TARGET }}
chroot: ${{ env.SYSTEM_RASPBIAN }}
- uses: actions/upload-artifact@v2
with:
name: "deepspeech-${{ matrix.build-flavor }}-${{ matrix.python-version }}-${{ matrix.arch }}.whl"
path: ${{ github.workspace }}/wheels/*.whl
build-nodejs-LinuxArmv7:
name: "LinArmv7|Build NodeJS and ElectronJS"
runs-on: ubuntu-20.04
needs: [ build-lib_LinuxArmv7, swig_Linux, tensorflow_opt-LinuxArmv7 ]
strategy:
matrix:
build-flavor: ["tflite"]
arch: [ "armv7" ]
env:
SYSTEM_TARGET: rpi3
SYSTEM_RASPBIAN: ${{ github.workspace }}/multistrap-raspbian-buster
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 1
- uses: actions/download-artifact@v2
with:
name: "native_client.${{ matrix.build-flavor }}.linux.${{ matrix.arch }}.tar.xz"
path: ${{ github.workspace }}/tensorflow/bazel-bin/native_client/
- run: |
cd ${{ github.workspace }}/tensorflow/bazel-bin/native_client/
tar xf native_client.tar.xz
ls -hal
cd ${{ github.workspace }}/
- uses: actions/download-artifact@v2
with:
name: "swig_Linux"
path: ${{ github.workspace }}/native_client/ds-swig/
- name: Link ds-swig into swig
run: |
ls -hal ${{ github.workspace }}/native_client/ds-swig/bin
ln -s ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
chmod +x ${{ github.workspace }}/native_client/ds-swig/bin/ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
- uses: actions/download-artifact@v2
with:
name: ${{ needs.tensorflow_opt-LinuxArmv7.outputs.cache_key }}
path: ${{ github.workspace }}/
if: needs.tensorflow_opt-LinuxArmv7.outputs.status == 'missing'
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-LinuxArmv7.outputs.cache_key }}
path: ${{ github.workspace }}/
download: true
if: needs.tensorflow_opt-LinuxArmv7.outputs.status == 'found'
- run: |
tar -xf ${{ github.workspace }}/home.tar.xz --skip-old-files
rm ${{ github.workspace }}/home.tar.xz
- uses: ./.github/actions/install-xldd
with:
target: ${{ env.SYSTEM_TARGET }}
- name: "Install chroot"
uses: ./.github/actions/multistrap
with:
arch: ${{ matrix.arch }}
- uses: actions/setup-node@v2
with:
node-version: 12
- uses: actions/cache@v2
id: node-headers-cache
with:
path: native_client/javascript/headers/nodejs/
key: node-headers-10.0.0_15.0.0
- uses: actions/cache@v2
id: electron-headers-cache
with:
path: native_client/javascript/headers/electronjs/
key: electron-headers-5.0.13_12.0.0
- uses: ./.github/actions/node-build
with:
nodejs_versions: "10.0.0 11.0.0 12.7.0 13.0.0 14.0.0 15.0.0"
electronjs_versions: "5.0.13 6.0.12 6.1.7 7.0.1 7.1.8 8.0.1 9.0.1 9.1.0 9.2.0 10.0.0 10.1.0 11.0.0 12.0.0"
target: ${{ env.SYSTEM_TARGET }}
chroot: ${{ env.SYSTEM_RASPBIAN }}
- uses: actions/upload-artifact@v2
with:
name: "nodewrapper-${{ matrix.build-flavor }}-Linux_${{ matrix.arch }}.tar.gz"
path: ${{ github.workspace }}/native_client/javascript/wrapper.tar.gz
- uses: actions/upload-artifact@v2
with:
name: "deepspeech_intermediate-${{ matrix.build-flavor }}-${{ matrix.arch }}.tgz"
path: ${{ github.workspace }}/native_client/javascript/deepspeech-*.tgz
build-python-LinuxAarch64:
name: "LinAarch64|Build python bindings"
runs-on: ubuntu-20.04
needs: [ build-lib_LinuxAarch64, swig_Linux, tensorflow_opt-LinuxAarch64 ]
strategy:
matrix:
build-flavor: ["tflite"]
python-version: [3.7]
arch: [ "aarch64" ]
env:
DEBIAN_FRONTEND: "noninteractive"
SYSTEM_TARGET: rpi3-armv8
SYSTEM_RASPBIAN: ${{ github.workspace }}/multistrap-armbian64-buster
steps:
- run: |
sudo apt-get install -y --no-install-recommends
- uses: actions/checkout@v2
with:
fetch-depth: 1
- uses: actions/download-artifact@v2
with:
name: "native_client.${{ matrix.build-flavor }}.linux.${{ matrix.arch }}.tar.xz"
path: ${{ github.workspace }}/tensorflow/bazel-bin/native_client/
- run: |
cd ${{ github.workspace }}/tensorflow/bazel-bin/native_client/
tar xf native_client.tar.xz
ls -hal
cd ${{ github.workspace }}/
- uses: actions/download-artifact@v2
with:
name: "swig_Linux"
path: ${{ github.workspace }}/native_client/ds-swig/
- name: Link ds-swig into swig
run: |
ls -hal ${{ github.workspace }}/native_client/ds-swig/bin
ln -s ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
chmod +x ${{ github.workspace }}/native_client/ds-swig/bin/ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
- uses: actions/download-artifact@v2
with:
name: ${{ needs.tensorflow_opt-LinuxAarch64.outputs.cache_key }}
path: ${{ github.workspace }}/
if: needs.tensorflow_opt-LinuxAarch64.outputs.status == 'missing'
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-LinuxAarch64.outputs.cache_key }}
path: ${{ github.workspace }}/
download: true
if: needs.tensorflow_opt-LinuxAarch64.outputs.status == 'found'
- run: |
tar -xf ${{ github.workspace }}/home.tar.xz --skip-old-files
rm ${{ github.workspace }}/home.tar.xz
- uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- uses: ./.github/actions/install-xldd
with:
target: ${{ env.SYSTEM_TARGET }}
- name: "Install chroot"
uses: ./.github/actions/multistrap
with:
arch: ${{ matrix.arch }}
- id: get_numpy
uses: ./.github/actions/numpy_vers
with:
pyver: ${{ matrix.python-version }}
- uses: ./.github/actions/python-build
with:
build_flavor: ${{ matrix.build-flavor }}
numpy_build: "${{ steps.get_numpy.outputs.build_version }}"
numpy_dep: "${{ steps.get_numpy.outputs.dep_version }}"
target: ${{ env.SYSTEM_TARGET }}
chroot: ${{ env.SYSTEM_RASPBIAN }}
- uses: actions/upload-artifact@v2
with:
name: "deepspeech-${{ matrix.build-flavor }}-${{ matrix.python-version }}-${{ matrix.arch }}.whl"
path: ${{ github.workspace }}/wheels/*.whl
build-nodejs-LinuxAarch64:
name: "LinAarch64|Build NodeJS and ElectronJS"
runs-on: ubuntu-20.04
needs: [ build-lib_LinuxAarch64, swig_Linux, tensorflow_opt-LinuxAarch64 ]
strategy:
matrix:
build-flavor: ["tflite"]
arch: [ "aarch64" ]
env:
SYSTEM_TARGET: rpi3-armv8
SYSTEM_RASPBIAN: ${{ github.workspace }}/multistrap-armbian64-buster
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 1
- uses: actions/download-artifact@v2
with:
name: "native_client.${{ matrix.build-flavor }}.linux.${{ matrix.arch }}.tar.xz"
path: ${{ github.workspace }}/tensorflow/bazel-bin/native_client/
- run: |
cd ${{ github.workspace }}/tensorflow/bazel-bin/native_client/
tar xf native_client.tar.xz
ls -hal
cd ${{ github.workspace }}/
- uses: actions/download-artifact@v2
with:
name: "swig_Linux"
path: ${{ github.workspace }}/native_client/ds-swig/
- name: Link ds-swig into swig
run: |
ls -hal ${{ github.workspace }}/native_client/ds-swig/bin
ln -s ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
chmod +x ${{ github.workspace }}/native_client/ds-swig/bin/ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
- uses: actions/download-artifact@v2
with:
name: ${{ needs.tensorflow_opt-LinuxAarch64.outputs.cache_key }}
path: ${{ github.workspace }}/
if: needs.tensorflow_opt-LinuxAarch64.outputs.status == 'missing'
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-LinuxAarch64.outputs.cache_key }}
path: ${{ github.workspace }}/
download: true
if: needs.tensorflow_opt-LinuxAarch64.outputs.status == 'found'
- run: |
tar -xf ${{ github.workspace }}/home.tar.xz --skip-old-files
rm ${{ github.workspace }}/home.tar.xz
- uses: ./.github/actions/install-xldd
with:
target: ${{ env.SYSTEM_TARGET }}
- name: "Install chroot"
uses: ./.github/actions/multistrap
with:
arch: ${{ matrix.arch }}
- uses: actions/setup-node@v2
with:
node-version: 12
- uses: actions/cache@v2
id: node-headers-cache
with:
path: native_client/javascript/headers/nodejs/
key: node-headers-10.0.0_15.0.0
- uses: actions/cache@v2
id: electron-headers-cache
with:
path: native_client/javascript/headers/electronjs/
key: electron-headers-5.0.13_12.0.0
- uses: ./.github/actions/node-build
with:
nodejs_versions: "10.0.0 11.0.0 12.7.0 13.0.0 14.0.0 15.0.0"
electronjs_versions: "5.0.13 6.0.12 6.1.7 7.0.1 7.1.8 8.0.1 9.0.1 9.1.0 9.2.0 10.0.0 10.1.0 11.0.0 12.0.0"
target: ${{ env.SYSTEM_TARGET }}
chroot: ${{ env.SYSTEM_RASPBIAN }}
- uses: actions/upload-artifact@v2
with:
name: "nodewrapper-${{ matrix.build-flavor }}-Linux_${{ matrix.arch }}.tar.gz"
path: ${{ github.workspace }}/native_client/javascript/wrapper.tar.gz
- uses: actions/upload-artifact@v2
with:
name: "deepspeech_intermediate-${{ matrix.build-flavor }}-${{ matrix.arch }}.tgz"
path: ${{ github.workspace }}/native_client/javascript/deepspeech-*.tgz
build-test-chroot:
name: "Lin|Build test chroot"
runs-on: ubuntu-20.04
if: ${{ github.event_name == 'pull_request' }}
strategy:
matrix:
arch: [ "armv7", "aarch64" ]
env:
CI_TMP_DIR: ${{ github.workspace }}/tmp
DEBIAN_FRONTEND: "noninteractive"
SYSTEM_RASPBIAN: ${{ github.workspace }}/chroot-${{ matrix.arch }}
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 1
- name: "Install and setup chroot"
uses: ./.github/actions/multistrap
with:
arch: ${{ matrix.arch }}
packages: "bash wget curl sox xxd libatlas3-base libopenblas-base ca-certificates python3 python3-pip gnupg libatk1.0-0 libatk-bridge2.0-0 libcairo2 libcups2 libdbus-1-3 libgdk-pixbuf2.0-0 libgtk-3-0 libgbm1 libnspr4 libnss3 libpango-1.0-0 libpangocairo-1.0-0 libx11-xcb1 libxcb-dri3-0 libxcomposite1 libxcursor1 libxdamage1 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 xauth"
- name: "Create a chroot tarball"
run: |
sudo tar -cf - -C ${{ env.SYSTEM_RASPBIAN }}/ --one-file-system . | xz -9 -T0 > ${{ github.workspace }}/chroot.tar.xz
- uses: actions/upload-artifact@v2
with:
name: chroot-${{ matrix.arch }}
path: ${{ github.workspace }}/chroot.tar.xz
test-cpp-LinuxArm:
name: "LinArm*|Test C++ binary"
runs-on: ubuntu-20.04
needs: [ build-lib_LinuxArmv7, build-lib_LinuxAarch64, train-test-model-Linux, build-test-chroot ]
if: ${{ github.event_name == 'pull_request' }}
strategy:
matrix:
arch: [ "armv7", "aarch64" ]
build-flavor: ["tflite"]
models: ["test", "prod"]
bitrate: ["8k", "16k"]
env:
CI_TMP_DIR: ${{ github.workspace }}/tmp
DEBIAN_FRONTEND: "noninteractive"
DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb
DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm
DEEPSPEECH_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
SYSTEM_RASPBIAN: ${{ github.workspace }}/chroot-${{ matrix.arch }}
steps:
- name: "Install QEMU"
run: |
sudo apt-get update -y
sudo apt-get install -y --no-install-recommends qemu-user-static
- uses: actions/checkout@v2
with:
fetch-depth: 1
- uses: actions/download-artifact@v2
with:
name: "chroot-${{ matrix.arch }}"
path: ${{ env.CI_TMP_DIR }}/
- run: |
mkdir ${{ env.SYSTEM_RASPBIAN }}/
sudo tar -xf ${{ env.CI_TMP_DIR }}/chroot.tar.xz -C ${{ env.SYSTEM_RASPBIAN }}/
rm ${{ env.CI_TMP_DIR }}/chroot.tar.xz
- uses: actions/download-artifact@v2
with:
name: "native_client.${{ matrix.build-flavor }}.linux.${{ matrix.arch }}.tar.xz"
path: ${{ env.CI_TMP_DIR }}/
- run: |
cd ${{ env.CI_TMP_DIR }}/
mkdir ds && cd ds && tar xf ../native_client.tar.xz
- uses: actions/download-artifact@v2
with:
name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip"
path: ${{ env.CI_TMP_DIR }}/
if: matrix.models == 'test'
- run: |
ls -hal ${{ env.CI_TMP_DIR }}/
if: matrix.models == 'test'
- name: "Check tests"
run: |
file ${{ env.SYSTEM_RASPBIAN }}/${{ env.CI_TMP_DIR }}/ds/*
- uses: ./.github/actions/chroot-bind-mount
with:
mounts: "/dev"
- uses: ./.github/actions/run-tests
with:
runtime: "cpp"
chroot: "sudo --preserve-env chroot --userspec=runner:docker ${{ env.SYSTEM_RASPBIAN }}/ ${{ github.workspace }}"
build-flavor: ${{ matrix.build-flavor }}
bitrate: ${{ matrix.bitrate }}
model-kind: ${{ matrix.models }}
test-py-LinuxArm:
name: "LinArm*|Test Python bindings"
runs-on: ubuntu-20.04
needs: [ build-python-LinuxArmv7, build-python-LinuxAarch64, train-test-model-Linux, build-test-chroot ]
if: ${{ github.event_name == 'pull_request' }}
strategy:
matrix:
arch: [ "armv7", "aarch64" ]
python-version: [3.7]
build-flavor: ["tflite"]
models: ["test", "prod"]
bitrate: ["8k", "16k"]
env:
CI_TMP_DIR: ${{ github.workspace }}/tmp
DEBIAN_FRONTEND: "noninteractive"
DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb
DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm
DEEPSPEECH_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
SYSTEM_RASPBIAN: ${{ github.workspace }}/chroot-${{ matrix.arch }}
PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple https://lissyx.github.io/deepspeech-python-wheels/"
steps:
- name: "Install QEMU"
run: |
sudo apt-get update -y
sudo apt-get install -y --no-install-recommends qemu-user-static
- uses: actions/checkout@v2
with:
fetch-depth: 1
- uses: actions/download-artifact@v2
with:
name: "chroot-${{ matrix.arch }}"
path: ${{ env.CI_TMP_DIR }}/
- run: |
mkdir ${{ env.SYSTEM_RASPBIAN }}/
sudo tar -xf ${{ env.CI_TMP_DIR }}/chroot.tar.xz -C ${{ env.SYSTEM_RASPBIAN }}/
rm ${{ env.CI_TMP_DIR }}/chroot.tar.xz
- uses: actions/download-artifact@v2
with:
name: "deepspeech-${{ matrix.build-flavor }}-${{ matrix.python-version }}-${{ matrix.arch }}.whl"
path: ${{ env.CI_TMP_DIR }}/
- uses: actions/download-artifact@v2
with:
name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip"
path: ${{ env.CI_TMP_DIR }}/
if: matrix.models == 'test'
- run: |
ls -hal ${{ env.CI_TMP_DIR }}/
if: matrix.models == 'test'
- uses: ./.github/actions/chroot-bind-mount
with:
mounts: "/dev"
- run: |
ls -hal ${{ env.CI_TMP_DIR }}/
ls -hal ${{ github.workspace }}/
ls -hal ${{ env.SYSTEM_RASPBIAN }}/${{ github.workspace }}/
sudo --preserve-env chroot --userspec=runner:docker ${{ env.SYSTEM_RASPBIAN }}/ pip3 install --only-binary :all: --upgrade ${{ env.CI_TMP_DIR }}/deepspeech*.whl
- uses: ./.github/actions/run-tests
with:
runtime: "python"
chroot: "sudo --preserve-env chroot --userspec=runner:docker ${{ env.SYSTEM_RASPBIAN }}/ ${{ github.workspace }}"
build-flavor: ${{ matrix.build-flavor }}
bitrate: ${{ matrix.bitrate }}
model-kind: ${{ matrix.models }}
test-nodejs-LinuxArm:
name: "LinArm*|Test NodeJS bindings"
runs-on: ubuntu-20.04
needs: [ build-nodejs-LinuxArmv7, build-nodejs-LinuxAarch64, train-test-model-Linux, build-test-chroot ]
if: ${{ github.event_name == 'pull_request' }}
strategy:
matrix:
arch: [ "armv7", "aarch64" ]
# https://nodejs.org/en/about/releases/
nodejs-version: [10, 12, 14, 15]
build-flavor: ["tflite"]
models: ["test"]
bitrate: ["16k"]
env:
CI_TMP_DIR: ${{ github.workspace }}/tmp
DEBIAN_FRONTEND: "noninteractive"
DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb
DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm
DEEPSPEECH_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
SYSTEM_RASPBIAN: ${{ github.workspace }}/chroot-${{ matrix.arch }}
steps:
- name: "Install QEMU"
run: |
sudo apt-get update -y
sudo apt-get install -y --no-install-recommends qemu-user-static
- uses: actions/checkout@v2
with:
fetch-depth: 1
- uses: actions/download-artifact@v2
with:
name: "chroot-${{ matrix.arch }}"
path: ${{ env.CI_TMP_DIR }}/
- run: |
mkdir ${{ env.SYSTEM_RASPBIAN }}/
sudo tar -xf ${{ env.CI_TMP_DIR }}/chroot.tar.xz -C ${{ env.SYSTEM_RASPBIAN }}/
rm ${{ env.CI_TMP_DIR }}/chroot.tar.xz
- name: "Install NodeJS"
uses: ./.github/actions/node-install
with:
node: ${{ matrix.nodejs-version }}
- uses: actions/download-artifact@v2
with:
name: "deepspeech_intermediate-${{ matrix.build-flavor }}-${{ matrix.arch }}.tgz"
path: ${{ env.CI_TMP_DIR }}/
- uses: actions/download-artifact@v2
with:
name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip"
path: ${{ env.CI_TMP_DIR }}/
if: matrix.models == 'test'
- run: |
ls -hal ${{ env.CI_TMP_DIR }}/
if: matrix.models == 'test'
- uses: ./.github/actions/chroot-bind-mount
with:
mounts: "/dev"
- name: Install deepspeech package
run: |
sudo --preserve-env chroot --userspec=runner:docker ${{ env.SYSTEM_RASPBIAN }}/ npm install --prefix ${{ github.workspace }}/ --verbose ${{ env.CI_TMP_DIR }}/deepspeech*.tgz
- uses: ./.github/actions/run-tests
with:
runtime: "node"
chroot: "sudo --preserve-env chroot --userspec=runner:docker ${{ env.SYSTEM_RASPBIAN }}/ ${{ github.workspace }}"
build-flavor: ${{ matrix.build-flavor }}
bitrate: ${{ matrix.bitrate }}
model-kind: ${{ matrix.models }}
test-electronjs-LinuxArm:
name: "LinArm*|Test ElectronJS bindings"
runs-on: ubuntu-20.04
needs: [ build-nodejs-LinuxArmv7, build-nodejs-LinuxAarch64, train-test-model-Linux, build-test-chroot ]
# Disable this task because it seems qemu does not work super-well with ElectronJS
if: ${{ github.event_name == 'disabled' }}
strategy:
matrix:
arch: [ "armv7", "aarch64" ]
electronjs-version: [5.0.13, 6.1.7, 7.1.8, 8.0.1, 9.2.0, 10.1.0, 11.0.0, 12.0.0]
build-flavor: ["tflite"]
models: ["test"]
bitrate: ["16k"]
env:
CI_TMP_DIR: ${{ github.workspace }}/tmp
DEBIAN_FRONTEND: "noninteractive"
DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb
DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm
DEEPSPEECH_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
SYSTEM_RASPBIAN: ${{ github.workspace }}/chroot-${{ matrix.arch }}
DISPLAY: ":99.0"
steps:
- name: "Install QEMU"
run: |
sudo apt-get update -y
sudo apt-get install -y --no-install-recommends qemu-user-static xvfb xauth
- uses: actions/checkout@v2
with:
fetch-depth: 1
- uses: actions/download-artifact@v2
with:
name: "chroot-${{ matrix.arch }}"
path: ${{ env.CI_TMP_DIR }}/
- run: |
mkdir ${{ env.SYSTEM_RASPBIAN }}/
sudo tar -xf ${{ env.CI_TMP_DIR }}/chroot.tar.xz -C ${{ env.SYSTEM_RASPBIAN }}/
rm ${{ env.CI_TMP_DIR }}/chroot.tar.xz
- name: "Install NodeJS"
uses: ./.github/actions/node-install
with:
node: 12
- uses: actions/download-artifact@v2
with:
name: "deepspeech_intermediate-${{ matrix.build-flavor }}-${{ matrix.arch }}.tgz"
path: ${{ env.CI_TMP_DIR }}/
- uses: actions/download-artifact@v2
with:
name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip"
path: ${{ env.CI_TMP_DIR }}/
if: matrix.models == 'test'
- run: |
ls -hal ${{ env.CI_TMP_DIR }}/
if: matrix.models == 'test'
- uses: ./.github/actions/chroot-bind-mount
with:
mounts: "/dev /proc /sys /run"
- name: Install deepspeech package
run: |
sudo --preserve-env chroot --userspec=runner:docker ${{ env.SYSTEM_RASPBIAN }}/ npm install --prefix ${{ github.workspace }}/ ${{ env.CI_TMP_DIR }}/deepspeech*.tgz
- run: |
sudo --preserve-env chroot --userspec=runner:docker ${{ env.SYSTEM_RASPBIAN }}/ npm install --prefix ${{ github.workspace }}/ electron@${{ matrix.electronjs-version }}
- name: "Fake X display"
run: |
sudo Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 &
xvfb_process=$!
echo $xvfb_process > ${{ env.CI_TMP_DIR }}/xvfb.pid
cat ${{ env.CI_TMP_DIR }}/xvfb.pid
- name: "Debug missing libs"
run: |
sudo --preserve-env chroot --userspec=runner:docker ${{ env.SYSTEM_RASPBIAN }}/ ls -hal ${{ github.workspace }}/node_modules/electron/dist/electron
sudo --preserve-env chroot --userspec=runner:docker ${{ env.SYSTEM_RASPBIAN }}/ ldd ${{ github.workspace }}/node_modules/electron/dist/electron
- uses: ./.github/actions/run-tests
with:
runtime: "electronjs"
chroot: "sudo --preserve-env chroot --userspec=runner:docker ${{ env.SYSTEM_RASPBIAN }}/ ${{ github.workspace }}"
build-flavor: ${{ matrix.build-flavor }}
bitrate: ${{ matrix.bitrate }}
model-kind: ${{ matrix.models }}
timeout-minutes: 5
- name: "Kill X"
run: |
cat ${{ env.CI_TMP_DIR }}/xvfb.pid
sudo kill -9 $(cat ${{ env.CI_TMP_DIR }}/xvfb.pid)

23
ci_scripts/aarch64-build.sh Executable file
Просмотреть файл

@ -0,0 +1,23 @@
#!/bin/bash
set -xe
source $(dirname "$0")/all-vars.sh
source $(dirname "$0")/all-utils.sh
source $(dirname "$0")/build-utils.sh
source $(dirname "$0")/tf-vars.sh
BAZEL_TARGETS="
//native_client:libdeepspeech.so
//native_client:generate_scorer_package
"
BAZEL_BUILD_FLAGS="${BAZEL_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS}"
BAZEL_ENV_FLAGS="TF_NEED_CUDA=0"
maybe_install_xldd
do_bazel_build
do_deepspeech_binary_build

Просмотреть файл

@ -112,9 +112,9 @@ symlink_electron()
if [ "${OS}" = "Darwin" ]; then
ln -s Electron.app/Contents/MacOS/Electron node_modules/electron/dist/node
else
ln -s electron "node_modules/electron/dist/node"
ln -s electron "${DS_ROOT_TASK}/node_modules/electron/dist/node"
if [ "${OS}" = "Linux" -a -f "node_modules/electron/dist/chrome-sandbox" ]; then
if [ "${OS}" = "Linux" -a -f "${DS_ROOT_TASK}/node_modules/electron/dist/chrome-sandbox" ]; then
export ELECTRON_DISABLE_SANDBOX=1
fi
fi
@ -122,5 +122,10 @@ symlink_electron()
export_node_bin_path()
{
export PATH=$(pwd)/node_modules/.bin/:$(pwd)/node_modules/electron/dist/:$PATH
export PATH=${DS_ROOT_TASK}/node_modules/.bin/:${DS_ROOT_TASK}/node_modules/electron/dist/:$PATH
}
export_py_bin_path()
{
export PATH=$HOME/.local/bin/:$PATH
}

23
ci_scripts/armv7-build.sh Executable file
Просмотреть файл

@ -0,0 +1,23 @@
#!/bin/bash
set -xe
source $(dirname "$0")/all-vars.sh
source $(dirname "$0")/all-utils.sh
source $(dirname "$0")/build-utils.sh
source $(dirname "$0")/tf-vars.sh
BAZEL_TARGETS="
//native_client:libdeepspeech.so
//native_client:generate_scorer_package
"
BAZEL_BUILD_FLAGS="${BAZEL_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS}"
BAZEL_ENV_FLAGS="TF_NEED_CUDA=0"
maybe_install_xldd
do_bazel_build
do_deepspeech_binary_build

Просмотреть файл

@ -19,7 +19,8 @@ download_model_prod
download_material
which deepspeech
export_py_bin_path
deepspeech --version
run_prod_inference_tests "${bitrate}"

Просмотреть файл

@ -11,7 +11,8 @@ set_ldc_sample_filename "${bitrate}"
download_data
which deepspeech
export_py_bin_path
deepspeech --version
run_all_inference_tests

Просмотреть файл

@ -18,7 +18,8 @@ download_model_prod
download_material
which deepspeech
export_py_bin_path
deepspeech --version
run_prodtflite_inference_tests "${bitrate}"

Просмотреть файл

@ -15,7 +15,8 @@ model_name_mmap=$(basename "${model_source}")
download_data
which deepspeech
export_py_bin_path
deepspeech --version
run_all_inference_tests

Просмотреть файл

@ -31,10 +31,10 @@ pushd ${DS_ROOT_TASK}/tensorflow/
"--linux-cuda"|"--windows-cuda")
eval "export ${TF_CUDA_FLAGS}" && (echo "" | TF_NEED_CUDA=1 ./configure) && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_CUDA_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BAZEL_OPT_FLAGS} ${BUILD_TARGET_LIB_CPP_API}
;;
"--linux-arm")
"--linux-armv7")
echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
;;
"--linux-arm64")
"--linux-aarch64")
echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
;;
"--android-armv7")

Просмотреть файл

@ -22,10 +22,7 @@ if [ "${OS}" = "Linux" ]; then
ANDROID_SDK_URL=https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip
ANDROID_SDK_SHA256=92ffee5a1d98d856634e8b71132e8a95d96c83a63fde1099be3d86df3106def9
SHA_SUM="sha256sum -c --strict"
WGET=/usr/bin/wget
TAR=tar
XZ="pixz -9"
elif [ "${OS}" = "${CI_MSYS_VERSION}" ]; then
if [ -z "${CI_TASK_DIR}" -o -z "${CI_ARTIFACTS_DIR}" ]; then
echo "Inconsistent Windows setup: missing some vars."
@ -53,10 +50,7 @@ elif [ "${OS}" = "${CI_MSYS_VERSION}" ]; then
CUDA_INSTALL_DIRECTORY=$(cygpath 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1')
SHA_SUM="sha256sum -c --strict"
WGET=wget
TAR=/usr/bin/tar.exe
XZ="xz -9 -T0"
elif [ "${OS}" = "Darwin" ]; then
if [ -z "${CI_TASK_DIR}" -o -z "${CI_ARTIFACTS_DIR}" ]; then
echo "Inconsistent OSX setup: missing some vars."
@ -71,11 +65,17 @@ elif [ "${OS}" = "Darwin" ]; then
BAZEL_SHA256=5cfa97031b43432b3c742c80e2e01c41c0acdca7ba1052fc8cf1e291271bc9cd
SHA_SUM="shasum -a 256 -c"
WGET=wget
TAR=gtar
XZ="xz -9 -T0"
fi;
WGET=${WGET:-"wget"}
TAR=${TAR:-"tar"}
XZ=${XZ:-"xz -9 -T0"}
ZIP=${ZIP:-"zip"}
UNXZ=${UNXZ:-"xz -T0 -d"}
UNGZ=${UNGZ:-"gunzip"}
SHA_SUM=${SHA_SUM:-"sha256sum -c --strict"}
# /tmp/artifacts for docker-worker on linux,
# and task subdir for generic-worker on osx
export CI_ARTIFACTS_DIR=${CI_ARTIFACTS_DIR:-/tmp/artifacts}

Просмотреть файл

@ -46,14 +46,14 @@ workspace_status.cc:
# variables over several runs
bindings: clean-keep-third-party workspace_status.cc $(DS_SWIG_DEP)
python -m pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==45.0.0
DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN_DIR):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
find temp_build -type f -name "*.o" -delete
DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
rm -rf temp_build
bindings-debug: clean-keep-third-party workspace_status.cc $(DS_SWIG_DEP)
python -m pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==45.0.0
DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN_DIR):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
$(GENERATE_DEBUG_SYMS)
find temp_build -type f -name "*.o" -delete
DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)

Просмотреть файл

@ -70,9 +70,11 @@ PYTHON_PACKAGES := numpy${NUMPY_BUILD_VERSION}
endif
ifeq ($(TARGET),rpi3)
TOOLCHAIN ?= ${TFDIR}/bazel-$(shell basename "${TFDIR}")/external/LinaroArmGcc72/bin/arm-linux-gnueabihf-
TOOLCHAIN_DIR ?= ${TFDIR}/bazel-$(shell basename "${TFDIR}")/external/LinaroArmGcc72/bin
TOOLCHAIN ?= $(TOOLCHAIN_DIR)/arm-linux-gnueabihf-
RASPBIAN ?= $(abspath $(NC_DIR)/../multistrap-raspbian-buster)
CFLAGS := -march=armv7-a -mtune=cortex-a53 -mfpu=neon-fp-armv8 -mfloat-abi=hard -D_GLIBCXX_USE_CXX11_ABI=0 --sysroot $(RASPBIAN)
# -D_XOPEN_SOURCE -D_FILE_OFFSET_BITS=64 => to avoid EOVERFLOW on readdir() with 64-bits inode
CFLAGS := -march=armv7-a -mtune=cortex-a53 -mfpu=neon-fp-armv8 -mfloat-abi=hard -D_GLIBCXX_USE_CXX11_ABI=0 -D_XOPEN_SOURCE -D_FILE_OFFSET_BITS=64 --sysroot $(RASPBIAN)
CXXFLAGS := $(CFLAGS)
LDFLAGS := -Wl,-rpath-link,$(RASPBIAN)/lib/arm-linux-gnueabihf/ -Wl,-rpath-link,$(RASPBIAN)/usr/lib/arm-linux-gnueabihf/
@ -90,7 +92,8 @@ TOOLCHAIN_LDD_OPTS := --root $(RASPBIAN)/
endif # ($(TARGET),rpi3)
ifeq ($(TARGET),rpi3-armv8)
TOOLCHAIN ?= ${TFDIR}/bazel-$(shell basename "${TFDIR}")/external/LinaroAarch64Gcc72/bin/aarch64-linux-gnu-
TOOLCHAIN_DIR ?= ${TFDIR}/bazel-$(shell basename "${TFDIR}")/external/LinaroAarch64Gcc72/bin
TOOLCHAIN ?= $(TOOLCHAIN_DIR)/aarch64-linux-gnu-
RASPBIAN ?= $(abspath $(NC_DIR)/../multistrap-raspbian64-buster)
CFLAGS := -march=armv8-a -mtune=cortex-a53 -D_GLIBCXX_USE_CXX11_ABI=0 --sysroot $(RASPBIAN)
CXXFLAGS := $(CFLAGS)

Просмотреть файл

@ -1,13 +1,13 @@
[General]
arch=arm64
noauth=true
noauth=false
unpack=true
debootstrap=Debian
aptsources=Debian
cleanup=true
[Debian]
packages=libc6 libc6-dev libstdc++-7-dev linux-libc-dev libffi-dev libpython3.7-dev libsox-dev python3-numpy python3-setuptools
packages=apt libc6 libc6-dev libstdc++-7-dev linux-libc-dev libffi-dev libpython3.7-dev libsox-dev python3-numpy python3-setuptools
source=http://deb.debian.org/debian
keyring=debian-archive-keyring
components=main

Просмотреть файл

@ -1,13 +1,13 @@
[General]
arch=armhf
noauth=true
noauth=false
unpack=true
debootstrap=Raspbian
aptsources=Raspbian
cleanup=true
[Raspbian]
packages=libc6 libc6-dev libffi-dev libstdc++-6-dev linux-libc-dev libpython3.7-dev libsox-dev python3-numpy python3-setuptools
packages=apt libc6 libc6-dev libffi-dev libstdc++-6-dev linux-libc-dev libpython3.7-dev libsox-dev python3-numpy python3-setuptools
source=http://raspbian.raspberrypi.org/raspbian/
keyring=raspbian-archive-keyring
components=main

Просмотреть файл

@ -10,7 +10,7 @@ bindings-clean:
# variables over several runs
bindings-build: ds-swig
pip3 install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==45.0.0
DISTUTILS_USE_SDK=1 PATH=$(TOOLCHAIN):$(DS_SWIG_BIN_PATH):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python3 ./setup.py build_ext $(PYTHON_PLATFORM_NAME)
DISTUTILS_USE_SDK=1 PATH=$(TOOLCHAIN_DIR):$(DS_SWIG_BIN_PATH):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python3 ./setup.py build_ext $(PYTHON_PLATFORM_NAME)
MANIFEST.in: bindings-build
> $@