Fix #3578: Re-instate Python TF/TFLite tests on GitHub Actions / macOS

.github/actions/node-build/action.yml

@@ -29,6 +29,9 @@ runs:
     - run: |
         npm update
       shell: bash
+    - run: |
+        mkdir -p tmp/headers/nodejs tmp/headers/electronjs
+      shell: bash
     - run: |
         for node in ${{ inputs.nodejs_versions }}; do
           EXTRA_CFLAGS=${{ inputs.local_cflags }} \
@@ -36,6 +39,7 @@ runs:
           EXTRA_LIBS=${{ inputs.local_libs }} \
             make -C native_client/javascript \
               NODE_ABI_TARGET=--target=${node} \
+              NODE_DEVDIR=--devdir=tmp/headers/nodejs/${node} \
             clean node-wrapper
         done;
       shell: bash
@@ -48,6 +52,7 @@ runs:
               NODE_ABI_TARGET=--target=${electron} \
               NODE_DIST_URL=--disturl=https://electronjs.org/headers \
               NODE_RUNTIME=--runtime=electron \
+              NODE_DEVDIR=--devdir=tmp/headers/electronjs/${electron} \
             clean node-wrapper
         done;
       shell: bash

.github/actions/run-tests/action.yml

@@ -1,6 +1,9 @@
-name: "C++ tests"
+name: "Tests execution"
-description: "Running C++ binary tests"
+description: "Running DeepSpeech tests"
 inputs:
+  runtime:
+    description: "Runtime to use for running test"
+    required: true
   build-flavor:
     description: "Running against TF or TFLite"
     required: true
@@ -23,5 +26,6 @@ runs:
         if [ "${{ inputs.model-kind }}" = "prod" ]; then
           model_kind="-prod"
         fi
-        ./ci_scripts/cpp${build}-ds-tests${model_kind}.sh ${{ inputs.bitrate }}
+
+        ./ci_scripts/${{ inputs.runtime }}${build}-tests${model_kind}.sh ${{ inputs.bitrate }}
       shell: bash

.github/workflows/macOS-amd64.yml

@@ -89,7 +89,6 @@ jobs:
     runs-on: macos-10.15
     strategy:
       matrix:
-        build-flavor: ["tf", "tflite"]
         bitrate: ["8k", "16k"]
     steps:
       - uses: actions/checkout@v2
@@ -124,14 +123,11 @@ jobs:
           mv data/smoke_test/LDC93S1_pcms16le_1_${bits}.wav data/smoke_test/LDC93S1.wav
 
           ./bin/run-tc-ldc93s1_new.sh 249 ${bits}
-          if [ "${{ matrix.build-flavor }}" = "tflite" ]; then
+          ./bin/run-tc-ldc93s1_tflite.sh ${bits}
-            ./bin/run-tc-ldc93s1_tflite.sh ${bits}
-          fi
       - run: |
           curl -vsSL https://github.com/mozilla/DeepSpeech/releases/download/v0.9.3/macOS.amd64.convert_graphdef_memmapped_format.xz | xz -d > /tmp/convert_graphdef_memmapped_format
           chmod +x /tmp/convert_graphdef_memmapped_format
           /tmp/convert_graphdef_memmapped_format --in_graph=/tmp/train/output_graph.pb --out_graph=/tmp/train/output_graph.pbmm
-        if: matrix.build-flavor == 'tf'
       - run: |
           tar -cf - \
             -C /tmp/ckpt/ . \
@@ -143,11 +139,15 @@ jobs:
           ls -hal /tmp/ ${{ github.workspace }}/tmp/
       - uses: actions/upload-artifact@v2
         with:
-          name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip"
+          name: "test-model.tf-${{ matrix.bitrate }}.zip"
-          path: ${{ github.workspace }}/tmp/output_graph.*
+          path: ${{ github.workspace }}/tmp/output_graph.pb*
       - uses: actions/upload-artifact@v2
         with:
-          name: "test-checkpoint.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip"
+          name: "test-model.tflite-${{ matrix.bitrate }}.zip"
+          path: ${{ github.workspace }}/tmp/output_graph.tflite
+      - uses: actions/upload-artifact@v2
+        with:
+          name: "test-checkpoint.${{ matrix.bitrate }}.zip"
           path: ${{ github.workspace }}/tmp/checkpoint.tar.xz
   tensorflow_opt-macOS:
     name: "Check cache for TensorFlow"
@@ -322,7 +322,7 @@ jobs:
           path: ${{ github.workspace }}/native_client/javascript/wrapper.tar.gz
       - uses: actions/upload-artifact@v2
         with:
-          name: "deepspeech-${{ matrix.build-flavor }}.tar.gz"
+          name: "deepspeech-${{ matrix.build-flavor }}.tgz"
           path: ${{ github.workspace }}/native_client/javascript/deepspeech-*.tgz
   test-cpp-macOS:
     name: "Test C++ binary on macOS"
@@ -358,8 +358,144 @@ jobs:
       - run: |
           ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/
         if: matrix.models == 'test'
-      - uses: ./.github/actions/cpp-tests
+      - uses: ./.github/actions/run-tests
         with:
+          runtime: "cpp"
           build-flavor: ${{ matrix.build-flavor }}
           bitrate: ${{ matrix.bitrate }}
           model-kind: ${{ matrix.models }}
+  test-py-macOS:
+    name: "Test Python bindings on macOS"
+    runs-on: macos-10.15
+    needs: [ build-python-macOS, train-test-model ]
+    strategy:
+      matrix:
+        python-version: [3.6.8, 3.7.9, 3.8.8, 3.9.2]
+        build-flavor: ["tf", "tflite"]
+        models: ["test", "prod"]
+        bitrate: ["8k", "16k"]
+    env:
+      TASKCLUSTER_TMP_DIR: ${{ github.workspace }}/tmp/
+      DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb
+      DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm
+      DEEPSPEECH_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
+      EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: actions/download-artifact@v2
+        with:
+          name: "deepspeech-${{ matrix.build-flavor }}-${{ matrix.python-version }}.whl"
+          path: ${{ env.TASKCLUSTER_TMP_DIR }}
+      - uses: actions/download-artifact@v2
+        with:
+          name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip"
+          path: ${{ env.TASKCLUSTER_TMP_DIR }}
+        if: matrix.models == 'test'
+      - run: |
+          ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/
+        if: matrix.models == 'test'
+      - run: |
+          ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/
+          pip3 install --only-binary :all: --upgrade ${{ env.TASKCLUSTER_TMP_DIR }}/deepspeech*.whl
+      - uses: ./.github/actions/run-tests
+        with:
+          runtime: "python"
+          build-flavor: ${{ matrix.build-flavor }}
+          bitrate: ${{ matrix.bitrate }}
+          model-kind: ${{ matrix.models }}
+  test-nodejs-macOS:
+    name: "Test NodeJS bindings on macOS"
+    runs-on: macos-10.15
+    needs: [ build-nodejs-macOS, train-test-model ]
+    strategy:
+      matrix:
+        nodejs-version: [10, 11, 12, 13, 14, 15]
+        build-flavor: ["tf", "tflite"]
+        models: ["test", "prod"]
+        bitrate: ["8k", "16k"]
+    env:
+      TASKCLUSTER_TMP_DIR: ${{ github.workspace }}/tmp/
+      DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb
+      DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm
+      DEEPSPEECH_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
+      EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - uses: actions/setup-node@v2
+        with:
+          node-version: ${{ matrix.nodejs-version }}
+      - uses: actions/download-artifact@v2
+        with:
+          name: "deepspeech-${{ matrix.build-flavor }}.tgz"
+          path: ${{ env.TASKCLUSTER_TMP_DIR }}
+      - uses: actions/download-artifact@v2
+        with:
+          name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip"
+          path: ${{ env.TASKCLUSTER_TMP_DIR }}
+        if: matrix.models == 'test'
+      - run: |
+          ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/
+        if: matrix.models == 'test'
+      - run: |
+          ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/
+          npm install ${{ env.TASKCLUSTER_TMP_DIR }}/deepspeech*.tgz
+      - uses: ./.github/actions/run-tests
+        with:
+          runtime: "node"
+          build-flavor: ${{ matrix.build-flavor }}
+          bitrate: ${{ matrix.bitrate }}
+          model-kind: ${{ matrix.models }}
+  test-electronjs-macOS:
+    name: "Test ElectronJS bindings on macOS"
+    runs-on: macos-10.15
+    needs: [ build-nodejs-macOS, train-test-model ]
+    strategy:
+      matrix:
+        electronjs-version: [5.0.13, 6.0.12, 6.1.7, 7.0.1, 7.1.8, 8.0.1, 9.0.1, 9.1.0, 9.2.0, 10.0.0, 10.1.0, 11.0.0, 12.0.0]
+        build-flavor: ["tf", "tflite"]
+        models: ["test"]
+        bitrate: ["8k", "16k"]
+    env:
+      TASKCLUSTER_TMP_DIR: ${{ github.workspace }}/tmp/
+      DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb
+      DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm
+      DEEPSPEECH_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
+      EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 12
+      - uses: actions/download-artifact@v2
+        with:
+          name: "deepspeech-${{ matrix.build-flavor }}.tgz"
+          path: ${{ env.TASKCLUSTER_TMP_DIR }}
+      - uses: actions/download-artifact@v2
+        with:
+          name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip"
+          path: ${{ env.TASKCLUSTER_TMP_DIR }}
+        if: matrix.models == 'test'
+      - run: |
+          ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/
+        if: matrix.models == 'test'
+      - run: |
+          ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/
+          npm install ${{ env.TASKCLUSTER_TMP_DIR }}/deepspeech*.tgz
+      - run: |
+          npm install electron@${{ matrix.electronjs-version }}
+      - uses: ./.github/actions/run-tests
+        with:
+          runtime: "electronjs"
+          build-flavor: ${{ matrix.build-flavor }}
+          bitrate: ${{ matrix.bitrate }}
+          model-kind: ${{ matrix.models }}
+        timeout-minutes: 5

ci_scripts/all-utils.sh

@@ -106,3 +106,13 @@ verify_bazel_rebuild()
     exit 1
   fi;
 }
+
+symlink_electron()
+{
+  ln -s Electron.app/Contents/MacOS/Electron node_modules/electron/dist/node
+}
+
+export_node_bin_path()
+{
+  export PATH=$(pwd)/node_modules/.bin/:$(pwd)/node_modules/electron/dist/:$PATH
+}

ci_scripts/asserts.sh

@@ -432,7 +432,7 @@ run_prod_concurrent_stream_tests()
   local _bitrate=$1
 
   set +e
-  output=$(python ${TASKCLUSTER_TMP_DIR}/test_sources/concurrent_streams.py \
+  output=$(python3 ${TASKCLUSTER_TMP_DIR}/test_sources/concurrent_streams.py \
              --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} \
              --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer \
              --audio1 ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_16000.wav \

ci_scripts/electronjs-tests.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -xe
+
+source $(dirname "$0")/all-vars.sh
+source $(dirname "$0")/all-utils.sh
+source $(dirname "$0")/asserts.sh
+
+bitrate=$1
+set_ldc_sample_filename "${bitrate}"
+
+download_data
+
+node --version
+npm --version
+
+symlink_electron
+
+export_node_bin_path
+
+which electron
+which node
+
+node --version
+
+deepspeech --version
+
+check_runtime_electronjs
+
+run_electronjs_inference_tests

ci_scripts/electronjs_tflite-tests.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -xe
+
+source $(dirname "$0")/all-vars.sh
+source $(dirname "$0")/all-utils.sh
+source $(dirname "$0")/asserts.sh
+
+bitrate=$1
+set_ldc_sample_filename "${bitrate}"
+
+model_source=${DEEPSPEECH_TEST_MODEL//.pb/.tflite}
+model_name=$(basename "${model_source}")
+model_name_mmap=$(basename "${model_source}")
+
+download_data
+
+node --version
+npm --version
+
+symlink_electron
+
+export_node_bin_path
+
+which electron
+which node
+
+node --version
+
+deepspeech --version
+
+check_runtime_electronjs
+
+run_electronjs_inference_tests

ci_scripts/node-tests-prod.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -xe
+
+source $(dirname "$0")/all-vars.sh
+source $(dirname "$0")/all-utils.sh
+source $(dirname "$0")/asserts.sh
+
+bitrate=$1
+set_ldc_sample_filename "${bitrate}"
+
+model_source=${DEEPSPEECH_PROD_MODEL}
+model_name=$(basename "${model_source}")
+
+model_source_mmap=${DEEPSPEECH_PROD_MODEL_MMAP}
+model_name_mmap=$(basename "${model_source_mmap}")
+
+download_model_prod
+
+download_data
+
+node --version
+npm --version
+
+export_node_bin_path
+
+check_runtime_nodejs
+
+run_prod_inference_tests "${bitrate}"
+
+run_js_streaming_prod_inference_tests "${bitrate}"

ci_scripts/node-tests.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -xe
+
+source $(dirname "$0")/all-vars.sh
+source $(dirname "$0")/all-utils.sh
+source $(dirname "$0")/asserts.sh
+
+bitrate=$1
+set_ldc_sample_filename "${bitrate}"
+
+download_data
+
+node --version
+npm --version
+
+export_node_bin_path
+
+check_runtime_nodejs
+
+run_all_inference_tests
+
+run_js_streaming_inference_tests
+
+run_hotword_tests

ci_scripts/node_tflite-tests-prod.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -xe
+
+source $(dirname "$0")/all-vars.sh
+source $(dirname "$0")/all-utils.sh
+source $(dirname "$0")/asserts.sh
+
+bitrate=$1
+set_ldc_sample_filename "${bitrate}"
+
+model_source=${DEEPSPEECH_PROD_MODEL//.pb/.tflite}
+model_name=$(basename "${model_source}")
+model_name_mmap=$(basename "${model_source}")
+model_source_mmap=${DEEPSPEECH_PROD_MODEL_MMAP//.pbmm/.tflite}
+
+download_model_prod
+
+download_data
+
+node --version
+npm --version
+
+export_node_bin_path
+
+check_runtime_nodejs
+
+run_prodtflite_inference_tests "${bitrate}"
+
+run_js_streaming_prodtflite_inference_tests "${bitrate}"

ci_scripts/node_tflite-tests.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -xe
+
+source $(dirname "$0")/all-vars.sh
+source $(dirname "$0")/all-utils.sh
+source $(dirname "$0")/asserts.sh
+
+bitrate=$1
+set_ldc_sample_filename "${bitrate}"
+
+model_source=${DEEPSPEECH_TEST_MODEL//.pb/.tflite}
+model_name=$(basename "${model_source}")
+model_name_mmap=$(basename "${model_source}")
+
+download_data
+
+node --version
+npm --version
+
+export_node_bin_path
+
+check_runtime_nodejs
+
+run_all_inference_tests
+
+run_js_streaming_inference_tests
+
+run_hotword_tests

ci_scripts/python-tests-prod.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -xe
+
+source $(dirname "$0")/all-vars.sh
+source $(dirname "$0")/all-utils.sh
+source $(dirname "$0")/asserts.sh
+
+bitrate=$1
+set_ldc_sample_filename "${bitrate}"
+
+model_source=${DEEPSPEECH_PROD_MODEL}
+model_name=$(basename "${model_source}")
+
+model_source_mmap=${DEEPSPEECH_PROD_MODEL_MMAP}
+model_name_mmap=$(basename "${model_source_mmap}")
+
+download_model_prod
+
+download_material
+
+which deepspeech
+deepspeech --version
+
+run_prod_inference_tests "${bitrate}"
+
+run_prod_concurrent_stream_tests "${bitrate}"

ci_scripts/python-tests.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -xe
+
+source $(dirname "$0")/all-vars.sh
+source $(dirname "$0")/all-utils.sh
+source $(dirname "$0")/asserts.sh
+
+bitrate=$1
+set_ldc_sample_filename "${bitrate}"
+
+download_data
+
+which deepspeech
+deepspeech --version
+
+run_all_inference_tests
+
+run_hotword_tests

ci_scripts/python_tflite-tests-prod.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -xe
+
+source $(dirname "$0")/all-vars.sh
+source $(dirname "$0")/all-utils.sh
+source $(dirname "$0")/asserts.sh
+
+bitrate=$1
+set_ldc_sample_filename "${bitrate}"
+
+model_source=${DEEPSPEECH_PROD_MODEL//.pb/.tflite}
+model_name=$(basename "${model_source}")
+model_name_mmap=$(basename "${model_source}")
+model_source_mmap=${DEEPSPEECH_PROD_MODEL_MMAP//.pbmm/.tflite}
+
+download_model_prod
+
+download_material
+
+which deepspeech
+deepspeech --version
+
+run_prodtflite_inference_tests "${bitrate}"

ci_scripts/python_tflite-tests.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -xe
+
+source $(dirname "$0")/all-vars.sh
+source $(dirname "$0")/all-utils.sh
+source $(dirname "$0")/asserts.sh
+
+bitrate=$1
+set_ldc_sample_filename "${bitrate}"
+
+model_source=${DEEPSPEECH_TEST_MODEL//.pb/.tflite}
+model_name=$(basename "${model_source}")
+model_name_mmap=$(basename "${model_source}")
+
+download_data
+
+which deepspeech
+deepspeech --version
+
+run_all_inference_tests
+
+run_hotword_tests