diff --git a/.github/workflows/python.yml b/.github/workflows/build.yml similarity index 65% rename from .github/workflows/python.yml rename to .github/workflows/build.yml index 2924061..2246b3c 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -name: "Python Bindings" +name: "Build" 'on': push: branches: @@ -11,6 +11,7 @@ name: "Python Bindings" - '**' env: qt_version: "6.2.1" # only used by build-macos + emsdk_version: 2.0.9 # For use in emscripten build ccache_basedir: ${{ github.workspace }} ccache_dir: "${{ github.workspace }}/.ccache" ccache_compilercheck: content @@ -230,16 +231,161 @@ jobs: with: path: ${{github.workspace}}/dist/bergamot-*.whl + build-wasm: + name: "emscripten" + runs-on: ubuntu-latest + steps: + + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Set ccache environment for emcc + run: | + # We are hardcoding this to mtime instead of env pickup. Rest use content. + echo "CCACHE_COMPILER_CHECK=mtime" >> $GITHUB_ENV + + echo "CCACHE_BASEDIR=${{ env.ccache_basedir }}" >> $GITHUB_ENV + echo "CCACHE_COMPRESS=${{ env.ccache_compress }}" >> $GITHUB_ENV + echo "CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }}" >> $GITHUB_ENV + echo "CCACHE_DIR=${{ env.ccache_dir }}" >> $GITHUB_ENV + echo "CCACHE_MAXSIZE=${{ env.ccache_maxsize }}" >> $GITHUB_ENV + # https://emscripten.org/docs/compiling/Building-Projects.html#using-a-compiler-wrapper + echo "EM_COMPILER_WRAPPER=ccache" >> $GITHUB_ENV + + # This need to be run before setup, so ccache build caching doesn't complain. + - name: Obtain emsdk sources + run: | + git clone --depth 1 https://github.com/emscripten-core/emsdk.git + + - name: Cache-op for build-cache through ccache + uses: actions/cache@v2 + with: + path: | + ${{ env.ccache_dir }} + ${{ github.workspace }}/emsdk/ccache/git-emscripten_64bit/ + key: ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-${{ steps.ccache_vars.outputs.timestamp }} + restore-keys: |- + ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }} + ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }} + ccache-${{ github.job }}-${{ env.emsdk_version }} + + - name: Setup Emscripten toolchain + run: | + (cd emsdk && ./emsdk install ${{ env.emsdk_version }} ccache-git-emscripten-64bit) + (cd emsdk && ./emsdk activate ${{ env.emsdk_version }} ccache-git-emscripten-64bit) + # mtime of this file is checked by ccache, we set it to avoid cache misses. + touch -m -d '1 Jan 2021 12:00' emsdk/.emscripten + + # These needs to be done in the activated shell. + eval $(./emsdk/emsdk construct_env \ + | sed 's/export PATH=\(.*\);/echo \1 >> $GITHUB_PATH;/' \ + | sed 's/export \(.*\);/echo \1 >> $GITHUB_ENV;/' ); + + # This looks more permanent than version pinned, so keeping temporarily to avoid failures. + echo "${{ github.workspace }}/emsdk/ccache/git-emscripten_64bit/bin" >> $GITHUB_PATH + + - name: Generate ccache_vars for ccache based on machine + shell: bash + id: ccache_vars + run: |- + echo "::set-output name=hash::$(echo ${{ env.ccache_compilercheck }})" + echo "::set-output name=timestamp::$(date '+%Y-%m-%dT%H.%M.%S')" + + - name: Verify Emscripten setup + run: | + emcc --version + emcmake cmake --version + emmake make --version + + - name: ccache prolog + run: |- + ccache -s # Print current cache stats + ccache -z # Zero cache entry + + # WORMHOLE=off + - name: "Configure builds for WORMHOLE=off" + run: | + mkdir -p build-wasm-without-wormhole + cd build-wasm-without-wormhole + emcmake cmake -DCOMPILE_WASM=on -DWORMHOLE=off .. + + + - name: "Compile with WORMHOLE=off" + working-directory: build-wasm-without-wormhole + run: | + emmake make -j2 + + - name: ccache epilog + run: | + ccache -s # Print current cache stats + + - name: Import GEMM library from a separate wasm module + working-directory: build-wasm-without-wormhole + run: bash ../wasm/patch-artifacts-import-gemm-module.sh + + + # WORMHOLE=on + - name: "Configure builds for WORMHOLE=on" + run: | + mkdir -p build-wasm-with-wormhole + cd build-wasm-with-wormhole + emcmake cmake -DCOMPILE_WASM=on -DWORMHOLE=on .. + + + - name: "Compile with WORMHOLE=on" + working-directory: build-wasm-with-wormhole + run: | + emmake make -j2 + + - name: ccache epilog + run: | + ccache -s # Print current cache stats + + - name: Instantiate simd wormhole + working-directory: build-wasm-with-wormhole + run: bash ../wasm/patch-artifacts-enable-wormhole.sh + + - name: Import GEMM library from a separate wasm module + working-directory: build-wasm-with-wormhole + run: bash ../wasm/patch-artifacts-import-gemm-module.sh + + # Rename the wormhole on builds + - name: Rename artefacts with wormhole + working-directory: build-wasm-with-wormhole + run: | + mv bergamot-translator-worker{,-with-wormhole}.js + mv bergamot-translator-worker{,-with-wormhole}.js.bak + mv bergamot-translator-worker{,-with-wormhole}.wasm + + + # Upload both together. + - name: Upload wasm artifact + uses: actions/upload-artifact@v2 + with: + name: wasm-artefacts + if-no-files-found: error + path: | + # Without wormhole + ${{github.workspace}}/build-wasm-without-wormhole/bergamot-translator-worker.js + ${{github.workspace}}/build-wasm-without-wormhole/bergamot-translator-worker.wasm + ${{github.workspace}}/build-wasm-without-wormhole/bergamot-translator-worker.js.bak + + ${{github.workspace}}/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.js + ${{github.workspace}}/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.wasm + ${{github.workspace}}/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.js.bak + # Try to upload a release using https://github.com/marvinpinto/actions/issues/177#issuecomment-917605585 as a model release-latest: name: Release Latest Build runs-on: ubuntu-latest - needs: [python-ubuntu, python-macos] + needs: [python-ubuntu, python-macos, build-wasm] if: github.ref == 'refs/heads/main' steps: - name: Download artifacts uses: actions/download-artifact@v2 - + - name: Update GitHub prerelease uses: marvinpinto/action-automatic-releases@latest with: @@ -248,12 +394,16 @@ jobs: prerelease: true title: "Latest Build" files: | - ${{github.workspace}}/artifact/*.whl + artifact/*.whl + wasm-artefacts/build-wasm-without-wormhole/bergamot-translator-worker.js + wasm-artefacts/build-wasm-without-wormhole/bergamot-translator-worker.wasm + wasm-artefacts/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.js + wasm-artefacts/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.wasm release-version: name: Release version runs-on: ubuntu-latest - needs: [python-ubuntu, python-macos] + needs: [python-ubuntu, python-macos, build-wasm] permissions: contents: "write" packages: "write" @@ -271,7 +421,12 @@ jobs: prerelease: false title: "${{ github.ref_name }}" files: | - ${{github.workspace}}/artifact/*.whl + artifact/*.whl + wasm-artefacts/build-wasm-without-wormhole/bergamot-translator-worker.js + wasm-artefacts/build-wasm-without-wormhole/bergamot-translator-worker.wasm + wasm-artefacts/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.js + wasm-artefacts/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.wasm + python-checks: name: "formatting and typechecks" diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml deleted file mode 100644 index a8c199b..0000000 --- a/.github/workflows/wasm.yml +++ /dev/null @@ -1,123 +0,0 @@ -name: WebAssembly - -on: - push: - branches: [ main, ci-sandbox ] - pull_request: - branches: [ '**' ] - -env: - ccache_basedir: ${{ github.workspace }} - ccache_dir: "${{ github.workspace }}/.ccache" - ccache_compilercheck: mtime - ccache_compress: 'true' - ccache_compresslevel: 9 - ccache_maxsize: 200M - ccache_cmake: -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache - emsdk_version: 2.0.9 - -jobs: - build-wasm: - name: "emscripten" - runs-on: ubuntu-latest - steps: - - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: recursive - - - name: Set ccache environment for emcc - run: | - echo "CCACHE_COMPILER_CHECK=${{ env.ccache_compilercheck }}" >> $GITHUB_ENV - echo "CCACHE_BASEDIR=${{ env.ccache_basedir }}" >> $GITHUB_ENV - echo "CCACHE_COMPRESS=${{ env.ccache_compress }}" >> $GITHUB_ENV - echo "CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }}" >> $GITHUB_ENV - echo "CCACHE_DIR=${{ env.ccache_dir }}" >> $GITHUB_ENV - echo "CCACHE_MAXSIZE=${{ env.ccache_maxsize }}" >> $GITHUB_ENV - # https://emscripten.org/docs/compiling/Building-Projects.html#using-a-compiler-wrapper - echo "EM_COMPILER_WRAPPER=ccache" >> $GITHUB_ENV - - # This need to be run before setup, so ccache build caching doesn't complain. - - name: Obtain emsdk sources - run: | - git clone --depth 1 https://github.com/emscripten-core/emsdk.git - - - name: Cache-op for build-cache through ccache - uses: actions/cache@v2 - with: - path: | - ${{ env.ccache_dir }} - ${{ github.workspace }}/emsdk/ccache/git-emscripten_64bit/ - key: ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-${{ steps.ccache_vars.outputs.timestamp }} - restore-keys: |- - ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }} - ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }} - ccache-${{ github.job }}-${{ env.emsdk_version }} - - - name: Setup Emscripten toolchain - run: | - (cd emsdk && ./emsdk install ${{ env.emsdk_version }} ccache-git-emscripten-64bit) - (cd emsdk && ./emsdk activate ${{ env.emsdk_version }} ccache-git-emscripten-64bit) - # mtime of this file is checked by ccache, we set it to avoid cache misses. - touch -m -d '1 Jan 2021 12:00' emsdk/.emscripten - - # These needs to be done in the activated shell. - eval $(./emsdk/emsdk construct_env \ - | sed 's/export PATH=\(.*\);/echo \1 >> $GITHUB_PATH;/' \ - | sed 's/export \(.*\);/echo \1 >> $GITHUB_ENV;/' ); - - # This looks more permanent than version pinned, so keeping temporarily to avoid failures. - echo "${{ github.workspace }}/emsdk/ccache/git-emscripten_64bit/bin" >> $GITHUB_PATH - - - name: Generate ccache_vars for ccache based on machine - shell: bash - id: ccache_vars - run: |- - echo "::set-output name=hash::$(echo ${{ env.ccache_compilercheck }})" - echo "::set-output name=timestamp::$(date '+%Y-%m-%dT%H.%M.%S')" - - - name: Verify Emscripten setup - run: | - emcc --version - emcmake cmake --version - emmake make --version - - - name: Configure builds - run: | - mkdir -p build-wasm - cd build-wasm - emcmake cmake -DCOMPILE_WASM=on .. - - - name: ccache prolog - run: |- - ccache -s # Print current cache stats - ccache -z # Zero cache entry - - - name: Compile - working-directory: build-wasm - run: | - emmake make -j2 - - - name: ccache epilog - run: | - ccache -s # Print current cache stats - - - name: Instantiate simd wormhole - working-directory: build-wasm - run: bash ../wasm/patch-artifacts-enable-wormhole.sh - - - name: Import GEMM library from a separate wasm module - working-directory: build-wasm - run: bash ../wasm/patch-artifacts-import-gemm-module.sh - - - name: Upload wasm artifact - uses: actions/upload-artifact@v2 - with: - name: bergamot-translator-worker - if-no-files-found: error - path: | - ${{github.workspace}}/build-wasm/bergamot-translator-worker.js - ${{github.workspace}}/build-wasm/bergamot-translator-worker.wasm - ${{github.workspace}}/build-wasm/bergamot-translator-worker.js.bak -