From 0d0013002abc5f9b49c202500e5c96a498eb7575 Mon Sep 17 00:00:00 2001 From: Pelle Johnsen Date: Mon, 1 Nov 2021 21:45:51 +0100 Subject: [PATCH] Add Wasm build (#3752) * Add wasm build * Run wasm ci on push * Add copyright notice to wasm files * [wasm] Update Emscripten * [wasm] Change global lambda to regular function * [wasm] Show detected core count during build * [wasm] Set JS version from CHANGES, GITHUB_RUN_ID Also remove custom docker emscripten build with brotli, as not used * [wasm] Change github actions to use npm-publish * [wasm] Us docker-compose up for CI * [wasm] pass GITHUB_RUN_ID to docker * [wasm] Change GITHUB_RUN_ID to GITHUB_RUN_NUMBER * [wasm] Fix GITHUB_RUN_NUMBER in docker-compose.yml --- .github/workflows/wasm.yml | 20 ++++++++ docker-compose.yml | 10 ++++ source/wasm/README.md | 43 +++++++++++++++++ source/wasm/build.sh | 78 ++++++++++++++++++++++++++++++ source/wasm/package.json | 17 +++++++ source/wasm/spirv-tools.cpp | 93 ++++++++++++++++++++++++++++++++++++ source/wasm/spirv-tools.d.ts | 56 ++++++++++++++++++++++ test/wasm/test.js | 64 +++++++++++++++++++++++++ 8 files changed, 381 insertions(+) create mode 100644 .github/workflows/wasm.yml create mode 100644 docker-compose.yml create mode 100644 source/wasm/README.md create mode 100755 source/wasm/build.sh create mode 100644 source/wasm/package.json create mode 100644 source/wasm/spirv-tools.cpp create mode 100644 source/wasm/spirv-tools.d.ts create mode 100644 test/wasm/test.js diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml new file mode 100644 index 00000000..eef0e2c1 --- /dev/null +++ b/.github/workflows/wasm.yml @@ -0,0 +1,20 @@ +name: Wasm Build + +on: push + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Build web + run: docker-compose up + - name: Run tests + run: node test/wasm/test.js + - name: Publish to npm + uses: JS-DevTools/npm-publish@v1 + with: + token: ${{ secrets.NPM_TOKEN }} + package: ./out/web/package.json + dry-run: true diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..fb6d114f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,10 @@ +version: "3" +services: + build: + image: emscripten/emsdk:2.0.2 + environment: + GITHUB_RUN_NUMBER: ${GITHUB_RUN_NUMBER:-} + working_dir: /app + command: ./source/wasm/build.sh + volumes: + - ./:/app diff --git a/source/wasm/README.md b/source/wasm/README.md new file mode 100644 index 00000000..aca0f70c --- /dev/null +++ b/source/wasm/README.md @@ -0,0 +1,43 @@ +# SPIRV-Tools + +Wasm (WebAssembly) build of https://github.com/KhronosGroup/SPIRV-Tools + +## Usage + +```js +const spirvTools = require("spirv-tools"); + +const test = async () => { + // Load the library + const spv = await spirvTools(); + + // assemble + const source = ` + OpCapability Linkage + OpCapability Shader + OpMemoryModel Logical GLSL450 + OpSource GLSL 450 + OpDecorate %spec SpecId 1 + %int = OpTypeInt 32 1 + %spec = OpSpecConstant %int 0 + %const = OpConstant %int 42`; + const asResult = spv.as( + source, + spv.SPV_ENV_UNIVERSAL_1_3, + spv.SPV_TEXT_TO_BINARY_OPTION_NONE + ); + console.log(`as returned ${asResult.byteLength} bytes`); + + // re-disassemble + const disResult = spv.dis( + asResult, + spv.SPV_ENV_UNIVERSAL_1_3, + spv.SPV_BINARY_TO_TEXT_OPTION_INDENT | + spv.SPV_BINARY_TO_TEXT_OPTION_FRIENDLY_NAMES | + spv.SPV_BINARY_TO_TEXT_OPTION_COLOR + ); + console.log("dis:\n", disResult); +}; + +test(); +``` diff --git a/source/wasm/build.sh b/source/wasm/build.sh new file mode 100755 index 00000000..f02ae525 --- /dev/null +++ b/source/wasm/build.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +# Copyright (c) 2020 The Khronos Group Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +NUM_CORES=$(nproc) +echo "Detected $NUM_CORES cores for building" + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +VERSION=$(sed -n '0,/^v20/ s/^v\(20[0-9.]*\).*/\1/p' $DIR/../../CHANGES).${GITHUB_RUN_NUMBER:-0} +echo "Version: $VERSION" + +build() { + type=$1 + shift + args=$@ + mkdir -p build/$type + pushd build/$type + echo $args + emcmake cmake \ + -DCMAKE_BUILD_TYPE=Release \ + $args \ + ../.. + emmake make -j $(( $NUM_CORES )) SPIRV-Tools-static + + echo Building js interface + emcc \ + --bind \ + -I../../include \ + -std=c++11 \ + ../../source/wasm/spirv-tools.cpp \ + source/libSPIRV-Tools.a \ + -o spirv-tools.js \ + -s MODULARIZE \ + -Oz + + popd + mkdir -p out/$type + + # copy other js files + cp source/wasm/spirv-tools.d.ts out/$type/ + sed -e 's/\("version"\s*:\s*\).*/\1"'$VERSION'",/' source/wasm/package.json > out/$type/package.json + cp source/wasm/README.md out/$type/ + cp LICENSE out/$type/ + + cp build/$type/spirv-tools.js out/$type/ + gzip -9 -k -f out/$type/spirv-tools.js + if [ -e build/$type/spirv-tools.wasm ] ; then + cp build/$type/spirv-tools.wasm out/$type/ + gzip -9 -k -f out/$type/spirv-tools.wasm + fi +} + +if [ ! -d external/spirv-headers ] ; then + echo "Fetching SPIRV-headers" + git clone https://github.com/KhronosGroup/SPIRV-Headers.git external/spirv-headers +fi + +echo Building ${BASH_REMATCH[1]} +build web\ + -DSPIRV_COLOR_TERMINAL=OFF\ + -DSPIRV_SKIP_TESTS=ON\ + -DSPIRV_SKIP_EXECUTABLES=ON + +wc -c out/*/* diff --git a/source/wasm/package.json b/source/wasm/package.json new file mode 100644 index 00000000..78273538 --- /dev/null +++ b/source/wasm/package.json @@ -0,0 +1,17 @@ +{ + "name": "spirv-tools", + "version": "VERSION", + "license": "Apache-2.0", + "main": "spirv-tools", + "types": "spirv-tools.d.ts", + "files": [ + "*.wasm", + "*.js", + "*.d.ts" + ], + "repository": { + "type": "git", + "url": "https://github.com/KhronosGroup/SPIRV-Tools" + }, + "homepage": "https://github.com/KhronosGroup/SPIRV-Tools" +} diff --git a/source/wasm/spirv-tools.cpp b/source/wasm/spirv-tools.cpp new file mode 100644 index 00000000..90407f32 --- /dev/null +++ b/source/wasm/spirv-tools.cpp @@ -0,0 +1,93 @@ +// Copyright (c) 2020 The Khronos Group Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "spirv-tools/libspirv.hpp" + +#include +#include +#include + +#include +#include +using namespace emscripten; + +void print_msg_to_stderr (spv_message_level_t, const char*, + const spv_position_t&, const char* m) { + std::cerr << "error: " << m << std::endl; +}; + +std::string dis(std::string const& buffer, uint32_t env, uint32_t options) { + spvtools::SpirvTools core(static_cast(env)); + core.SetMessageConsumer(print_msg_to_stderr); + + std::vector spirv; + const uint32_t* ptr = reinterpret_cast(buffer.data()); + spirv.assign(ptr, ptr + buffer.size() / 4); + std::string disassembly; + if (!core.Disassemble(spirv, &disassembly, options)) return "Error"; + return disassembly; +} + +emscripten::val as(std::string const& source, uint32_t env, uint32_t options) { + spvtools::SpirvTools core(static_cast(env)); + core.SetMessageConsumer(print_msg_to_stderr); + + std::vector spirv; + if (!core.Assemble(source, &spirv, options)) spirv.clear(); + const uint8_t* ptr = reinterpret_cast(spirv.data()); + return emscripten::val(emscripten::typed_memory_view(spirv.size() * 4, + ptr)); +} + +EMSCRIPTEN_BINDINGS(my_module) { + function("dis", &dis); + function("as", &as); + + constant("SPV_ENV_UNIVERSAL_1_0", static_cast(SPV_ENV_UNIVERSAL_1_0)); + constant("SPV_ENV_VULKAN_1_0", static_cast(SPV_ENV_VULKAN_1_0)); + constant("SPV_ENV_UNIVERSAL_1_1", static_cast(SPV_ENV_UNIVERSAL_1_1)); + constant("SPV_ENV_OPENCL_2_1", static_cast(SPV_ENV_OPENCL_2_1)); + constant("SPV_ENV_OPENCL_2_2", static_cast(SPV_ENV_OPENCL_2_2)); + constant("SPV_ENV_OPENGL_4_0", static_cast(SPV_ENV_OPENGL_4_0)); + constant("SPV_ENV_OPENGL_4_1", static_cast(SPV_ENV_OPENGL_4_1)); + constant("SPV_ENV_OPENGL_4_2", static_cast(SPV_ENV_OPENGL_4_2)); + constant("SPV_ENV_OPENGL_4_3", static_cast(SPV_ENV_OPENGL_4_3)); + constant("SPV_ENV_OPENGL_4_5", static_cast(SPV_ENV_OPENGL_4_5)); + constant("SPV_ENV_UNIVERSAL_1_2", static_cast(SPV_ENV_UNIVERSAL_1_2)); + constant("SPV_ENV_OPENCL_1_2", static_cast(SPV_ENV_OPENCL_1_2)); + constant("SPV_ENV_OPENCL_EMBEDDED_1_2", static_cast(SPV_ENV_OPENCL_EMBEDDED_1_2)); + constant("SPV_ENV_OPENCL_2_0", static_cast(SPV_ENV_OPENCL_2_0)); + constant("SPV_ENV_OPENCL_EMBEDDED_2_0", static_cast(SPV_ENV_OPENCL_EMBEDDED_2_0)); + constant("SPV_ENV_OPENCL_EMBEDDED_2_1", static_cast(SPV_ENV_OPENCL_EMBEDDED_2_1)); + constant("SPV_ENV_OPENCL_EMBEDDED_2_2", static_cast(SPV_ENV_OPENCL_EMBEDDED_2_2)); + constant("SPV_ENV_UNIVERSAL_1_3", static_cast(SPV_ENV_UNIVERSAL_1_3)); + constant("SPV_ENV_VULKAN_1_1", static_cast(SPV_ENV_VULKAN_1_1)); + constant("SPV_ENV_WEBGPU_0", static_cast(SPV_ENV_WEBGPU_0)); + constant("SPV_ENV_UNIVERSAL_1_4", static_cast(SPV_ENV_UNIVERSAL_1_4)); + constant("SPV_ENV_VULKAN_1_1_SPIRV_1_4", static_cast(SPV_ENV_VULKAN_1_1_SPIRV_1_4)); + constant("SPV_ENV_UNIVERSAL_1_5", static_cast(SPV_ENV_UNIVERSAL_1_5)); + constant("SPV_ENV_VULKAN_1_2", static_cast(SPV_ENV_VULKAN_1_2)); + + + constant("SPV_BINARY_TO_TEXT_OPTION_NONE", static_cast(SPV_BINARY_TO_TEXT_OPTION_NONE)); + constant("SPV_BINARY_TO_TEXT_OPTION_PRINT", static_cast(SPV_BINARY_TO_TEXT_OPTION_PRINT)); + constant("SPV_BINARY_TO_TEXT_OPTION_COLOR", static_cast(SPV_BINARY_TO_TEXT_OPTION_COLOR)); + constant("SPV_BINARY_TO_TEXT_OPTION_INDENT", static_cast(SPV_BINARY_TO_TEXT_OPTION_INDENT)); + constant("SPV_BINARY_TO_TEXT_OPTION_SHOW_BYTE_OFFSET", static_cast(SPV_BINARY_TO_TEXT_OPTION_SHOW_BYTE_OFFSET)); + constant("SPV_BINARY_TO_TEXT_OPTION_NO_HEADER", static_cast(SPV_BINARY_TO_TEXT_OPTION_NO_HEADER)); + constant("SPV_BINARY_TO_TEXT_OPTION_FRIENDLY_NAMES", static_cast(SPV_BINARY_TO_TEXT_OPTION_FRIENDLY_NAMES)); + + constant("SPV_TEXT_TO_BINARY_OPTION_NONE", static_cast(SPV_TEXT_TO_BINARY_OPTION_NONE)); + constant("SPV_TEXT_TO_BINARY_OPTION_PRESERVE_NUMERIC_IDS", static_cast(SPV_TEXT_TO_BINARY_OPTION_PRESERVE_NUMERIC_IDS)); +} \ No newline at end of file diff --git a/source/wasm/spirv-tools.d.ts b/source/wasm/spirv-tools.d.ts new file mode 100644 index 00000000..9c197973 --- /dev/null +++ b/source/wasm/spirv-tools.d.ts @@ -0,0 +1,56 @@ +// Copyright (c) 2020 The Khronos Group Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +declare interface SpirvTools { + as(input: string, env: number, options: number): Uint8Array; + dis(input: Uint8Array, env: number, options: number): string; + + SPV_ENV_UNIVERSAL_1_0: number; + SPV_ENV_VULKAN_1_0: number; + SPV_ENV_UNIVERSAL_1_1: number; + SPV_ENV_OPENCL_2_1: number; + SPV_ENV_OPENCL_2_2: number; + SPV_ENV_OPENGL_4_0: number; + SPV_ENV_OPENGL_4_1: number; + SPV_ENV_OPENGL_4_2: number; + SPV_ENV_OPENGL_4_3: number; + SPV_ENV_OPENGL_4_5: number; + SPV_ENV_UNIVERSAL_1_2: number; + SPV_ENV_OPENCL_1_2: number; + SPV_ENV_OPENCL_EMBEDDED_1_2: number; + SPV_ENV_OPENCL_2_0: number; + SPV_ENV_OPENCL_EMBEDDED_2_0: number; + SPV_ENV_OPENCL_EMBEDDED_2_1: number; + SPV_ENV_OPENCL_EMBEDDED_2_2: number; + SPV_ENV_UNIVERSAL_1_3: number; + SPV_ENV_VULKAN_1_1: number; + SPV_ENV_WEBGPU_0: number; + SPV_ENV_UNIVERSAL_1_4: number; + SPV_ENV_VULKAN_1_1_SPIRV_1_4: number; + SPV_ENV_UNIVERSAL_1_5: number; + SPV_ENV_VULKAN_1_2: number; + + SPV_TEXT_TO_BINARY_OPTION_NONE: number; + SPV_TEXT_TO_BINARY_OPTION_PRESERVE_NUMERIC_IDS: number; + + SPV_BINARY_TO_TEXT_OPTION_NONE: number; + SPV_BINARY_TO_TEXT_OPTION_PRINT: number; + SPV_BINARY_TO_TEXT_OPTION_COLOR: number; + SPV_BINARY_TO_TEXT_OPTION_INDENT: number; + SPV_BINARY_TO_TEXT_OPTION_SHOW_BYTE_OFFSET: number; + SPV_BINARY_TO_TEXT_OPTION_NO_HEADER: number; + SPV_BINARY_TO_TEXT_OPTION_FRIENDLY_NAMES: number; +} + +export default function (): Promise; diff --git a/test/wasm/test.js b/test/wasm/test.js new file mode 100644 index 00000000..7f0d8f3c --- /dev/null +++ b/test/wasm/test.js @@ -0,0 +1,64 @@ +// Copyright (c) 2020 The Khronos Group Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const spirvTools = require("../../out/web/spirv-tools"); +const fs = require("fs"); +const util = require("util"); +const readFile = util.promisify(fs.readFile); + +const SPV_PATH = "./test/fuzzers/corpora/spv/simple.spv"; + +const test = async () => { + const spv = await spirvTools(); + + // disassemble from file + const buffer = await readFile(SPV_PATH); + const disFileResult = spv.dis( + buffer, + spv.SPV_ENV_UNIVERSAL_1_3, + spv.SPV_BINARY_TO_TEXT_OPTION_INDENT | + spv.SPV_BINARY_TO_TEXT_OPTION_FRIENDLY_NAMES | + spv.SPV_BINARY_TO_TEXT_OPTION_COLOR + ); + console.log("dis from file:\n", disFileResult); + + // assemble + const source = ` + OpCapability Linkage + OpCapability Shader + OpMemoryModel Logical GLSL450 + OpSource GLSL 450 + OpDecorate %spec SpecId 1 + %int = OpTypeInt 32 1 + %spec = OpSpecConstant %int 0 + %const = OpConstant %int 42`; + const asResult = spv.as( + source, + spv.SPV_ENV_UNIVERSAL_1_3, + spv.SPV_TEXT_TO_BINARY_OPTION_NONE + ); + console.log(`as returned ${asResult.byteLength} bytes`); + + // re-disassemble + const disResult = spv.dis( + asResult, + spv.SPV_ENV_UNIVERSAL_1_3, + spv.SPV_BINARY_TO_TEXT_OPTION_INDENT | + spv.SPV_BINARY_TO_TEXT_OPTION_FRIENDLY_NAMES | + spv.SPV_BINARY_TO_TEXT_OPTION_COLOR + ); + console.log("dis:\n", disResult); +}; + +test();