From 2e331715ff7b304529a7ce65a98184681eceaab9 Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Fri, 8 Jul 2022 16:27:53 -0700 Subject: [PATCH] devops: support universal webkit builds (#15496) --- .../checkout_build_archive_upload.sh | 7 + browser_patches/webkit/EXPECTED_BUILDS | 1 + browser_patches/webkit/archive.sh | 33 +++-- browser_patches/webkit/build.sh | 127 ++++++++++++++---- 4 files changed, 132 insertions(+), 36 deletions(-) diff --git a/browser_patches/checkout_build_archive_upload.sh b/browser_patches/checkout_build_archive_upload.sh index 9009eeeec1..5a5e211829 100755 --- a/browser_patches/checkout_build_archive_upload.sh +++ b/browser_patches/checkout_build_archive_upload.sh @@ -306,6 +306,13 @@ elif [[ "$BUILD_FLAVOR" == "firefox-beta-win64" ]]; then # =========================== # WEBKIT COMPILATION # =========================== +elif [[ "$BUILD_FLAVOR" == "webkit-universal" ]]; then + BROWSER_NAME="webkit" + EXTRA_BUILD_ARGS="--full --universal" + EXTRA_ARCHIVE_ARGS="--universal" + EXPECTED_HOST_OS="Ubuntu" + EXPECTED_HOST_OS_VERSION="20.04" + BUILD_BLOB_NAME="webkit-linux-universal.zip" elif [[ "$BUILD_FLAVOR" == "webkit-ubuntu-18.04" ]]; then BROWSER_NAME="webkit" EXTRA_BUILD_ARGS="--full" diff --git a/browser_patches/webkit/EXPECTED_BUILDS b/browser_patches/webkit/EXPECTED_BUILDS index b395d8b1a0..59fd5739b7 100644 --- a/browser_patches/webkit/EXPECTED_BUILDS +++ b/browser_patches/webkit/EXPECTED_BUILDS @@ -1,3 +1,4 @@ +webkit-linux-universal.zip webkit-ubuntu-18.04.zip webkit-ubuntu-20.04.zip webkit-ubuntu-20.04-arm64.zip diff --git a/browser_patches/webkit/archive.sh b/browser_patches/webkit/archive.sh index 15adba3ba9..e3b727d788 100755 --- a/browser_patches/webkit/archive.sh +++ b/browser_patches/webkit/archive.sh @@ -3,7 +3,7 @@ set -e set +x if [[ ("$1" == "-h") || ("$1" == "--help") ]]; then - echo "usage: $(basename "$0") [output-absolute-path]" + echo "usage: $(basename "$0") [output-absolute-path] [--universal]" echo echo "Generate distributable .zip archive from ./checkout folder that was previously built." echo @@ -28,6 +28,11 @@ if ! [[ -d $(dirname "$ZIP_PATH") ]]; then exit 1 fi +IS_UNIVERSAL="" +if [[ $2 == "--universal" ]]; then + IS_UNIVERSAL=1 +fi + main() { if [[ ! -z "${WK_CHECKOUT_PATH}" ]]; then cd "${WK_CHECKOUT_PATH}" @@ -52,7 +57,7 @@ main() { createZipForLinux() { # create a TMP directory to copy all necessary files - local tmpdir=$(mktemp -d -t webkit-deploy-XXXXXXXXXX) + local tmpdir=$(mktemp -d -p "$(pwd)/WebKitBuild" -t webkit-deploy-XXXXXXXXXX) mkdir -p "$tmpdir" # copy runner @@ -62,15 +67,27 @@ createZipForLinux() { # Generate and unpack MiniBrowser bundles for each port for port in gtk wpe; do - WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/${port^^} Tools/Scripts/generate-bundle \ - --bundle=MiniBrowser --release \ - --platform=${port} --destination="${tmpdir}" - unzip "${tmpdir}"/MiniBrowser_${port}_release.zip -d "${tmpdir}"/minibrowser-${port} - rm -f "${tmpdir}"/MiniBrowser_${port}_release.zip + if [[ -n "${IS_UNIVERSAL}" ]]; then + Tools/Scripts/generate-bundle \ + --syslibs=bundle-all \ + --bundle=MiniBrowser --release \ + --platform=${port} --destination="${tmpdir}" + else + WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/${port^^} \ + Tools/Scripts/generate-bundle \ + --bundle=MiniBrowser --release \ + --platform=${port} --destination="${tmpdir}" + fi + + unzip "${tmpdir}"/MiniBrowser_${port}_release.zip -d "${tmpdir}"/minibrowser-${port} + rm -f "${tmpdir}"/MiniBrowser_${port}_release.zip done - # tar resulting directory and cleanup TMP. cd "$tmpdir" + # de-duplicate common files: convert to relative symlinks identical files (same hash) + rdfind -deterministic true -makesymlinks true -makehardlinks false -makeresultsfile false . + symlinks -rc . + # zip resulting directory and cleanup TMP. zip --symlinks -r "$ZIP_PATH" ./ cd - rm -rf "$tmpdir" diff --git a/browser_patches/webkit/build.sh b/browser_patches/webkit/build.sh index 063d2c413f..4af7549432 100755 --- a/browser_patches/webkit/build.sh +++ b/browser_patches/webkit/build.sh @@ -7,9 +7,18 @@ cd "$(dirname "$0")" SCRIPT_FOLDER="$(pwd -P)" source "${SCRIPT_FOLDER}/../utils.sh" +# On Linux, Universal build uses Flatpak rather then JHBuild +# and packs into a universal binary that can run on any linux +# distribution. +IS_UNIVERSAL_BUILD="" + build_gtk() { - if ! [[ -d ./WebKitBuild/GTK/DependenciesGTK ]]; then - yes | WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/GTK DEBIAN_FRONTEND=noninteractive ./Tools/Scripts/update-webkitgtk-libs + if [[ -z "${IS_UNIVERSAL_BUILD}" && ! -d "./WebKitBuild/GTK/DependenciesGTK" ]]; then + yes | WEBKIT_JHBUILD=1 \ + WEBKIT_JHBUILD_MODULESET=minimal \ + WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/GTK \ + DEBIAN_FRONTEND=noninteractive \ + ./Tools/Scripts/update-webkitgtk-libs fi local CMAKE_ARGS=( --cmakeargs=-DENABLE_INTROSPECTION=OFF @@ -18,12 +27,23 @@ build_gtk() { if [[ -n "${EXPORT_COMPILE_COMMANDS}" ]]; then CMAKE_ARGS+=("--cmakeargs=-DCMAKE_EXPORT_COMPILE_COMMANDS=1") fi - WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/GTK ./Tools/Scripts/build-webkit --gtk --release "${CMAKE_ARGS}" --touch-events --orientation-events --no-bubblewrap-sandbox "${CMAKE_ARGS[@]}" MiniBrowser + if [[ -n "${IS_UNIVERSAL_BUILD}" ]]; then + ./Tools/Scripts/build-webkit --gtk --release "${CMAKE_ARGS}" --touch-events --orientation-events --no-bubblewrap-sandbox "${CMAKE_ARGS[@]}" MiniBrowser + else + WEBKIT_JHBUILD=1 \ + WEBKIT_JHBUILD_MODULESET=minimal \ + WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/GTK \ + ./Tools/Scripts/build-webkit --gtk --release "${CMAKE_ARGS}" --touch-events --orientation-events --no-bubblewrap-sandbox "${CMAKE_ARGS[@]}" MiniBrowser + fi } build_wpe() { - if ! [[ -d ./WebKitBuild/WPE/DependenciesWPE ]]; then - yes | WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/WPE DEBIAN_FRONTEND=noninteractive ./Tools/Scripts/update-webkitwpe-libs + if [[ -z "${IS_UNIVERSAL_BUILD}" && ! -d "./WebKitBuild/WPE/DependenciesWPE" ]]; then + yes | WEBKIT_JHBUILD=1 \ + WEBKIT_JHBUILD_MODULESET=minimal \ + WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/WPE \ + DEBIAN_FRONTEND=noninteractive \ + ./Tools/Scripts/update-webkitwpe-libs fi local CMAKE_ARGS=( --cmakeargs=-DENABLE_COG=OFF @@ -34,14 +54,37 @@ build_wpe() { if [[ -n "${EXPORT_COMPILE_COMMANDS}" ]]; then CMAKE_ARGS+=("--cmakeargs=-DCMAKE_EXPORT_COMPILE_COMMANDS=1") fi - WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/WPE ./Tools/Scripts/build-webkit --wpe --release "${CMAKE_ARGS}" --touch-events --orientation-events --no-bubblewrap-sandbox "${CMAKE_ARGS[@]}" MiniBrowser + + if [[ -n "${IS_UNIVERSAL_BUILD}" ]]; then + ./Tools/Scripts/build-webkit --wpe --release "${CMAKE_ARGS}" --touch-events --orientation-events --no-bubblewrap-sandbox "${CMAKE_ARGS[@]}" MiniBrowser + else + WEBKIT_JHBUILD=1 \ + WEBKIT_JHBUILD_MODULESET=minimal \ + WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/WPE \ + ./Tools/Scripts/build-webkit --wpe --release "${CMAKE_ARGS}" --touch-events --orientation-events --no-bubblewrap-sandbox "${CMAKE_ARGS[@]}" MiniBrowser + fi } ensure_linux_deps() { + SUDO="" ; [ $UID -ne 0 ] && SUDO="sudo" + + # These two packages are needed to de-duplicate files on the GTK+WPE bundle and reduce its size. + DEBIAN_FRONTEND=noninteractive ${SUDO} apt-get install -y symlinks rdfind + + if [[ -n "${IS_UNIVERSAL_BUILD}" ]]; then + DEBIAN_FRONTEND=noninteractive ${SUDO} apt-get install -y flatpak + fi + yes | DEBIAN_FRONTEND=noninteractive ./Tools/gtk/install-dependencies yes | DEBIAN_FRONTEND=noninteractive ./Tools/wpe/install-dependencies - yes | DEBIAN_FRONTEND=noninteractive WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/WPE ./Tools/Scripts/update-webkitwpe-libs - yes | DEBIAN_FRONTEND=noninteractive WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/GTK ./Tools/Scripts/update-webkitgtk-libs + if [[ -z "${IS_UNIVERSAL_BUILD}" ]]; then + # In non-universal build install JHBuild deps. + yes | DEBIAN_FRONTEND=noninteractive WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/WPE ./Tools/Scripts/update-webkitwpe-libs + yes | DEBIAN_FRONTEND=noninteractive WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/GTK ./Tools/Scripts/update-webkitgtk-libs + else + yes | ./Tools/Scripts/update-webkitwpe-libs + yes | ./Tools/Scripts/update-webkitgtk-libs + fi } if [[ ! -z "${WK_CHECKOUT_PATH}" ]]; then @@ -55,29 +98,57 @@ if is_mac; then selectXcodeVersionOrDie $(node "$SCRIPT_FOLDER/../get_xcode_version.js" webkit) ./Tools/Scripts/build-webkit --release --touch-events --orientation-events elif is_linux; then - if [[ $# == 0 || (-z "$1") ]]; then - echo - echo BUILDING: GTK and WPE - echo - build_wpe - build_gtk - elif [[ "$1" == "--full" ]]; then - echo - echo BUILDING: GTK and WPE - echo + args=("$@") + IS_FULL="" + BUILD_GTK="" + BUILD_WPE="" + for ((i="${#args[@]}"-1; i >= 0; --i)); do + case ${args[i]} in + --full) IS_FULL="1"; unset args[i]; ;; + --gtk) BUILD_GTK="1"; unset args[i]; ;; + --wpe) BUILD_WPE="1"; unset args[i]; ;; + --universal) IS_UNIVERSAL_BUILD="1"; unset args[i]; ;; + esac + done + + # if neither gtk nor wpe is requested then build both. + if [[ -z "${BUILD_GTK}" && -z "${BUILD_WPE}" ]]; then + BUILD_GTK="1" + BUILD_WPE="1" + fi + + echo "== BUILD CONFIGURATION ==" + if [[ -n "${IS_UNIVERSAL_BUILD}" ]]; then + echo "- universal build: YES" + else + echo "- universal build: NO" + fi + if [[ -n "${IS_FULL}" ]]; then + echo "- install dependencies: YES" + else + echo "- install dependencies: NO" + fi + if [[ -n "${BUILD_GTK}" ]]; then + echo "- build GTK: YES" + else + echo "- build GTK: NO" + fi + if [[ -n "${BUILD_WPE}" ]]; then + echo "- build WPE: YES" + else + echo "- build WPE: NO" + fi + + if [[ -n "${IS_FULL}" ]]; then ensure_linux_deps + fi + + if [[ -n "${BUILD_WPE}" ]]; then build_wpe + fi + + if [[ -n "${BUILD_GTK}" ]]; then build_gtk - elif [[ "$1" == "--gtk" ]]; then - echo - echo BUILDING: GTK - echo - build_gtk - elif [[ "$1" == "--wpe" ]]; then - echo - echo BUILDING: WPE - echo - build_wpe fi elif is_win; then /c/Windows/System32/cmd.exe "/c $(cygpath -w "${SCRIPT_FOLDER}"/buildwin.bat)"