devops: support universal webkit builds (#15496)

2022-07-08 16:27:53 -07:00 · 2022-07-08 16:27:53 -07:00 · 2e331715ff
--- a/browser_patches/checkout_build_archive_upload.sh
+++ b/browser_patches/checkout_build_archive_upload.sh
@ -306,6 +306,13 @@ elif [[ "$BUILD_FLAVOR" == "firefox-beta-win64" ]]; then
 # ===========================
 #    WEBKIT COMPILATION
 # ===========================
+elif [[ "$BUILD_FLAVOR" == "webkit-universal" ]]; then
+  BROWSER_NAME="webkit"
+  EXTRA_BUILD_ARGS="--full --universal"
+  EXTRA_ARCHIVE_ARGS="--universal"
+  EXPECTED_HOST_OS="Ubuntu"
+  EXPECTED_HOST_OS_VERSION="20.04"
+  BUILD_BLOB_NAME="webkit-linux-universal.zip"
 elif [[ "$BUILD_FLAVOR" == "webkit-ubuntu-18.04" ]]; then
  BROWSER_NAME="webkit"
  EXTRA_BUILD_ARGS="--full"
--- a/browser_patches/webkit/EXPECTED_BUILDS
+++ b/browser_patches/webkit/EXPECTED_BUILDS
@ -1,3 +1,4 @@
+webkit-linux-universal.zip
 webkit-ubuntu-18.04.zip
 webkit-ubuntu-20.04.zip
 webkit-ubuntu-20.04-arm64.zip
--- a/browser_patches/webkit/archive.sh
+++ b/browser_patches/webkit/archive.sh
@ -3,7 +3,7 @@ set -e
 set +x

 if [[ ("$1" == "-h") || ("$1" == "--help") ]]; then
-  echo "usage: $(basename "$0") [output-absolute-path]"
+  echo "usage: $(basename "$0") [output-absolute-path] [--universal]"
  echo
  echo "Generate distributable .zip archive from ./checkout folder that was previously built."
  echo
@ -28,6 +28,11 @@ if ! [[ -d $(dirname "$ZIP_PATH") ]]; then
  exit 1
 fi

+IS_UNIVERSAL=""
+if [[ $2 == "--universal" ]]; then
+  IS_UNIVERSAL=1
+fi
+
 main() {
  if [[ ! -z "${WK_CHECKOUT_PATH}" ]]; then
    cd "${WK_CHECKOUT_PATH}"
@ -52,7 +57,7 @@ main() {

 createZipForLinux() {
  # create a TMP directory to copy all necessary files
-  local tmpdir=$(mktemp -d -t webkit-deploy-XXXXXXXXXX)
+  local tmpdir=$(mktemp -d -p "$(pwd)/WebKitBuild" -t webkit-deploy-XXXXXXXXXX)
  mkdir -p "$tmpdir"

  # copy runner
@ -62,15 +67,27 @@ createZipForLinux() {

  # Generate and unpack MiniBrowser bundles for each port
  for port in gtk wpe; do
-    WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/${port^^} Tools/Scripts/generate-bundle \
-        --bundle=MiniBrowser --release \
-        --platform=${port} --destination="${tmpdir}"
-     unzip "${tmpdir}"/MiniBrowser_${port}_release.zip -d "${tmpdir}"/minibrowser-${port}
-     rm -f "${tmpdir}"/MiniBrowser_${port}_release.zip
+    if [[ -n "${IS_UNIVERSAL}" ]]; then
+      Tools/Scripts/generate-bundle \
+          --syslibs=bundle-all \
+          --bundle=MiniBrowser --release \
+          --platform=${port} --destination="${tmpdir}"
+    else
+      WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/${port^^} \
+      Tools/Scripts/generate-bundle \
+          --bundle=MiniBrowser --release \
+          --platform=${port} --destination="${tmpdir}"
+    fi
+
+    unzip "${tmpdir}"/MiniBrowser_${port}_release.zip -d "${tmpdir}"/minibrowser-${port}
+    rm -f "${tmpdir}"/MiniBrowser_${port}_release.zip
  done

-  # tar resulting directory and cleanup TMP.
  cd "$tmpdir"
+  # de-duplicate common files: convert to relative symlinks identical files (same hash)
+  rdfind -deterministic true -makesymlinks true -makehardlinks false -makeresultsfile false .
+  symlinks -rc .
+  # zip resulting directory and cleanup TMP.
  zip --symlinks -r "$ZIP_PATH" ./
  cd -
  rm -rf "$tmpdir"
--- a/browser_patches/webkit/build.sh
+++ b/browser_patches/webkit/build.sh
@ -7,9 +7,18 @@ cd "$(dirname "$0")"
 SCRIPT_FOLDER="$(pwd -P)"
 source "${SCRIPT_FOLDER}/../utils.sh"

+# On Linux, Universal build uses Flatpak rather then JHBuild
+# and packs into a universal binary that can run on any linux
+# distribution.
+IS_UNIVERSAL_BUILD=""
+
 build_gtk() {
-  if ! [[ -d ./WebKitBuild/GTK/DependenciesGTK ]]; then
-    yes | WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/GTK DEBIAN_FRONTEND=noninteractive ./Tools/Scripts/update-webkitgtk-libs
+  if [[ -z "${IS_UNIVERSAL_BUILD}" && ! -d "./WebKitBuild/GTK/DependenciesGTK" ]]; then
+    yes | WEBKIT_JHBUILD=1 \
+          WEBKIT_JHBUILD_MODULESET=minimal \
+          WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/GTK \
+          DEBIAN_FRONTEND=noninteractive \
+          ./Tools/Scripts/update-webkitgtk-libs
  fi
  local CMAKE_ARGS=(
    --cmakeargs=-DENABLE_INTROSPECTION=OFF
@ -18,12 +27,23 @@ build_gtk() {
  if [[ -n "${EXPORT_COMPILE_COMMANDS}" ]]; then
    CMAKE_ARGS+=("--cmakeargs=-DCMAKE_EXPORT_COMPILE_COMMANDS=1")
  fi
-  WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/GTK ./Tools/Scripts/build-webkit --gtk --release "${CMAKE_ARGS}" --touch-events --orientation-events --no-bubblewrap-sandbox "${CMAKE_ARGS[@]}" MiniBrowser
+  if [[ -n "${IS_UNIVERSAL_BUILD}" ]]; then
+    ./Tools/Scripts/build-webkit --gtk --release "${CMAKE_ARGS}" --touch-events --orientation-events --no-bubblewrap-sandbox "${CMAKE_ARGS[@]}" MiniBrowser
+  else
+    WEBKIT_JHBUILD=1 \
+    WEBKIT_JHBUILD_MODULESET=minimal \
+    WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/GTK \
+    ./Tools/Scripts/build-webkit --gtk --release "${CMAKE_ARGS}" --touch-events --orientation-events --no-bubblewrap-sandbox "${CMAKE_ARGS[@]}" MiniBrowser
+  fi
 }

 build_wpe() {
-  if ! [[ -d ./WebKitBuild/WPE/DependenciesWPE ]]; then
-    yes | WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/WPE DEBIAN_FRONTEND=noninteractive ./Tools/Scripts/update-webkitwpe-libs
+  if [[ -z "${IS_UNIVERSAL_BUILD}" && ! -d "./WebKitBuild/WPE/DependenciesWPE" ]]; then
+    yes | WEBKIT_JHBUILD=1 \
+          WEBKIT_JHBUILD_MODULESET=minimal \
+          WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/WPE \
+          DEBIAN_FRONTEND=noninteractive \
+          ./Tools/Scripts/update-webkitwpe-libs
  fi
  local CMAKE_ARGS=(
    --cmakeargs=-DENABLE_COG=OFF
@ -34,14 +54,37 @@ build_wpe() {
  if [[ -n "${EXPORT_COMPILE_COMMANDS}" ]]; then
    CMAKE_ARGS+=("--cmakeargs=-DCMAKE_EXPORT_COMPILE_COMMANDS=1")
  fi
-  WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/WPE ./Tools/Scripts/build-webkit --wpe --release "${CMAKE_ARGS}" --touch-events --orientation-events --no-bubblewrap-sandbox "${CMAKE_ARGS[@]}" MiniBrowser
+
+  if [[ -n "${IS_UNIVERSAL_BUILD}" ]]; then
+    ./Tools/Scripts/build-webkit --wpe --release "${CMAKE_ARGS}" --touch-events --orientation-events --no-bubblewrap-sandbox "${CMAKE_ARGS[@]}" MiniBrowser
+  else
+    WEBKIT_JHBUILD=1 \
+    WEBKIT_JHBUILD_MODULESET=minimal \
+    WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/WPE \
+    ./Tools/Scripts/build-webkit --wpe --release "${CMAKE_ARGS}" --touch-events --orientation-events --no-bubblewrap-sandbox "${CMAKE_ARGS[@]}" MiniBrowser
+  fi
 }

 ensure_linux_deps() {
+  SUDO="" ; [ $UID -ne 0 ] && SUDO="sudo"
+
+  # These two packages are needed to de-duplicate files on the GTK+WPE bundle and reduce its size.
+  DEBIAN_FRONTEND=noninteractive ${SUDO} apt-get install -y symlinks rdfind
+
+  if [[ -n "${IS_UNIVERSAL_BUILD}" ]]; then
+    DEBIAN_FRONTEND=noninteractive ${SUDO} apt-get install -y flatpak
+  fi
+
  yes | DEBIAN_FRONTEND=noninteractive ./Tools/gtk/install-dependencies
  yes | DEBIAN_FRONTEND=noninteractive ./Tools/wpe/install-dependencies
-  yes | DEBIAN_FRONTEND=noninteractive WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/WPE ./Tools/Scripts/update-webkitwpe-libs
-  yes | DEBIAN_FRONTEND=noninteractive WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/GTK ./Tools/Scripts/update-webkitgtk-libs
+  if [[ -z "${IS_UNIVERSAL_BUILD}" ]]; then
+    # In non-universal build install JHBuild deps.
+    yes | DEBIAN_FRONTEND=noninteractive WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/WPE ./Tools/Scripts/update-webkitwpe-libs
+    yes | DEBIAN_FRONTEND=noninteractive WEBKIT_JHBUILD=1 WEBKIT_JHBUILD_MODULESET=minimal WEBKIT_OUTPUTDIR=$(pwd)/WebKitBuild/GTK ./Tools/Scripts/update-webkitgtk-libs
+  else
+    yes | ./Tools/Scripts/update-webkitwpe-libs
+    yes | ./Tools/Scripts/update-webkitgtk-libs
+  fi
 }

 if [[ ! -z "${WK_CHECKOUT_PATH}" ]]; then
@ -55,29 +98,57 @@ if is_mac; then
  selectXcodeVersionOrDie $(node "$SCRIPT_FOLDER/../get_xcode_version.js" webkit)
  ./Tools/Scripts/build-webkit --release --touch-events --orientation-events
 elif is_linux; then
-  if [[ $# == 0 || (-z "$1") ]]; then
-    echo
-    echo BUILDING: GTK and WPE
-    echo
-    build_wpe
-    build_gtk
-  elif [[ "$1" == "--full" ]]; then
-    echo
-    echo BUILDING: GTK and WPE
-    echo
+  args=("$@")
+  IS_FULL=""
+  BUILD_GTK=""
+  BUILD_WPE=""
+  for ((i="${#args[@]}"-1; i >= 0; --i)); do
+      case ${args[i]} in
+          --full) IS_FULL="1"; unset args[i]; ;;
+          --gtk) BUILD_GTK="1"; unset args[i]; ;;
+          --wpe) BUILD_WPE="1"; unset args[i]; ;;
+          --universal) IS_UNIVERSAL_BUILD="1"; unset args[i]; ;;
+      esac
+  done
+
+  # if neither gtk nor wpe is requested then build both.
+  if [[ -z "${BUILD_GTK}" && -z "${BUILD_WPE}" ]]; then
+    BUILD_GTK="1"
+    BUILD_WPE="1"
+  fi
+
+  echo "== BUILD CONFIGURATION =="
+  if [[ -n "${IS_UNIVERSAL_BUILD}" ]]; then
+    echo "- universal build: YES"
+  else
+    echo "- universal build: NO"
+  fi
+  if [[ -n "${IS_FULL}" ]]; then
+    echo "- install dependencies: YES"
+  else
+    echo "- install dependencies: NO"
+  fi
+  if [[ -n "${BUILD_GTK}" ]]; then
+    echo "- build GTK: YES"
+  else
+    echo "- build GTK: NO"
+  fi
+  if [[ -n "${BUILD_WPE}" ]]; then
+    echo "- build WPE: YES"
+  else
+    echo "- build WPE: NO"
+  fi
+
+  if [[ -n "${IS_FULL}" ]]; then
    ensure_linux_deps
+  fi
+
+  if [[ -n "${BUILD_WPE}" ]]; then
    build_wpe
+  fi
+
+  if [[ -n "${BUILD_GTK}" ]]; then
    build_gtk
-  elif [[ "$1" == "--gtk" ]]; then
-    echo
-    echo BUILDING: GTK
-    echo
-    build_gtk
-  elif [[ "$1" == "--wpe" ]]; then
-    echo
-    echo BUILDING: WPE
-    echo
-    build_wpe
  fi
 elif is_win; then
  /c/Windows/System32/cmd.exe "/c $(cygpath -w "${SCRIPT_FOLDER}"/buildwin.bat)"