devops: refactor chromium automation scripts (#5486)

Split preparing checkout, archiving and compiling into separate
scripts similarly how we do it with other browsers.
This commit is contained in:
Andrey Lushnikov 2021-02-17 14:43:19 -08:00 коммит произвёл GitHub
Родитель b2227c1bcf
Коммит 8c18b90038
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
4 изменённых файлов: 249 добавлений и 161 удалений

Просмотреть файл

@ -33,15 +33,25 @@ fi
BROWSER_NAME=""
EXTRA_BUILD_ARGS=""
EXTRA_ARCHIVE_ARGS=""
BUILD_FLAVOR="$1"
BUILD_BLOB_NAME=""
EXPECTED_HOST_OS=""
EXPECTED_HOST_OS_VERSION=""
EXPECTED_ARCH="x86_64"
# ===========================
# WINLDD COMPILATION
# ===========================
if [[ "$BUILD_FLAVOR" == "winldd-win64" ]]; then
BROWSER_NAME="winldd"
EXPECTED_HOST_OS="MINGW"
BUILD_BLOB_NAME="winldd-win64.zip"
# ===========================
# FFMPEG COMPILATION
# ===========================
elif [[ "$BUILD_FLAVOR" == "ffmpeg-mac" ]]; then
BROWSER_NAME="ffmpeg"
EXTRA_BUILD_ARGS="--mac"
@ -66,58 +76,81 @@ elif [[ "$BUILD_FLAVOR" == "ffmpeg-cross-compile-win64" ]]; then
EXPECTED_HOST_OS="Ubuntu"
EXPECTED_HOST_OS_VERSION="20.04"
BUILD_BLOB_NAME="ffmpeg-win64.zip"
# ===========================
# CHROMIUM COMPILATION
# ===========================
elif [[ "$BUILD_FLAVOR" == "chromium-win32" ]]; then
BROWSER_NAME="chromium"
EXTRA_BUILD_ARGS="--compile-win32"
EXTRA_ARCHIVE_ARGS="--compile-win32"
EXPECTED_HOST_OS="MINGW"
BUILD_BLOB_NAME="chromium-win32.zip"
elif [[ "$BUILD_FLAVOR" == "chromium-win64" ]]; then
BROWSER_NAME="chromium"
EXTRA_BUILD_ARGS="--compile-win64"
EXTRA_ARCHIVE_ARGS="--compile-win64"
EXPECTED_HOST_OS="MINGW"
BUILD_BLOB_NAME="chromium-win64.zip"
elif [[ "$BUILD_FLAVOR" == "chromium-mac" ]]; then
BROWSER_NAME="chromium"
EXTRA_BUILD_ARGS="--compile-mac"
EXTRA_ARCHIVE_ARGS="--compile-mac"
EXPECTED_HOST_OS="Darwin"
EXPECTED_HOST_OS_VERSION="10.15"
BUILD_BLOB_NAME="chromium-mac.zip"
elif [[ "$BUILD_FLAVOR" == "chromium-mac-arm64" ]]; then
BROWSER_NAME="chromium"
EXTRA_BUILD_ARGS="--compile-mac-arm64"
EXTRA_ARCHIVE_ARGS="--compile-mac-arm64"
EXPECTED_HOST_OS="Darwin"
EXPECTED_HOST_OS_VERSION="10.15"
BUILD_BLOB_NAME="chromium-mac-arm64.zip"
elif [[ "$BUILD_FLAVOR" == "chromium-linux" ]]; then
BROWSER_NAME="chromium"
EXTRA_BUILD_ARGS="--compile-linux"
EXTRA_ARCHIVE_ARGS="--compile-linux"
EXPECTED_HOST_OS="Ubuntu"
EXPECTED_HOST_OS_VERSION="18.04"
BUILD_BLOB_NAME="chromium-linux.zip"
# ===========================
# CHROMIUM MIRRORING
# ===========================
elif [[ "$BUILD_FLAVOR" == "chromium-linux-mirror-to-cdn" ]]; then
BROWSER_NAME="chromium"
EXTRA_BUILD_ARGS="--mirror-linux"
EXTRA_ARCHIVE_ARGS="--mirror-linux"
EXPECTED_HOST_OS="Ubuntu"
EXPECTED_HOST_OS_VERSION="18.04"
BUILD_BLOB_NAME="chromium-linux.zip"
elif [[ "$BUILD_FLAVOR" == "chromium-mac-mirror-to-cdn" ]]; then
BROWSER_NAME="chromium"
EXTRA_BUILD_ARGS="--mirror-mac"
EXTRA_ARCHIVE_ARGS="--mirror-mac"
EXPECTED_HOST_OS="Ubuntu"
EXPECTED_HOST_OS_VERSION="18.04"
BUILD_BLOB_NAME="chromium-mac.zip"
elif [[ "$BUILD_FLAVOR" == "chromium-win32-mirror-to-cdn" ]]; then
BROWSER_NAME="chromium"
EXTRA_BUILD_ARGS="--mirror-win32"
EXTRA_ARCHIVE_ARGS="--mirror-win32"
EXPECTED_HOST_OS="Ubuntu"
EXPECTED_HOST_OS_VERSION="18.04"
BUILD_BLOB_NAME="chromium-win32.zip"
elif [[ "$BUILD_FLAVOR" == "chromium-win64-mirror-to-cdn" ]]; then
BROWSER_NAME="chromium"
EXTRA_BUILD_ARGS="--mirror-win64"
EXTRA_ARCHIVE_ARGS="--mirror-win64"
EXPECTED_HOST_OS="Ubuntu"
EXPECTED_HOST_OS_VERSION="18.04"
BUILD_BLOB_NAME="chromium-win64.zip"
# ===========================
# FIREFOX COMPILATION
# ===========================
elif [[ "$BUILD_FLAVOR" == "firefox-ubuntu-18.04" ]]; then
BROWSER_NAME="firefox"
EXTRA_BUILD_ARGS="--full"
@ -146,6 +179,11 @@ elif [[ "$BUILD_FLAVOR" == "firefox-win64" ]]; then
EXTRA_BUILD_ARGS="--win64"
EXPECTED_HOST_OS="MINGW"
BUILD_BLOB_NAME="firefox-win64.zip"
# ===========================
# WEBKIT COMPILATION
# ===========================
elif [[ "$BUILD_FLAVOR" == "webkit-ubuntu-18.04" ]]; then
BROWSER_NAME="webkit"
EXTRA_BUILD_ARGS="--full"
@ -255,7 +293,7 @@ function generate_and_upload_browser_build {
fi
echo "-- archiving to $ZIP_PATH"
if ! ./$BROWSER_NAME/archive.sh $ZIP_PATH; then
if ! ./$BROWSER_NAME/archive.sh $ZIP_PATH "$EXTRA_ARCHIVE_ARGS"; then
return 23
fi

Просмотреть файл

@ -2,33 +2,170 @@
set -e
set +x
if [[ ("$1" == "-h") || ("$1" == "--help") ]]; then
echo "usage: $(basename $0) [output-absolute-path]"
echo
echo "Generate distributable .zip archive from ./output folder that was previously downloaded."
echo
exit 0
fi
ZIP_PATH=$1
if [[ $ZIP_PATH != /* ]]; then
echo "ERROR: path $ZIP_PATH is not absolute"
exit 1
fi
if [[ $ZIP_PATH != *.zip ]]; then
echo "ERROR: path $ZIP_PATH must have .zip extension"
exit 1
fi
if [[ -f $ZIP_PATH ]]; then
echo "ERROR: path $ZIP_PATH exists; can't do anything."
exit 1
fi
if ! [[ -d $(dirname $ZIP_PATH) ]]; then
echo "ERROR: folder for path $($ZIP_PATH) does not exist."
exit 1
fi
trap "cd $(pwd -P)" EXIT
cd "$(dirname $0)"
SCRIPT_PATH=$(pwd -P)
cp output/build.zip $ZIP_PATH
main() {
if [[ ("$1" == "-h") || ("$1" == "--help") ]]; then
echo "usage: $(basename $0) [output-absolute-path]"
echo
echo "Generate distributable .zip archive from ./output folder that was previously downloaded."
echo
exit 0
fi
ZIP_PATH=$1
if [[ $ZIP_PATH != /* ]]; then
echo "ERROR: path $ZIP_PATH is not absolute"
exit 1
fi
if [[ $ZIP_PATH != *.zip ]]; then
echo "ERROR: path $ZIP_PATH must have .zip extension"
exit 1
fi
if [[ -f $ZIP_PATH ]]; then
echo "ERROR: path $ZIP_PATH exists; can't do anything."
exit 1
fi
if ! [[ -d $(dirname $ZIP_PATH) ]]; then
echo "ERROR: folder for path $($ZIP_PATH) does not exist."
exit 1
fi
BUILD_TYPE=$2
if [[ "${BUILD_TYPE}" == "--compile"* ]]; then
archive_compiled_chromium "${BUILD_TYPE}"
elif [[ "${BUILD_TYPE}" == "--mirror"* ]]; then
archive_mirrored_chromium "${BUILD_TYPE}"
else
echo "ERROR: unknown build type - ${BUILD_TYPE}"
exit 1
fi
cd "${SCRIPT_PATH}"
cp output/build.zip $ZIP_PATH
}
function archive_compiled_chromium() {
CHROMIUM_FOLDER_NAME=""
CHROMIUM_FILES_TO_ARCHIVE=()
if [[ $1 == "--compile-mac"* ]]; then
CHROMIUM_FOLDER_NAME="chrome-mac"
CHROMIUM_FILES_TO_ARCHIVE=("Chromium.app")
elif [[ $1 == "--compile-linux" ]]; then
CHROMIUM_FOLDER_NAME="chrome-linux"
CHROMIUM_FILES_TO_ARCHIVE=(
"chrome"
"chrome_100_percent.pak"
"chrome_200_percent.pak"
"chrome_sandbox"
"chrome-wrapper"
"ClearKeyCdm"
"crashpad_handler"
"icudtl.dat"
"libEGL.so"
"libGLESv2.so"
"locales"
"MEIPreload"
"nacl_helper"
"nacl_helper_bootstrap"
"nacl_helper_nonsfi"
"nacl_irt_x86_64.nexe"
"product_logo_48.png"
"resources"
"resources.pak"
"swiftshader"
"v8_context_snapshot.bin"
"xdg-mime"
"xdg-settings"
)
elif [[ $1 == "--compile-win"* ]]; then
CHROMIUM_FOLDER_NAME="chrome-win"
CHROMIUM_FILES_TO_ARCHIVE=(
"chrome.dll"
"chrome.exe"
"chrome_100_percent.pak"
"chrome_200_percent.pak"
"chrome_elf.dll"
"chrome_proxy.exe"
"chrome_pwa_launcher.exe"
"D3DCompiler_47.dll"
"elevation_service.exe"
"eventlog_provider.dll"
"First Run"
"icudtl.dat"
"libEGL.dll"
"libGLESv2.dll"
"locales"
"MEIPreload"
"mojo_core.dll"
"nacl_irt_x86_64.nexe"
"notification_helper.exe"
"resources.pak"
"swiftshader/libEGL.dll"
"swiftshader/libGLESv2.dll"
"v8_context_snapshot.bin"
)
else
echo "ERROR: unknown command, use --help for details"
exit 1
fi
# Prepare resulting archive.
cd "$SCRIPT_PATH"
rm -rf output
mkdir -p "output/${CHROMIUM_FOLDER_NAME}"
# On Mac, use 'ditto' to copy directories instead of 'cp'.
COPY_COMMAND="cp -R"
if [[ $(uname) == "Darwin" ]]; then
COPY_COMMAND="ditto"
fi
for ((i = 0; i < ${#CHROMIUM_FILES_TO_ARCHIVE[@]}; i++)) do
file="${CHROMIUM_FILES_TO_ARCHIVE[$i]}"
mkdir -p "output/${CHROMIUM_FOLDER_NAME}/$(dirname $file)"
$COPY_COMMAND "${CR_CHECKOUT_PATH}/src/out/Default/${file}" "output/${CHROMIUM_FOLDER_NAME}/${file}"
done
if [[ $1 == "--compile-win"* ]]; then
$COPY_COMMAND "${CR_CHECKOUT_PATH}/src/out/Default/"*.manifest "output/${CHROMIUM_FOLDER_NAME}/"
fi
cd output
zip --symlinks -r build.zip "${CHROMIUM_FOLDER_NAME}"
}
archive_mirrored_chromium() {
cd "${SCRIPT_PATH}/output"
CHROMIUM_FOLDER_NAME=""
CHROMIUM_FILES_TO_REMOVE=()
PLATFORM="$1"
if [[ "${PLATFORM}" == "--mirror-win32" ]]; then
CHROMIUM_FOLDER_NAME="chrome-win"
CHROMIUM_FILES_TO_REMOVE+=("chrome-win/interactive_ui_tests.exe")
elif [[ "${PLATFORM}" == "--mirror-win64" ]]; then
CHROMIUM_FOLDER_NAME="chrome-win"
CHROMIUM_FILES_TO_REMOVE+=("chrome-win/interactive_ui_tests.exe")
elif [[ "${PLATFORM}" == "--mirror-mac" ]]; then
CHROMIUM_FOLDER_NAME="chrome-mac"
elif [[ "${PLATFORM}" == "--mirror-linux" ]]; then
CHROMIUM_FOLDER_NAME="chrome-linux"
else
echo "ERROR: unknown platform to build: $PLATFORM"
exit 1
fi
for file in ${CHROMIUM_FILES_TO_REMOVE[@]}; do
rm -f "${file}"
done
zip --symlinks -r build.zip "${CHROMIUM_FOLDER_NAME}"
}
main "$@"

Просмотреть файл

@ -36,6 +36,11 @@ compile_chromium() {
exit 1
fi
if [[ -z "${CR_CHECKOUT_PATH}/src" ]]; then
echo "ERROR: CR_CHECKOUT_PATH does not have src/ subfolder; is this a chromium checkout?"
exit 1
fi
# install depot_tools if they are not in system
# NOTE: as of Feb 8, 2021, windows requires manual and separate
# installation of depot_tools.
@ -46,114 +51,23 @@ compile_chromium() {
export PATH="${SCRIPT_PATH}/depot_tools:$PATH"
fi
CHROMIUM_FOLDER_NAME=""
CHROMIUM_FILES_TO_ARCHIVE=()
if [[ $1 == "--compile-mac-arm64" ]]; then
if [[ $1 == "--compile-mac"* ]]; then
# As of Jan, 2021 Chromium mac compilation requires Xcode12.2
if [[ ! -d /Applications/Xcode12.2.app ]]; then
echo "ERROR: chromium mac arm64 compilation requires XCode 12.2 to be available"
echo "in the Applications folder!"
echo "ERROR: chromium mac compilation requires /Applications/Xcode12.2.app"
echo "Download one from https://developer.apple.com/download/more/"
exit 1
fi
export DEVELOPER_DIR=/Applications/Xcode12.2.app/Contents/Developer
# As of Jan, 2021 Chromium mac compilation is only possible on Intel macbooks.
# See https://chromium.googlesource.com/chromium/src.git/+/master/docs/mac_arm64.md
if [[ $(uname -m) != "x86_64" ]]; then
if [[ $1 == "--compile-mac-arm64" && $(uname -m) != "x86_64" ]]; then
echo "ERROR: chromium mac arm64 compilation is (ironically) only supported on Intel Macbooks"
exit 1
fi
CHROMIUM_FOLDER_NAME="chrome-mac"
CHROMIUM_FILES_TO_ARCHIVE=("Chromium.app")
elif [[ $1 == "--compile-mac" ]]; then
export DEVELOPER_DIR=/Applications/Xcode12.2.app/Contents/Developer
CHROMIUM_FOLDER_NAME="chrome-mac"
CHROMIUM_FILES_TO_ARCHIVE=("Chromium.app")
elif [[ $1 == "--compile-linux" ]]; then
CHROMIUM_FOLDER_NAME="chrome-linux"
CHROMIUM_FILES_TO_ARCHIVE=(
"chrome"
"chrome_100_percent.pak"
"chrome_200_percent.pak"
"chrome_sandbox"
"chrome-wrapper"
"ClearKeyCdm"
"crashpad_handler"
"icudtl.dat"
"libEGL.so"
"libGLESv2.so"
"locales"
"MEIPreload"
"nacl_helper"
"nacl_helper_bootstrap"
"nacl_helper_nonsfi"
"nacl_irt_x86_64.nexe"
"product_logo_48.png"
"resources"
"resources.pak"
"swiftshader"
"v8_context_snapshot.bin"
"xdg-mime"
"xdg-settings"
)
elif [[ $1 == "--compile-win"* ]]; then
CHROMIUM_FOLDER_NAME="chrome-win"
CHROMIUM_FILES_TO_ARCHIVE=(
"chrome.dll"
"chrome.exe"
"chrome_100_percent.pak"
"chrome_200_percent.pak"
"chrome_elf.dll"
"chrome_proxy.exe"
"chrome_pwa_launcher.exe"
"D3DCompiler_47.dll"
"elevation_service.exe"
"eventlog_provider.dll"
"First Run"
"icudtl.dat"
"libEGL.dll"
"libGLESv2.dll"
"locales"
"MEIPreload"
"mojo_core.dll"
"nacl_irt_x86_64.nexe"
"notification_helper.exe"
"resources.pak"
"swiftshader/libEGL.dll"
"swiftshader/libGLESv2.dll"
"v8_context_snapshot.bin"
)
else
echo "ERROR: unknown command, use --help for details"
exit 1
fi
# Get chromium SHA from the build revision.
# This will get us the last redirect URL from the crrev.com service.
REVISION_URL=$(curl -ILs -o /dev/null -w %{url_effective} "https://crrev.com/${CRREV}")
CRSHA="${REVISION_URL##*/}"
# Update Chromium checkout. One might think that this step should go to `prepare_checkout.sh`
# script, but the `prepare_checkout.sh` is in fact designed to prepare a fork checkout, whereas
# we don't fork Chromium.
#
# This is based on https://chromium.googlesource.com/chromium/src/+/master/docs/linux/build_instructions.md#get-the-code
if [[ ! -d "${CR_CHECKOUT_PATH}/src" ]]; then
rm -rf "${CR_CHECKOUT_PATH}"
mkdir -p "${CR_CHECKOUT_PATH}"
cd "${CR_CHECKOUT_PATH}"
fetch --nohooks chromium
cd src
if [[ $(uname) == "Linux" ]]; then
./build/install-build-deps.sh
fi
gclient runhooks
fi
cd "${CR_CHECKOUT_PATH}/src"
git checkout master
git pull origin master
git checkout "${CRSHA}"
gclient sync
# Prepare build folder.
mkdir -p "./out/Default"
@ -196,30 +110,6 @@ EOF
ninja -j 200 -C out/Default $TARGETS
fi
fi
# Prepare resulting archive.
cd "$SCRIPT_PATH"
rm -rf output
mkdir -p "output/${CHROMIUM_FOLDER_NAME}"
# On Mac, use 'ditto' to copy directories instead of 'cp'.
COPY_COMMAND="cp -R"
if [[ $(uname) == "Darwin" ]]; then
COPY_COMMAND="ditto"
fi
for ((i = 0; i < ${#CHROMIUM_FILES_TO_ARCHIVE[@]}; i++)) do
file="${CHROMIUM_FILES_TO_ARCHIVE[$i]}"
mkdir -p "output/${CHROMIUM_FOLDER_NAME}/$(dirname $file)"
$COPY_COMMAND "${CR_CHECKOUT_PATH}/src/out/Default/${file}" "output/${CHROMIUM_FOLDER_NAME}/${file}"
done
if [[ $1 == "--compile-win"* ]]; then
$COPY_COMMAND "${CR_CHECKOUT_PATH}/src/out/Default/"*.manifest "output/${CHROMIUM_FOLDER_NAME}/"
fi
cd output
zip --symlinks -r build.zip "${CHROMIUM_FOLDER_NAME}"
}
mirror_chromium() {
@ -229,8 +119,6 @@ mirror_chromium() {
cd output
CHROMIUM_URL=""
CHROMIUM_FOLDER_NAME=""
CHROMIUM_FILES_TO_REMOVE=()
PLATFORM="$1"
if [[ "${PLATFORM}" == "--mirror" ]]; then
@ -249,18 +137,12 @@ mirror_chromium() {
if [[ "${PLATFORM}" == "--mirror-win32" ]]; then
CHROMIUM_URL="https://storage.googleapis.com/chromium-browser-snapshots/Win/${CRREV}/chrome-win.zip"
CHROMIUM_FOLDER_NAME="chrome-win"
CHROMIUM_FILES_TO_REMOVE+=("chrome-win/interactive_ui_tests.exe")
elif [[ "${PLATFORM}" == "--mirror-win64" ]]; then
CHROMIUM_URL="https://storage.googleapis.com/chromium-browser-snapshots/Win_x64/${CRREV}/chrome-win.zip"
CHROMIUM_FOLDER_NAME="chrome-win"
CHROMIUM_FILES_TO_REMOVE+=("chrome-win/interactive_ui_tests.exe")
elif [[ "${PLATFORM}" == "--mirror-mac" ]]; then
CHROMIUM_URL="https://storage.googleapis.com/chromium-browser-snapshots/Mac/${CRREV}/chrome-mac.zip"
CHROMIUM_FOLDER_NAME="chrome-mac"
elif [[ "${PLATFORM}" == "--mirror-linux" ]]; then
CHROMIUM_URL="https://storage.googleapis.com/chromium-browser-snapshots/Linux_x64/${CRREV}/chrome-linux.zip"
CHROMIUM_FOLDER_NAME="chrome-linux"
else
echo "ERROR: unknown platform to build: $1"
exit 1
@ -270,11 +152,6 @@ mirror_chromium() {
curl --output chromium-upstream.zip "${CHROMIUM_URL}"
unzip chromium-upstream.zip
for file in ${CHROMIUM_FILES_TO_REMOVE[@]}; do
rm -f "${file}"
done
zip --symlinks -r build.zip "${CHROMIUM_FOLDER_NAME}"
}
main $1

Просмотреть файл

@ -4,6 +4,7 @@ set +x
trap "cd $(pwd -P)" EXIT
cd "$(dirname "$0")"
SCRIPT_PATH=$(pwd -P)
REMOTE_BROWSER_UPSTREAM="browser_upstream"
BUILD_BRANCH="playwright-build"
@ -26,6 +27,41 @@ if [[ $# == 0 ]]; then
exit 1
fi
function prepare_chromium_checkout {
cd "${SCRIPT_PATH}"
if [[ -z "${CR_CHECKOUT_PATH}" ]]; then
echo "ERROR: chromium compilation requires CR_CHECKOUT_PATH to be set to reuse checkout."
echo "NOTE: we expect '\$CR_CHECKOUT_PATH/src' to exist to be a valid chromium checkout."
exit 1
fi
# Get chromium SHA from the build revision.
# This will get us the last redirect URL from the crrev.com service.
CRREV=$(head -1 ./chromium/BUILD_NUMBER)
REVISION_URL=$(curl -ILs -o /dev/null -w %{url_effective} "https://crrev.com/${CRREV}")
CRSHA="${REVISION_URL##*/}"
# Update Chromium checkout.
#
# This is based on https://chromium.googlesource.com/chromium/src/+/master/docs/linux/build_instructions.md#get-the-code
if [[ ! -d "${CR_CHECKOUT_PATH}/src" ]]; then
rm -rf "${CR_CHECKOUT_PATH}"
mkdir -p "${CR_CHECKOUT_PATH}"
cd "${CR_CHECKOUT_PATH}"
fetch --nohooks chromium
cd src
if [[ $(uname) == "Linux" ]]; then
./build/install-build-deps.sh
fi
gclient runhooks
fi
cd "${CR_CHECKOUT_PATH}/src"
git checkout master
git pull origin master
git checkout "${CRSHA}"
gclient sync -D
}
# FRIENDLY_CHECKOUT_PATH is used only for logging.
FRIENDLY_CHECKOUT_PATH="";
CHECKOUT_PATH=""
@ -34,7 +70,7 @@ BUILD_NUMBER=""
WEBKIT_EXTRA_FOLDER_PATH=""
FIREFOX_EXTRA_FOLDER_PATH=""
if [[ ("$1" == "chromium") || ("$1" == "chromium/") || ("$1" == "cr") ]]; then
echo "FYI: chromium checkout is not supported. Use '//browser_patches/chromium/build.sh' instead"
prepare_chromium_checkout
exit 0
elif [[ ("$1" == "ffmpeg") || ("$1" == "ffmpeg/") ]]; then
echo "FYI: ffmpeg checkout is not supported. Use '//browser_patches/ffmpeg/build.sh' instead"