devops: start uploading build logs to bots (#1870)
Logs will be saved next to the build archive, with the same name and double extension `.log.zip`.
This commit is contained in:
Andrey Lushnikov
GitHub
Родитель
ef7815e95b
Коммит
ea95a9122d
|
|
@ -30,46 +30,57 @@ BROWSER_NAME=""
|
|||
EXTRA_BUILD_ARGS=""
|
||||
EXTRA_ARCHIVE_ARGS=""
|
||||
BUILD_FLAVOR="$1"
|
||||
BUILD_BLOB_NAME=""
|
||||
EXPECTED_HOST_OS=""
|
||||
EXPECTED_HOST_OS_VERSION=""
|
||||
if [[ "$BUILD_FLAVOR" == "firefox-linux" ]]; then
|
||||
BROWSER_NAME="firefox"
|
||||
EXPECTED_HOST_OS="Linux"
|
||||
BUILD_BLOB_NAME="firefox-linux.zip"
|
||||
elif [[ "$BUILD_FLAVOR" == "firefox-mac" ]]; then
|
||||
BROWSER_NAME="firefox"
|
||||
EXPECTED_HOST_OS="Darwin"
|
||||
EXPECTED_HOST_OS_VERSION="10.14"
|
||||
BUILD_BLOB_NAME="firefox-mac.zip"
|
||||
elif [[ "$BUILD_FLAVOR" == "firefox-win32" ]]; then
|
||||
BROWSER_NAME="firefox"
|
||||
EXPECTED_HOST_OS="MINGW"
|
||||
BUILD_BLOB_NAME="firefox-win32.zip"
|
||||
elif [[ "$BUILD_FLAVOR" == "firefox-win64" ]]; then
|
||||
BROWSER_NAME="firefox"
|
||||
EXTRA_BUILD_ARGS="--win64"
|
||||
EXPECTED_HOST_OS="MINGW"
|
||||
BUILD_BLOB_NAME="firefox-win64.zip"
|
||||
elif [[ "$BUILD_FLAVOR" == "webkit-gtk" ]]; then
|
||||
BROWSER_NAME="webkit"
|
||||
EXTRA_BUILD_ARGS="--gtk"
|
||||
EXTRA_ARCHIVE_ARGS="--gtk"
|
||||
EXPECTED_HOST_OS="Linux"
|
||||
BUILD_BLOB_NAME="minibrowser-gtk.zip"
|
||||
elif [[ "$BUILD_FLAVOR" == "webkit-wpe" ]]; then
|
||||
BROWSER_NAME="webkit"
|
||||
EXTRA_BUILD_ARGS="--wpe"
|
||||
EXTRA_ARCHIVE_ARGS="--wpe"
|
||||
EXPECTED_HOST_OS="Linux"
|
||||
BUILD_BLOB_NAME="minibrowser-wpe.zip"
|
||||
elif [[ "$BUILD_FLAVOR" == "webkit-gtk-wpe" ]]; then
|
||||
BROWSER_NAME="webkit"
|
||||
EXPECTED_HOST_OS="Linux"
|
||||
BUILD_BLOB_NAME="minibrowser-gtk-wpe.zip"
|
||||
elif [[ "$BUILD_FLAVOR" == "webkit-win64" ]]; then
|
||||
BROWSER_NAME="webkit"
|
||||
EXPECTED_HOST_OS="MINGW"
|
||||
BUILD_BLOB_NAME="minibrowser-win64.zip"
|
||||
elif [[ "$BUILD_FLAVOR" == "webkit-mac-10.14" ]]; then
|
||||
BROWSER_NAME="webkit"
|
||||
EXPECTED_HOST_OS="Darwin"
|
||||
EXPECTED_HOST_OS_VERSION="10.14"
|
||||
BUILD_BLOB_NAME="minibrowser-mac-10.14.zip"
|
||||
elif [[ "$BUILD_FLAVOR" == "webkit-mac-10.15" ]]; then
|
||||
BROWSER_NAME="webkit"
|
||||
EXPECTED_HOST_OS="Darwin"
|
||||
EXPECTED_HOST_OS_VERSION="10.15"
|
||||
BUILD_BLOB_NAME="minibrowser-mac-10.15.zip"
|
||||
else
|
||||
echo ERROR: unknown build flavor - "$BUILD_FLAVOR"
|
||||
exit 1
|
||||
|
|
@ -91,79 +102,106 @@ fi
|
|||
|
||||
if [[ $(uname) == MINGW* ]]; then
|
||||
ZIP_PATH="$PWD/archive-$BROWSER_NAME.zip"
|
||||
LOG_PATH="$PWD/log-$BROWSER_NAME.zip"
|
||||
else
|
||||
ZIP_PATH="/tmp/archive-$BROWSER_NAME.zip"
|
||||
LOG_PATH="/tmp/log-$BROWSER_NAME.zip"
|
||||
fi
|
||||
|
||||
if [[ -f $ZIP_PATH ]]; then
|
||||
echo "Archive $ZIP_PATH already exists - remove and re-run the script."
|
||||
exit 1
|
||||
fi
|
||||
trap "rm -rf ${ZIP_PATH}; cd $(pwd -P);" INT TERM EXIT
|
||||
trap "rm -rf ${ZIP_PATH}; rm -rf ${LOG_PATH}; cd $(pwd -P);" INT TERM EXIT
|
||||
cd "$(dirname "$0")"
|
||||
BUILD_NUMBER=$(cat ./$BROWSER_NAME/BUILD_NUMBER)
|
||||
BUILD_BLOB_PATH="${BROWSER_NAME}/${BUILD_NUMBER}/${BUILD_BLOB_NAME}"
|
||||
LOG_BLOB_PATH="${BROWSER_NAME}/${BUILD_NUMBER}/${BUILD_BLOB_NAME%.zip}.log.zip"
|
||||
|
||||
# pull from upstream and check if a new build has to be uploaded.
|
||||
if ! [[ ($2 == '-f') || ($2 == '--force') ]]; then
|
||||
if ./upload.sh $BUILD_FLAVOR --check; then
|
||||
if ./upload.sh "${BUILD_BLOB_PATH}" --check; then
|
||||
echo "Build is already uploaded - no changes."
|
||||
exit 0
|
||||
else
|
||||
echo "Build is missing - rebuilding"
|
||||
elif ./upload.sh "${LOG_BLOB_PATH}" --check; then
|
||||
echo "This build has already been attempted - skip building."
|
||||
exit 0
|
||||
fi
|
||||
echo "Build is missing and has not been attempted - rebuilding"
|
||||
else
|
||||
echo "Force-rebuilding the build."
|
||||
fi
|
||||
|
||||
source ./buildbots/send_telegram_message.sh
|
||||
BUILD_ALIAS="$BUILD_FLAVOR r$BUILD_NUMBER"
|
||||
|
||||
send_telegram_message "$BUILD_ALIAS -- started"
|
||||
|
||||
if [[ "$BUILD_FLAVOR" == "webkit-gtk-wpe" ]]; then
|
||||
echo "-- combining binaries together"
|
||||
if ! ./webkit/download_gtk_and_wpe_and_zip_together.sh $ZIP_PATH; then
|
||||
send_telegram_message "$BUILD_ALIAS -- ./download_gtk_and_wpe_and_zip_together.sh failed! ❌"
|
||||
exit 1
|
||||
FAILED_STEP=""
|
||||
function generate_and_upload_browser_build {
|
||||
# webkit-gtk-wpe is a special build doesn't need to be built.
|
||||
if [[ "$BUILD_FLAVOR" == "webkit-gtk-wpe" ]]; then
|
||||
echo "-- combining binaries together"
|
||||
if ! ./webkit/download_gtk_and_wpe_and_zip_together.sh $ZIP_PATH; then
|
||||
FAILED_STEP="./download_gtk_and_wpe_and_zip_together.sh"
|
||||
return 1
|
||||
fi
|
||||
echo "-- uploading"
|
||||
if ! ./upload.sh $BUILD_BLOB_PATH $ZIP_PATH; then
|
||||
FAILED_STEP="./upload.sh "
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
else
|
||||
|
||||
# Other browser flavors follow typical build flow.
|
||||
echo "-- preparing checkout"
|
||||
if ! ./prepare_checkout.sh $BROWSER_NAME; then
|
||||
send_telegram_message "$BUILD_ALIAS -- ./prepare_checkout.sh failed! ❌"
|
||||
exit 1
|
||||
FAILED_STEP="./prepare_checkout.sh"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "-- cleaning"
|
||||
if ! ./$BROWSER_NAME/clean.sh; then
|
||||
send_telegram_message "$BUILD_ALIAS -- ./clean.sh failed! ❌"
|
||||
exit 1
|
||||
FAILED_STEP="./clean.sh"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "-- building"
|
||||
if ! ./$BROWSER_NAME/build.sh "$EXTRA_BUILD_ARGS"; then
|
||||
send_telegram_message "$BUILD_ALIAS -- ./build.sh failed! ❌"
|
||||
exit 1
|
||||
FAILED_STEP="./build.sh "
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "-- archiving to $ZIP_PATH"
|
||||
if ! ./$BROWSER_NAME/archive.sh $ZIP_PATH "$EXTRA_ARCHIVE_ARGS"; then
|
||||
send_telegram_message "$BUILD_ALIAS -- ./archive.sh failed! ❌"
|
||||
exit 1
|
||||
FAILED_STEP="./archive.sh "
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "-- uploading"
|
||||
if ! ./upload.sh $BUILD_BLOB_PATH $ZIP_PATH; then
|
||||
FAILED_STEP="./upload.sh "
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
source ./buildbots/send_telegram_message.sh
|
||||
BUILD_ALIAS="$BUILD_FLAVOR r$BUILD_NUMBER"
|
||||
send_telegram_message "$BUILD_ALIAS -- started"
|
||||
|
||||
if generate_and_upload_browser_build 2>&1 | ./sanitize_env.js | zip > $LOG_PATH; then
|
||||
# Report successful build. Note: we don't know how to get zip size on MINGW.
|
||||
if [[ $(uname) == MINGW* ]]; then
|
||||
send_telegram_message "$BUILD_ALIAS -- uploaded"
|
||||
else
|
||||
UPLOAD_SIZE=$(du -h "$ZIP_PATH" | awk '{print $1}')
|
||||
send_telegram_message "$BUILD_ALIAS -- $UPLOAD_SIZE uploaded"
|
||||
fi
|
||||
# Check if we uploaded the last build.
|
||||
if ./tools/check_cdn.sh $BROWSER_NAME --has-all-builds; then
|
||||
LAST_COMMIT_MESSAGE=$(git log --format=%s -n 1 HEAD -- ./$BROWSER_NAME/BUILD_NUMBER)
|
||||
send_telegram_message "<b>$BROWSER_NAME r${BUILD_NUMBER} COMPLETE! ✅</b> $LAST_COMMIT_MESSAGE"
|
||||
fi
|
||||
else
|
||||
# Upload logs only in case of failure and report failure.
|
||||
./upload.sh ${LOG_BLOB_PATH} ${LOG_PATH} || true
|
||||
send_telegram_message "$BUILD_ALIAS -- ${FAILED_STEP} failed! ❌ <a href='https://playwright.azureedge.net/builds/${LOG_BLOB_PATH}'>see logs</a>"
|
||||
fi
|
||||
|
||||
echo "-- uploading"
|
||||
if ! ./upload.sh $BUILD_FLAVOR $ZIP_PATH; then
|
||||
send_telegram_message "$BUILD_ALIAS -- ./upload.sh failed! ❌"
|
||||
exit 1
|
||||
fi
|
||||
UPLOAD_SIZE=$(du -h "$ZIP_PATH" | awk '{print $1}')
|
||||
send_telegram_message "$BUILD_ALIAS -- $UPLOAD_SIZE uploaded"
|
||||
|
||||
if ./tools/check_cdn.sh $BROWSER_NAME --has-all-builds; then
|
||||
LAST_COMMIT_MESSAGE=$(git log --format=%s -n 1 HEAD -- ./$BROWSER_NAME/BUILD_NUMBER)
|
||||
send_telegram_message "<b>$BROWSER_NAME r${BUILD_NUMBER} COMPLETE! ✅</b> $LAST_COMMIT_MESSAGE"
|
||||
fi
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,66 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const readline = require('readline');
|
||||
|
||||
// These env variable values should be removed from logs no matter what.
|
||||
const BLOCKLIST_ENV_KEYS = new Set([
|
||||
'AZ_ACCOUNT_NAME',
|
||||
'AZ_ACCOUNT_KEY',
|
||||
'TELEGRAM_BOT_KEY',
|
||||
]);
|
||||
|
||||
// These env variable values can stay in logs - they are harmless.
|
||||
const ALLOWLIST_ENV_KEYS = new Set([
|
||||
'SHELL',
|
||||
'TERM',
|
||||
'USER',
|
||||
'PWD',
|
||||
'EDITOR',
|
||||
'LANG',
|
||||
'HOME',
|
||||
'LOGNAME',
|
||||
'COLORTERM',
|
||||
'TMPDIR',
|
||||
]);
|
||||
|
||||
const sanitizeEnv = Object.entries(process.env).filter(([key, value]) => {
|
||||
if (BLOCKLIST_ENV_KEYS.has(key))
|
||||
return true;
|
||||
if (ALLOWLIST_ENV_KEYS.has(key))
|
||||
return false;
|
||||
// Sanitize all env variables that have `KEY` or `ACCOUNT` as a name.
|
||||
if (key.toUpperCase().includes('KEY') || key.toUpperCase().includes('ACCOUNT'))
|
||||
return true;
|
||||
// We shouldn't try sanitizing env values that are too short.
|
||||
if (value.trim().length < 7)
|
||||
return false;
|
||||
return true;
|
||||
});
|
||||
|
||||
const rl = readline.createInterface({
|
||||
input: process.stdin,
|
||||
crlfDelay: Infinity,
|
||||
});
|
||||
|
||||
rl.on('line', line => {
|
||||
for (const [key, value] of sanitizeEnv)
|
||||
line = line.split(value).join(`<${key}>`);
|
||||
console.log(line);
|
||||
});
|
||||
|
||||
|
|
@ -6,12 +6,12 @@ trap "cd $(pwd -P)" EXIT
|
|||
cd "$(dirname "$0")"
|
||||
|
||||
if [[ ($1 == '--help') || ($1 == '-h') ]]; then
|
||||
echo "usage: $(basename $0) [firefox-linux|firefox-win32|firefox-win64|webkit-gtk|webkit-wpe|webkit-gtk-wpe|webkit-win64|webkit-mac-10.14|webkit-mac-10.15] [--check] [zip-path]"
|
||||
echo "usage: $(basename $0) [BLOB-PATH] [--check|ZIP-PATH]"
|
||||
echo
|
||||
echo "Upload .zip as a browser build."
|
||||
echo "Upload ZIP-PATH to BLOB-PATH in `builds` container."
|
||||
echo
|
||||
echo "--check pass |--check| as a second parameter instead of a zip-path to check for"
|
||||
echo " the build existing in the CDN"
|
||||
echo " existance of BLOB-PATH"
|
||||
echo
|
||||
echo "NOTE: \$AZ_ACCOUNT_KEY (azure account name) and \$AZ_ACCOUNT_NAME (azure account name)"
|
||||
echo "env variables are required to upload builds to CDN."
|
||||
|
|
@ -25,54 +25,16 @@ if [[ (-z $AZ_ACCOUNT_KEY) || (-z $AZ_ACCOUNT_NAME) ]]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
if [[ $# < 1 ]]; then
|
||||
echo "missing browser: 'firefox' or 'webkit'"
|
||||
if [[ $# < 2 ]]; then
|
||||
echo "not enought arguments!"
|
||||
echo "try '$(basename $0) --help' for more information"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BUILD_FLAVOR="$1"
|
||||
BROWSER_NAME=""
|
||||
BLOB_NAME=""
|
||||
if [[ "$BUILD_FLAVOR" == "firefox-linux" ]]; then
|
||||
BROWSER_NAME="firefox"
|
||||
BLOB_NAME="firefox-linux.zip"
|
||||
elif [[ "$BUILD_FLAVOR" == "firefox-mac" ]]; then
|
||||
BROWSER_NAME="firefox"
|
||||
BLOB_NAME="firefox-mac.zip"
|
||||
elif [[ "$BUILD_FLAVOR" == "firefox-win32" ]]; then
|
||||
BROWSER_NAME="firefox"
|
||||
BLOB_NAME="firefox-win32.zip"
|
||||
elif [[ "$BUILD_FLAVOR" == "firefox-win64" ]]; then
|
||||
BROWSER_NAME="firefox"
|
||||
BLOB_NAME="firefox-win64.zip"
|
||||
elif [[ "$BUILD_FLAVOR" == "webkit-gtk" ]]; then
|
||||
BROWSER_NAME="webkit"
|
||||
BLOB_NAME="minibrowser-gtk.zip"
|
||||
elif [[ "$BUILD_FLAVOR" == "webkit-wpe" ]]; then
|
||||
BROWSER_NAME="webkit"
|
||||
BLOB_NAME="minibrowser-wpe.zip"
|
||||
elif [[ "$BUILD_FLAVOR" == "webkit-gtk-wpe" ]]; then
|
||||
BROWSER_NAME="webkit"
|
||||
BLOB_NAME="minibrowser-gtk-wpe.zip"
|
||||
elif [[ "$BUILD_FLAVOR" == "webkit-win64" ]]; then
|
||||
BROWSER_NAME="webkit"
|
||||
BLOB_NAME="minibrowser-win64.zip"
|
||||
elif [[ "$BUILD_FLAVOR" == "webkit-mac-10.14" ]]; then
|
||||
BROWSER_NAME="webkit"
|
||||
BLOB_NAME="minibrowser-mac-10.14.zip"
|
||||
elif [[ "$BUILD_FLAVOR" == "webkit-mac-10.15" ]]; then
|
||||
BROWSER_NAME="webkit"
|
||||
BLOB_NAME="minibrowser-mac-10.15.zip"
|
||||
else
|
||||
echo ERROR: unknown build flavor - "$BUILD_FLAVOR"
|
||||
exit 1
|
||||
fi
|
||||
BLOB_PATH="$1"
|
||||
ZIP_PATH="$2"
|
||||
|
||||
BUILD_NUMBER=$(cat ./$BROWSER_NAME/BUILD_NUMBER)
|
||||
BLOB_PATH="$BROWSER_NAME/$BUILD_NUMBER/$BLOB_NAME"
|
||||
|
||||
if [[ ("$2" == '--check') || ("$3" == '--check') ]]; then
|
||||
if [[ ("$2" == '--check') ]]; then
|
||||
EXISTS=$(az storage blob exists -c builds --account-key $AZ_ACCOUNT_KEY --account-name $AZ_ACCOUNT_NAME -n "$BLOB_PATH" --query "exists")
|
||||
if [[ $EXISTS == "true" ]]; then
|
||||
exit 0
|
||||
|
|
@ -81,14 +43,6 @@ if [[ ("$2" == '--check') || ("$3" == '--check') ]]; then
|
|||
fi
|
||||
fi
|
||||
|
||||
if [[ $# < 2 ]]; then
|
||||
echo "missing path to zip archive to upload"
|
||||
echo "try '$(basename $0) --help' for more information"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ZIP_PATH="$2"
|
||||
|
||||
if ! [[ -f $ZIP_PATH ]]; then
|
||||
echo "ERROR: $ZIP_PATH does not exist"
|
||||
exit 1
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче