зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1618390 - Use mozlog in condprof r=whimboo
This patch refactors the condprof logging so it uses mozlog Differential Revision: https://phabricator.services.mozilla.com/D64720 --HG-- extra : moz-landing-system : lando
This commit is contained in:
Tarek Ziadé
Родитель
5277c4f9a1
Коммит
71814e3255
|
|
@ -523,6 +523,12 @@ ARCHIVE_FILES = {
|
|||
'pattern': '**',
|
||||
'dest': 'condprofile/mozdevice',
|
||||
},
|
||||
{
|
||||
'source': buildconfig.topsrcdir,
|
||||
'base': 'testing/mozbase/mozlog',
|
||||
'pattern': '**',
|
||||
'dest': 'condprofile/mozlog',
|
||||
},
|
||||
{
|
||||
'source': buildconfig.topsrcdir,
|
||||
'base': 'third_party/python/virtualenv',
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ import attr
|
|||
from arsenic.services import Geckodriver, free_port, subprocess_based_service
|
||||
from mozdevice import ADBDevice, ADBError
|
||||
|
||||
from condprof.util import write_yml_file, LOG, DEFAULT_PREFS, ERROR, BaseEnv
|
||||
from condprof.util import write_yml_file, logger, DEFAULT_PREFS, BaseEnv
|
||||
|
||||
|
||||
# XXX most of this code should migrate into mozdevice - see Bug 1574849
|
||||
|
|
@ -42,7 +42,7 @@ class AndroidDevice:
|
|||
self.log_file = log_file
|
||||
if self.log_file is None:
|
||||
return
|
||||
LOG("Setting ADB log file to %s" % self.log_file)
|
||||
logger.info("Setting ADB log file to %s" % self.log_file)
|
||||
adb_logger = logging.getLogger("adb")
|
||||
adb_logger.setLevel(logging.DEBUG)
|
||||
self._adb_fh = logging.FileHandler(self.log_file)
|
||||
|
|
@ -78,7 +78,7 @@ class AndroidDevice:
|
|||
try:
|
||||
self.device = ADBDevice(verbose=self.verbose, logger_name="adb")
|
||||
except Exception:
|
||||
ERROR("Cannot initialize device")
|
||||
logger.error("Cannot initialize device")
|
||||
raise
|
||||
device = self.device
|
||||
self.profile = profile
|
||||
|
|
@ -88,22 +88,23 @@ class AndroidDevice:
|
|||
raise Exception("%s is not installed" % self.app_name)
|
||||
|
||||
# debug flag
|
||||
LOG("Setting %s as the debug app on the phone" % self.app_name)
|
||||
logger.info("Setting %s as the debug app on the phone" % self.app_name)
|
||||
device.shell(
|
||||
"am set-debug-app --persistent %s" % self.app_name, stdout_callback=LOG
|
||||
"am set-debug-app --persistent %s" % self.app_name,
|
||||
stdout_callback=logger.info,
|
||||
)
|
||||
|
||||
# creating the profile on the device
|
||||
LOG("Creating the profile on the device")
|
||||
logger.info("Creating the profile on the device")
|
||||
remote_test_root = posixpath.join(device.test_root, "condprof")
|
||||
remote_profile = posixpath.join(remote_test_root, "profile")
|
||||
LOG("The profile on the phone will be at %s" % remote_profile)
|
||||
logger.info("The profile on the phone will be at %s" % remote_profile)
|
||||
device.rm(remote_test_root, force=True, recursive=True)
|
||||
device.mkdir(remote_test_root)
|
||||
device.chmod(remote_test_root, recursive=True, root=True)
|
||||
|
||||
device.rm(remote_profile, force=True, recursive=True)
|
||||
LOG("Pushing %s on the phone" % self.profile)
|
||||
logger.info("Pushing %s on the phone" % self.profile)
|
||||
device.push(profile, remote_profile)
|
||||
device.chmod(remote_profile, recursive=True, root=True)
|
||||
self.profile = profile
|
||||
|
|
@ -128,7 +129,7 @@ class AndroidDevice:
|
|||
device.push(yml_on_host, yml_on_device)
|
||||
device.chmod(yml_on_device, recursive=True, root=True)
|
||||
except Exception:
|
||||
LOG("could not create the yaml file on device. Permission issue?")
|
||||
logger.info("could not create the yaml file on device. Permission issue?")
|
||||
raise
|
||||
|
||||
# command line 'extra' args not used with geckoview apps; instead we use
|
||||
|
|
@ -164,7 +165,7 @@ class AndroidDevice:
|
|||
if not device.process_exist(self.app_name):
|
||||
raise Exception("Could not start %s" % self.app_name)
|
||||
|
||||
LOG("Creating socket forwarding on port %d" % self.marionette_port)
|
||||
logger.info("Creating socket forwarding on port %d" % self.marionette_port)
|
||||
device.forward(
|
||||
local="tcp:%d" % self.marionette_port,
|
||||
remote="tcp:%d" % self.marionette_port,
|
||||
|
|
@ -172,19 +173,19 @@ class AndroidDevice:
|
|||
|
||||
# we don't have a clean way for now to check that GV or Fenix
|
||||
# is ready to handle our tests. So here we just wait 30s
|
||||
LOG("Sleeping for 30s")
|
||||
logger.info("Sleeping for 30s")
|
||||
time.sleep(30)
|
||||
|
||||
def stop_browser(self):
|
||||
LOG("Stopping %s" % self.app_name)
|
||||
logger.info("Stopping %s" % self.app_name)
|
||||
try:
|
||||
self.device.stop_application(self.app_name, root=True)
|
||||
except ADBError:
|
||||
LOG("Could not stop the application using force-stop")
|
||||
logger.info("Could not stop the application using force-stop")
|
||||
|
||||
time.sleep(5)
|
||||
if self.device.process_exist(self.app_name):
|
||||
LOG("%s still running, trying SIGKILL" % self.app_name)
|
||||
logger.info("%s still running, trying SIGKILL" % self.app_name)
|
||||
num_tries = 0
|
||||
while self.device.process_exist(self.app_name) and num_tries < 5:
|
||||
try:
|
||||
|
|
@ -193,10 +194,10 @@ class AndroidDevice:
|
|||
pass
|
||||
num_tries += 1
|
||||
time.sleep(1)
|
||||
LOG("%s stopped" % self.app_name)
|
||||
logger.info("%s stopped" % self.app_name)
|
||||
|
||||
def collect_profile(self):
|
||||
LOG("Collecting profile from %s" % self.remote_profile)
|
||||
logger.info("Collecting profile from %s" % self.remote_profile)
|
||||
self.device.pull(self.remote_profile, self.profile)
|
||||
|
||||
def close(self):
|
||||
|
|
@ -206,7 +207,7 @@ class AndroidDevice:
|
|||
try:
|
||||
self.device.remove_forwards("tcp:%d" % self.marionette_port)
|
||||
except ADBError:
|
||||
LOG("Could not remove forward port")
|
||||
logger.warning("Could not remove forward port")
|
||||
|
||||
|
||||
# XXX redundant, remove
|
||||
|
|
@ -224,8 +225,8 @@ class AndroidGeckodriver(Geckodriver):
|
|||
async def start(self):
|
||||
port = free_port()
|
||||
await self._check_version()
|
||||
LOG("Running Webdriver on port %d" % port)
|
||||
LOG("Running Marionette on port 2828")
|
||||
logger.info("Running Webdriver on port %d" % port)
|
||||
logger.info("Running Marionette on port 2828")
|
||||
pargs = [
|
||||
self.binary,
|
||||
"--log",
|
||||
|
|
@ -235,7 +236,7 @@ class AndroidGeckodriver(Geckodriver):
|
|||
"--marionette-port",
|
||||
"2828",
|
||||
]
|
||||
LOG("Connecting on Android device")
|
||||
logger.info("Connecting on Android device")
|
||||
pargs.append("--connect-existing")
|
||||
return await subprocess_based_service(
|
||||
pargs, f"http://localhost:{port}", self.log_file
|
||||
|
|
@ -256,20 +257,20 @@ class AndroidEnv(BaseEnv):
|
|||
return "%s-%s" % (self.device_name, app)
|
||||
|
||||
def dump_logs(self):
|
||||
LOG("Dumping Android logs")
|
||||
logger.info("Dumping Android logs")
|
||||
try:
|
||||
logcat = self.device.get_logcat()
|
||||
if logcat:
|
||||
# local path, not using posixpath
|
||||
logfile = os.path.join(self.archive, "logcat.log")
|
||||
LOG("Writing logcat at %s" % logfile)
|
||||
logger.info("Writing logcat at %s" % logfile)
|
||||
with open(logfile, "wb") as f:
|
||||
for line in logcat:
|
||||
f.write(line.encode("utf8", errors="replace") + b"\n")
|
||||
else:
|
||||
LOG("logcat came back empty")
|
||||
logger.info("logcat came back empty")
|
||||
except Exception:
|
||||
ERROR("Could not extract the logcat")
|
||||
logger.error("Could not extract the logcat", exc_info=True)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def get_browser(self):
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ import time
|
|||
|
||||
from condprof import check_install # NOQA
|
||||
from condprof import progress
|
||||
from condprof.util import download_file, TASK_CLUSTER, LOG, ERROR, ArchiveNotFound
|
||||
from condprof.util import download_file, TASK_CLUSTER, logger, ArchiveNotFound
|
||||
from condprof.changelog import Changelog
|
||||
|
||||
|
||||
|
|
@ -58,14 +58,14 @@ def get_profile(
|
|||
"task_id": task_id,
|
||||
"repo": repo,
|
||||
}
|
||||
LOG("Getting conditioned profile with arguments: %s" % params)
|
||||
logger.info("Getting conditioned profile with arguments: %s" % params)
|
||||
filename = ARTIFACT_NAME % params
|
||||
if task_id is None:
|
||||
url = TC_LINK % params + filename
|
||||
else:
|
||||
url = DIRECT_LINK % params + filename
|
||||
|
||||
LOG("preparing download dir")
|
||||
logger.info("preparing download dir")
|
||||
if not download_cache:
|
||||
download_dir = tempfile.mkdtemp()
|
||||
else:
|
||||
|
|
@ -75,12 +75,12 @@ def get_profile(
|
|||
os.makedirs(download_dir)
|
||||
|
||||
downloaded_archive = os.path.join(download_dir, filename)
|
||||
LOG("Downloaded archive path: %s" % downloaded_archive)
|
||||
logger.info("Downloaded archive path: %s" % downloaded_archive)
|
||||
retries = 0
|
||||
|
||||
while retries < RETRIES:
|
||||
try:
|
||||
LOG("Getting %s" % url)
|
||||
logger.info("Getting %s" % url)
|
||||
try:
|
||||
archive = download_file(url, target=downloaded_archive)
|
||||
except ArchiveNotFound:
|
||||
|
|
@ -88,7 +88,7 @@ def get_profile(
|
|||
|
||||
try:
|
||||
with tarfile.open(archive, "r:gz") as tar:
|
||||
LOG("Extracting the tarball content in %s" % target_dir)
|
||||
logger.info("Extracting the tarball content in %s" % target_dir)
|
||||
size = len(list(tar))
|
||||
with progress.Bar(expected_size=size) as bar:
|
||||
|
||||
|
|
@ -101,35 +101,35 @@ def get_profile(
|
|||
tar.extract = functools.partial(_extract, tar)
|
||||
tar.extractall(target_dir)
|
||||
except (OSError, tarfile.ReadError) as e:
|
||||
LOG("Failed to extract the tarball")
|
||||
logger.info("Failed to extract the tarball")
|
||||
if download_cache and os.path.exists(archive):
|
||||
LOG("Removing cached file to attempt a new download")
|
||||
logger.info("Removing cached file to attempt a new download")
|
||||
os.remove(archive)
|
||||
raise ProfileNotFoundError(str(e))
|
||||
finally:
|
||||
if not download_cache:
|
||||
shutil.rmtree(download_dir)
|
||||
LOG("Success, we have a profile to work with")
|
||||
logger.info("Success, we have a profile to work with")
|
||||
return target_dir
|
||||
except Exception:
|
||||
LOG("Failed to get the profile.")
|
||||
logger.info("Failed to get the profile.")
|
||||
retries += 1
|
||||
if os.path.exists(downloaded_archive):
|
||||
try:
|
||||
os.remove(downloaded_archive)
|
||||
except Exception:
|
||||
ERROR("Could not remove the file")
|
||||
logger.error("Could not remove the file")
|
||||
time.sleep(RETRY_PAUSE)
|
||||
|
||||
# If we reach that point, it means all attempts failed
|
||||
ERROR("All attempt failed")
|
||||
logger.error("All attempt failed")
|
||||
raise ProfileNotFoundError(url)
|
||||
|
||||
|
||||
def read_changelog(platform, repo="mozilla-central"):
|
||||
params = {"platform": platform, "repo": repo}
|
||||
changelog_url = CHANGELOG_LINK % params
|
||||
LOG("Getting %s" % changelog_url)
|
||||
logger.info("Getting %s" % changelog_url)
|
||||
download_dir = tempfile.mkdtemp()
|
||||
downloaded_changelog = os.path.join(download_dir, "changelog.json")
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ import os
|
|||
from arsenic import get_session
|
||||
from arsenic.browsers import Firefox
|
||||
|
||||
from condprof.util import fresh_profile, LOG, ERROR, obfuscate_file, obfuscate
|
||||
from condprof.util import fresh_profile, logger, obfuscate_file, obfuscate
|
||||
from condprof.scenarii import scenarii
|
||||
from condprof.client import get_profile, ProfileNotFoundError
|
||||
from condprof.archiver import Archiver
|
||||
|
|
@ -58,15 +58,17 @@ class ProfileCreator:
|
|||
return os.path.join(self.archive, filename)
|
||||
|
||||
async def run(self, headless=True):
|
||||
LOG("Building %s x %s" % (self.scenario, self.customization_data["name"]))
|
||||
logger.info(
|
||||
"Building %s x %s" % (self.scenario, self.customization_data["name"])
|
||||
)
|
||||
|
||||
if self.scenario in self.customization_data.get("ignore_scenario", []):
|
||||
LOG("Skipping (ignored scenario in that customization)")
|
||||
logger.info("Skipping (ignored scenario in that customization)")
|
||||
return
|
||||
|
||||
filter_by_platform = self.customization_data.get("platforms")
|
||||
if filter_by_platform and self.env.target_platform not in filter_by_platform:
|
||||
LOG("Skipping (ignored platform in that customization)")
|
||||
logger.info("Skipping (ignored platform in that customization)")
|
||||
return
|
||||
|
||||
with self.env.get_device(2828, verbose=True) as device:
|
||||
|
|
@ -79,7 +81,7 @@ class ProfileCreator:
|
|||
if not self.archive:
|
||||
return
|
||||
|
||||
LOG("Creating archive")
|
||||
logger.info("Creating archive")
|
||||
archiver = Archiver(self.scenario, self.env.profile, self.archive)
|
||||
# the archive name is of the form
|
||||
# profile-<platform>-<scenario>-<customization>.tgz
|
||||
|
|
@ -90,10 +92,10 @@ class ProfileCreator:
|
|||
if not os.path.exists(dir):
|
||||
os.makedirs(dir)
|
||||
archiver.create_archive(archive_name)
|
||||
LOG("Archive created at %s" % archive_name)
|
||||
logger.info("Archive created at %s" % archive_name)
|
||||
statinfo = os.stat(archive_name)
|
||||
LOG("Current size is %d" % statinfo.st_size)
|
||||
LOG("Extracting logs")
|
||||
logger.info("Current size is %d" % statinfo.st_size)
|
||||
logger.info("Extracting logs")
|
||||
if "logs" in metadata:
|
||||
logs = metadata.pop("logs")
|
||||
for prefix, prefixed_logs in logs.items():
|
||||
|
|
@ -103,7 +105,7 @@ class ProfileCreator:
|
|||
f.write(content.encode("utf-8"))
|
||||
|
||||
if metadata.get("result", 0) != 0:
|
||||
LOG("The scenario returned a bad exit code")
|
||||
logger.info("The scenario returned a bad exit code")
|
||||
raise Exception(metadata.get("result_message", "scenario error"))
|
||||
self.changelog.append("update", **metadata)
|
||||
|
||||
|
|
@ -115,7 +117,7 @@ class ProfileCreator:
|
|||
scenario_func = scenarii[scenario]
|
||||
if scenario in customization_data.get("scenario", {}):
|
||||
options = customization_data["scenario"][scenario]
|
||||
LOG("Loaded options for that scenario %s" % str(options))
|
||||
logger.info("Loaded options for that scenario %s" % str(options))
|
||||
else:
|
||||
options = {}
|
||||
|
||||
|
|
@ -132,14 +134,14 @@ class ProfileCreator:
|
|||
else:
|
||||
fresh_profile(profile, customization_data)
|
||||
|
||||
LOG("Updating profile located at %r" % profile)
|
||||
logger.info("Updating profile located at %r" % profile)
|
||||
metadata = Metadata(profile)
|
||||
|
||||
LOG("Starting the Gecko app...")
|
||||
logger.info("Starting the Gecko app...")
|
||||
adb_logs = self._log_filename("adb")
|
||||
self.env.prepare(logfile=adb_logs)
|
||||
geckodriver_logs = self._log_filename("geckodriver")
|
||||
LOG("Writing geckodriver logs in %s" % geckodriver_logs)
|
||||
logger.info("Writing geckodriver logs in %s" % geckodriver_logs)
|
||||
step = START
|
||||
try:
|
||||
firefox_instance = Firefox(**self.env.get_browser_args(headless))
|
||||
|
|
@ -150,19 +152,21 @@ class ProfileCreator:
|
|||
async with get_session(geckodriver, firefox_instance) as session:
|
||||
step = START_SCENARIO
|
||||
self.env.check_session(session)
|
||||
LOG("Running the %s scenario" % scenario)
|
||||
logger.info("Running the %s scenario" % scenario)
|
||||
metadata.update(await scenario_func(session, options))
|
||||
LOG("%s scenario done." % scenario)
|
||||
logger.info("%s scenario done." % scenario)
|
||||
except Exception:
|
||||
ERROR("%s scenario broke!" % scenario)
|
||||
logger.error("%s scenario broke!" % scenario)
|
||||
if step == START:
|
||||
LOG("Could not initialize the browser")
|
||||
logger.info("Could not initialize the browser")
|
||||
elif step == INIT_GECKODRIVER:
|
||||
LOG("Could not initialize Geckodriver")
|
||||
logger.info("Could not initialize Geckodriver")
|
||||
elif step == START_SESSION:
|
||||
LOG("Could not start the session, check %s first" % geckodriver_logs)
|
||||
logger.info(
|
||||
"Could not start the session, check %s first" % geckodriver_logs
|
||||
)
|
||||
else:
|
||||
LOG("Could not run the scenario, probably a faulty scenario")
|
||||
logger.info("Could not run the scenario, probably a faulty scenario")
|
||||
raise
|
||||
finally:
|
||||
self.env.stop_browser()
|
||||
|
|
@ -179,6 +183,5 @@ class ProfileCreator:
|
|||
platform=self.env.target_platform,
|
||||
)
|
||||
|
||||
LOG("Profile at %s" % profile)
|
||||
LOG("Done.")
|
||||
logger.info("Profile at %s.\nDone." % profile)
|
||||
return metadata
|
||||
|
|
|
|||
|
|
@ -10,9 +10,8 @@ from arsenic.services import Geckodriver, free_port, subprocess_based_service
|
|||
from condprof.util import (
|
||||
BaseEnv,
|
||||
latest_nightly,
|
||||
LOG,
|
||||
logger,
|
||||
get_version,
|
||||
ERROR,
|
||||
get_current_platform,
|
||||
DEFAULT_PREFS,
|
||||
)
|
||||
|
|
@ -23,8 +22,8 @@ class DesktopGeckodriver(Geckodriver):
|
|||
async def start(self):
|
||||
port = free_port()
|
||||
await self._check_version()
|
||||
LOG("Running Webdriver on port %d" % port)
|
||||
LOG("Running Marionette on port 2828")
|
||||
logger.info("Running Webdriver on port %d" % port)
|
||||
logger.info("Running Marionette on port 2828")
|
||||
pargs = [
|
||||
self.binary,
|
||||
"--log",
|
||||
|
|
@ -76,8 +75,8 @@ class DesktopEnv(BaseEnv):
|
|||
def get_browser_version(self):
|
||||
try:
|
||||
return get_version(self.firefox)
|
||||
except Exception as e:
|
||||
ERROR("Could not get Firefox version %s" % str(e))
|
||||
except Exception:
|
||||
logger.error("Could not get Firefox version", exc_info=True)
|
||||
return "unknown"
|
||||
|
||||
def get_geckodriver(self, log_file):
|
||||
|
|
|
|||
|
|
@ -3,8 +3,7 @@
|
|||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
""" Helpers to build scenarii
|
||||
"""
|
||||
from condprof.util import ERROR, LOG
|
||||
|
||||
from condprof.util import logger
|
||||
|
||||
_SUPPORTED_MOBILE_BROWSERS = "fenix", "gecko", "fennec"
|
||||
|
||||
|
|
@ -46,7 +45,7 @@ class TabSwitcher:
|
|||
self.handles = await self.session.get_window_handles()
|
||||
self.current = 0
|
||||
except Exception:
|
||||
ERROR("Could not get window handles")
|
||||
logger.error("Could not get window handles")
|
||||
return
|
||||
|
||||
handle = self.handles[self.current]
|
||||
|
|
@ -57,7 +56,7 @@ class TabSwitcher:
|
|||
try:
|
||||
await self.session.switch_to_window(handle)
|
||||
except Exception:
|
||||
ERROR("Could not switch to handle %s" % str(handle))
|
||||
logger.error("Could not switch to handle %s" % str(handle))
|
||||
|
||||
|
||||
# 10 minutes
|
||||
|
|
@ -68,21 +67,21 @@ async def execute_async_script(session, script, *args):
|
|||
# switch to the right context if needed
|
||||
current_context = await session._request(url="/moz/context", method="GET")
|
||||
if current_context != "chrome":
|
||||
LOG("Switching to chrome context")
|
||||
logger.info("Switching to chrome context")
|
||||
await session._request(
|
||||
url="/moz/context", method="POST", data={"context": "chrome"}
|
||||
)
|
||||
switch_back = True
|
||||
else:
|
||||
switch_back = False
|
||||
LOG("Setting up script timeout")
|
||||
logger.info("Setting up script timeout")
|
||||
await session._request(
|
||||
url="/timeouts", method="POST", data={"script": _SCRIPT_TIMEOUT}
|
||||
)
|
||||
try:
|
||||
attempts = 0
|
||||
while True:
|
||||
LOG("Running triggerSync()")
|
||||
logger.info("Running triggerSync()")
|
||||
try:
|
||||
return await session._request(
|
||||
url="/execute/async",
|
||||
|
|
@ -91,8 +90,7 @@ async def execute_async_script(session, script, *args):
|
|||
)
|
||||
except Exception as e:
|
||||
attempts += 1
|
||||
LOG("The script failed.")
|
||||
LOG("Error:" + str(e))
|
||||
logger.error("The script failed.", exc_info=True)
|
||||
if attempts > 2:
|
||||
return {
|
||||
"result": 1,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import datetime
|
|||
from collections.abc import MutableMapping
|
||||
import json
|
||||
|
||||
from condprof.util import LOG
|
||||
from condprof.util import logger
|
||||
|
||||
|
||||
METADATA_NAME = "condprofile.json"
|
||||
|
|
@ -20,9 +20,9 @@ class Metadata(MutableMapping):
|
|||
|
||||
def __init__(self, profile_dir):
|
||||
self.metadata_file = os.path.join(profile_dir, METADATA_NAME)
|
||||
LOG("Reading existing metadata at %s" % self.metadata_file)
|
||||
logger.info("Reading existing metadata at %s" % self.metadata_file)
|
||||
if not os.path.exists(self.metadata_file):
|
||||
LOG("Could not find the metadata file in that profile")
|
||||
logger.info("Could not find the metadata file in that profile")
|
||||
self._data = {}
|
||||
else:
|
||||
with open(self.metadata_file) as f:
|
||||
|
|
@ -67,7 +67,7 @@ class Metadata(MutableMapping):
|
|||
|
||||
def write(self, **extras):
|
||||
# writing metadata
|
||||
LOG("Creating metadata...")
|
||||
logger.info("Creating metadata...")
|
||||
self._data.update(**extras)
|
||||
ts = str(datetime.datetime.now())
|
||||
if "created" not in self._data:
|
||||
|
|
@ -80,6 +80,6 @@ class Metadata(MutableMapping):
|
|||
# adding info about the firefox version
|
||||
# XXX build ID ??
|
||||
# XXX android ??
|
||||
LOG("Saving metadata file in %s" % self.metadata_file)
|
||||
logger.info("Saving metadata file in %s" % self.metadata_file)
|
||||
with open(self.metadata_file, "w") as f:
|
||||
f.write(json.dumps(self._data))
|
||||
|
|
|
|||
|
|
@ -23,8 +23,7 @@ from condprof.android import AndroidEnv # NOQA
|
|||
from condprof.changelog import Changelog # NOQA
|
||||
from condprof.scenarii import scenarii # NOQA
|
||||
from condprof.util import (
|
||||
LOG,
|
||||
ERROR,
|
||||
logger,
|
||||
get_version,
|
||||
get_current_platform,
|
||||
extract_from_dmg,
|
||||
|
|
@ -93,7 +92,7 @@ def main(args=sys.argv[1:]):
|
|||
raise IOError("Cannot find scenario %r" % args.scenario)
|
||||
|
||||
if not args.android and args.firefox is not None:
|
||||
LOG("Verifying Desktop Firefox binary")
|
||||
logger.info("Verifying Desktop Firefox binary")
|
||||
# we want to verify we do have a firefox binary
|
||||
# XXX so lame
|
||||
if not os.path.exists(args.firefox):
|
||||
|
|
@ -106,15 +105,15 @@ def main(args=sys.argv[1:]):
|
|||
raise IOError("Cannot find %s" % args.firefox)
|
||||
|
||||
version = get_version(args.firefox)
|
||||
LOG("Working with Firefox %s" % version)
|
||||
logger.info("Working with Firefox %s" % version)
|
||||
|
||||
LOG(os.environ)
|
||||
logger.info(os.environ)
|
||||
args.archive = os.path.abspath(args.archive)
|
||||
LOG("Archives directory is %s" % args.archive)
|
||||
logger.info("Archives directory is %s" % args.archive)
|
||||
if not os.path.exists(args.archive):
|
||||
os.makedirs(args.archive, exist_ok=True)
|
||||
|
||||
LOG("Verifying Geckodriver binary presence")
|
||||
logger.info("Verifying Geckodriver binary presence")
|
||||
if shutil.which(args.geckodriver) is None and not os.path.exists(args.geckodriver):
|
||||
raise IOError("Cannot find %s" % args.geckodriver)
|
||||
|
||||
|
|
@ -124,9 +123,9 @@ def main(args=sys.argv[1:]):
|
|||
else:
|
||||
plat = get_current_platform()
|
||||
changelog = read_changelog(plat)
|
||||
LOG("Got the changelog from TaskCluster")
|
||||
logger.info("Got the changelog from TaskCluster")
|
||||
except ProfileNotFoundError:
|
||||
LOG("changelog not found on TaskCluster, creating a local one.")
|
||||
logger.info("changelog not found on TaskCluster, creating a local one.")
|
||||
changelog = Changelog(args.archive)
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
|
|
@ -161,30 +160,34 @@ def main(args=sys.argv[1:]):
|
|||
# for the current platform when "all" is selected
|
||||
res = []
|
||||
failures = 0
|
||||
|
||||
def display_error(scenario, customization):
|
||||
logger.error("%s x %s failed." % (scenario, customization), exc_info=True)
|
||||
|
||||
for scenario in selected_scenario:
|
||||
if args.customization != "all":
|
||||
try:
|
||||
res.append(await one_run(scenario, args.customization))
|
||||
except Exception:
|
||||
failures += 1
|
||||
ERROR("Something went wrong on this one.")
|
||||
display_error(scenario, args.customization)
|
||||
if args.strict:
|
||||
raise
|
||||
else:
|
||||
for customization in get_customizations():
|
||||
LOG("Customization %s" % customization)
|
||||
logger.info("Customization %s" % customization)
|
||||
try:
|
||||
res.append(await one_run(scenario, customization))
|
||||
except Exception:
|
||||
failures += 1
|
||||
ERROR("Something went wrong on this one.")
|
||||
display_error(scenario, customization)
|
||||
if args.strict:
|
||||
raise
|
||||
return failures, [one_res for one_res in res if one_res]
|
||||
|
||||
try:
|
||||
failures, results = loop.run_until_complete(run_all(args))
|
||||
LOG("Saving changelog in %s" % args.archive)
|
||||
logger.info("Saving changelog in %s" % args.archive)
|
||||
changelog.save(args.archive)
|
||||
if failures > 0:
|
||||
raise Exception("At least one scenario failed")
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import random
|
|||
import os
|
||||
import asyncio
|
||||
|
||||
from condprof.util import LOG, get_credentials
|
||||
from condprof.util import logger, get_credentials
|
||||
from condprof.helpers import TabSwitcher, execute_async_script, is_mobile
|
||||
|
||||
|
||||
|
|
@ -54,8 +54,8 @@ class Builder:
|
|||
if not self.syncing:
|
||||
return
|
||||
# now that we've visited all pages, we want to upload to FXSync
|
||||
LOG("Syncing profile to FxSync")
|
||||
LOG("Username is %s, password is %s" % (self.username, self.password))
|
||||
logger.info("Syncing profile to FxSync")
|
||||
logger.info("Username is %s, password is %s" % (self.username, self.password))
|
||||
script_res = await execute_async_script(
|
||||
session,
|
||||
self.sync_js,
|
||||
|
|
@ -78,7 +78,7 @@ class Builder:
|
|||
visited = 0
|
||||
|
||||
for current, url in enumerate(self.urls):
|
||||
LOG("%d/%d %s" % (current + 1, self.max_urls, url))
|
||||
logger.info("%d/%d %s" % (current + 1, self.max_urls, url))
|
||||
retries = 0
|
||||
while retries < 3:
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -14,15 +14,15 @@ import shutil
|
|||
import contextlib
|
||||
import yaml
|
||||
from subprocess import Popen, PIPE
|
||||
import traceback
|
||||
import sys
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
|
||||
import requests
|
||||
from requests.exceptions import ConnectionError
|
||||
from requests.packages.urllib3.util.retry import Retry
|
||||
|
||||
import mozlog
|
||||
|
||||
from condprof import progress
|
||||
|
||||
|
||||
|
|
@ -50,76 +50,84 @@ DEFAULT_PREFS = {
|
|||
DEFAULT_CUSTOMIZATION = os.path.join(
|
||||
os.path.dirname(__file__), "customization", "default.json"
|
||||
)
|
||||
|
||||
_LOGGER = None
|
||||
STRUCTLOG_PAD_SIZE = 20
|
||||
|
||||
|
||||
class NullLogger:
|
||||
def info(self, *args, **kw):
|
||||
# XXX only if debug
|
||||
# print("%s %s" % (str(args), str(kw)))
|
||||
pass
|
||||
class BridgeLogger:
|
||||
def __init__(self, logger):
|
||||
self.logger = logger
|
||||
|
||||
def visit_url(self, index, total, url):
|
||||
print("%d/%d %s" % (index, total, url))
|
||||
def _find(self, text, *names):
|
||||
# structlog's ConsoleRenderer pads values
|
||||
for name in names:
|
||||
if name + " " * STRUCTLOG_PAD_SIZE in text:
|
||||
return True
|
||||
return False
|
||||
|
||||
def msg(self, event):
|
||||
print(event)
|
||||
def _convert(self, message):
|
||||
return obfuscate(message)[1]
|
||||
|
||||
def error(self, event, *args, **kw):
|
||||
print(event)
|
||||
traceback.print_exc(file=sys.stdout)
|
||||
def info(self, message, *args, **kw):
|
||||
if not isinstance(message, str):
|
||||
message = str(message)
|
||||
# converting Arsenic request/response struct log
|
||||
if self._find(message, "request", "response"):
|
||||
self.logger.debug(self._convert(message), *args, **kw)
|
||||
else:
|
||||
self.logger.info(self._convert(message), *args, **kw)
|
||||
|
||||
def error(self, message, *args, **kw):
|
||||
self.logger.error(self._convert(message), *args, **kw)
|
||||
|
||||
def warning(self, message, *args, **kw):
|
||||
self.logger.warning(self._convert(message), *args, **kw)
|
||||
|
||||
|
||||
def _get_logger():
|
||||
global _LOGGER
|
||||
if _LOGGER is not None:
|
||||
return _LOGGER
|
||||
logger = None
|
||||
|
||||
|
||||
def get_logger():
|
||||
global logger
|
||||
if logger is not None:
|
||||
return logger
|
||||
new_logger = mozlog.get_default_logger("condprof")
|
||||
if new_logger is None:
|
||||
new_logger = mozlog.unstructured.getLogger("condprof")
|
||||
|
||||
# wrap the logger into the BridgeLogger
|
||||
new_logger = BridgeLogger(new_logger)
|
||||
|
||||
# bridge for Arsenic
|
||||
if sys.version_info.major == 3:
|
||||
# plugging the logger into arsenic
|
||||
try:
|
||||
from arsenic import connection
|
||||
from structlog import wrap_logger
|
||||
from arsenic import connection
|
||||
from structlog import wrap_logger
|
||||
|
||||
logger = wrap_logger(NullLogger(), processors=[])
|
||||
connection.log = logger
|
||||
except ImportError:
|
||||
logger = NullLogger()
|
||||
else:
|
||||
# on python 2, just using the plain logger
|
||||
logger = NullLogger()
|
||||
connection.log = wrap_logger(new_logger)
|
||||
|
||||
_LOGGER = logger
|
||||
return _LOGGER
|
||||
logger = new_logger
|
||||
return logger
|
||||
|
||||
|
||||
def LOG(msg):
|
||||
msg = "[%s] %s" % (datetime.now().isoformat(), msg)
|
||||
_get_logger().msg(obfuscate(msg)[1])
|
||||
|
||||
|
||||
def ERROR(msg):
|
||||
msg = "[%s] %s" % (datetime.now().isoformat(), msg)
|
||||
_get_logger().error(obfuscate(msg)[1])
|
||||
# initializing the logger right away
|
||||
get_logger()
|
||||
|
||||
|
||||
def fresh_profile(profile, customization_data):
|
||||
from mozprofile import create_profile # NOQA
|
||||
|
||||
# XXX on android we mgiht need to run it on the device?
|
||||
LOG("Creating a fresh profile")
|
||||
logger.info("Creating a fresh profile")
|
||||
new_profile = create_profile(app="firefox")
|
||||
prefs = customization_data["prefs"]
|
||||
prefs.update(DEFAULT_PREFS)
|
||||
LOG("Setting prefs %s" % str(prefs.items()))
|
||||
logger.info("Setting prefs %s" % str(prefs.items()))
|
||||
new_profile.set_preferences(prefs)
|
||||
extensions = []
|
||||
for name, url in customization_data["addons"].items():
|
||||
LOG("Downloading addon %s" % name)
|
||||
logger.info("Downloading addon %s" % name)
|
||||
extension = download_file(url)
|
||||
extensions.append(extension)
|
||||
LOG("Installing addons")
|
||||
logger.info("Installing addons")
|
||||
new_profile.addons.install(extensions, unpack=True)
|
||||
new_profile.addons.install(extensions)
|
||||
shutil.copytree(new_profile.profile, profile)
|
||||
|
|
@ -175,7 +183,7 @@ def check_exists(archive, server=None):
|
|||
def download_file(url, target=None):
|
||||
present, headers = check_exists(url)
|
||||
if not present:
|
||||
LOG("Cannot find %r" % url)
|
||||
logger.info("Cannot find %r" % url)
|
||||
raise ArchiveNotFound(url)
|
||||
|
||||
etag = headers.get("ETag")
|
||||
|
|
@ -190,18 +198,20 @@ def download_file(url, target=None):
|
|||
with open(target + ".etag") as f:
|
||||
current_etag = f.read()
|
||||
if etag == current_etag:
|
||||
LOG("Already Downloaded.")
|
||||
logger.info("Already Downloaded.")
|
||||
# should at least check the size?
|
||||
return target
|
||||
else:
|
||||
LOG("Changed!")
|
||||
logger.info("Changed!")
|
||||
|
||||
LOG("Downloading %s" % url)
|
||||
logger.info("Downloading %s" % url)
|
||||
req = requests.get(url, stream=True)
|
||||
total_length = int(req.headers.get("content-length"))
|
||||
target_dir = os.path.dirname(target)
|
||||
if target_dir != "" and not os.path.exists(target_dir):
|
||||
logger.info("Creating dir %s" % target_dir)
|
||||
os.makedirs(target_dir)
|
||||
|
||||
with open(target, "wb") as f:
|
||||
if TASK_CLUSTER:
|
||||
for chunk in req.iter_content(chunk_size=1024):
|
||||
|
|
@ -249,7 +259,7 @@ def latest_nightly(binary=None):
|
|||
if binary is None:
|
||||
# we want to use the latest nightly
|
||||
nightly_archive = get_firefox_download_link()
|
||||
LOG("Downloading %s" % nightly_archive)
|
||||
logger.info("Downloading %s" % nightly_archive)
|
||||
target = download_file(nightly_archive)
|
||||
# on macOs we just mount the DMG
|
||||
# XXX replace with extract_from_dmg
|
||||
|
|
@ -274,7 +284,7 @@ def latest_nightly(binary=None):
|
|||
# XXX replace with extract_from_dmg
|
||||
if mounted:
|
||||
if platform.system() == "Darwin":
|
||||
LOG("Unmounting Firefox")
|
||||
logger.info("Unmounting Firefox")
|
||||
time.sleep(10)
|
||||
os.system("hdiutil detach /Volumes/Nightly")
|
||||
elif platform.system() == "Linux":
|
||||
|
|
@ -283,12 +293,12 @@ def latest_nightly(binary=None):
|
|||
|
||||
|
||||
def write_yml_file(yml_file, yml_data):
|
||||
LOG("writing %s to %s" % (yml_data, yml_file))
|
||||
logger.info("writing %s to %s" % (yml_data, yml_file))
|
||||
try:
|
||||
with open(yml_file, "w") as outfile:
|
||||
yaml.dump(yml_data, outfile, default_flow_style=False)
|
||||
except Exception as e:
|
||||
ERROR("failed to write yaml file, exeption: %s" % e)
|
||||
except Exception:
|
||||
logger.error("failed to write yaml file", exc_info=True)
|
||||
|
||||
|
||||
def get_version(firefox):
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
../mozbase/mozfile
|
||||
../mozbase/mozprofile
|
||||
../mozbase/mozlog
|
||||
|
||||
requests==2.22.0
|
||||
pyyaml==5.1.2
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
../mozbase/mozfile
|
||||
../mozbase/mozprofile
|
||||
../mozbase/mozdevice
|
||||
../mozbase/mozlog
|
||||
|
||||
aiohttp==3.5.4
|
||||
https://pypi.pub.build.mozilla.org/pub/arsenic-19.1-py3-none-any.whl
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
./mozfile
|
||||
./mozprofile
|
||||
./mozdevice
|
||||
./mozlog
|
||||
|
||||
aiohttp==3.5.4
|
||||
https://pypi.pub.build.mozilla.org/pub/arsenic-19.1-py3-none-any.whl
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче