Bug 1597336 - add a fxa+sync scenario r=lina,markh

This patch adds a scenario that ends up syncing the generated profile into FxSync.

Differential Revision: https://phabricator.services.mozilla.com/D53789

--HG--
extra : moz-landing-system : lando
This commit is contained in:
Tarek Ziadé 2020-02-23 07:48:19 +00:00
Родитель 49926cf5f0
Коммит f2659ce7c3
20 изменённых файлов: 768 добавлений и 37 удалений

Просмотреть файл

@ -33,6 +33,7 @@ exclude =
testing/condprofile/condprof/runner.py,
testing/condprofile/condprof/scenarii/heavy.py,
testing/condprofile/condprof/scenarii/settled.py,
testing/condprofile/condprof/scenarii/synced.py
testing/condprofile/condprof/helpers.py,
testing/jsshell/benchmark.py,
testing/marionette/mach_commands.py,

Просмотреть файл

@ -0,0 +1,10 @@
"use strict";
module.exports = {
"env": {
"browser": true
},
"rules": {
"no-unused-vars": ["error", {"args": "none", "varsIgnorePattern": "^EXPORTED_SYMBOLS|^Sync"}],
}
};

Просмотреть файл

@ -0,0 +1,2 @@
# Register resource aliases
resource services-automation resource://gre/modules/services-automation/

Просмотреть файл

@ -0,0 +1,426 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
/*
* This module is used in automation to connect the browser to
* a specific FxA account and trigger FX Sync.
*
* To use it, you can call this sequence:
*
* initConfig("https://accounts.stage.mozaws.net");
* await Authentication.signIn(username, password);
* await Sync.triggerSync();
* await Authentication.signOut();
*
*
* Where username is your FxA e-mail. it will connect your browser
* to that account and trigger a Sync (on stage servers.)
*
* You can also use the convenience function that does everything:
*
* await triggerSync(username, password, "https://accounts.stage.mozaws.net");
*
*/
var EXPORTED_SYMBOLS = ["Sync", "Authentication", "initConfig", "triggerSync"];
const { XPCOMUtils } = ChromeUtils.import(
"resource://gre/modules/XPCOMUtils.jsm"
);
XPCOMUtils.defineLazyModuleGetters(this, {
Services: "resource://gre/modules/Services.jsm",
Log: "resource://gre/modules/Log.jsm",
Weave: "resource://services-sync/main.js",
Svc: "resource://services-sync/util.js",
fxAccounts: "resource://gre/modules/FxAccounts.jsm",
FxAccountsClient: "resource://gre/modules/FxAccountsClient.jsm",
FxAccountsConfig: "resource://gre/modules/FxAccountsConfig.jsm",
OS: "resource://gre/modules/osfile.jsm",
setTimeout: "resource://gre/modules/Timer.jsm",
clearTimeout: "resource://gre/modules/Timer.jsm",
});
XPCOMUtils.defineLazyGlobalGetters(this, ["fetch"]);
const AUTOCONFIG_PREF = "identity.fxaccounts.autoconfig.uri";
/*
* Log helpers.
*/
var _LOG = [];
function LOG(msg, error) {
console.debug(msg);
_LOG.push(msg);
if (error) {
console.debug(JSON.stringify(error));
_LOG.push(JSON.stringify(error));
}
}
function dumpLogs() {
let res = _LOG.join("\n");
_LOG = [];
return res;
}
function promiseObserver(aEventName) {
LOG("wait for " + aEventName);
return new Promise(resolve => {
let handler = () => {
Svc.Obs.remove(aEventName, handler);
resolve();
};
let handlerTimeout = () => {
Svc.Obs.remove(aEventName, handler);
LOG("handler timed out " + aEventName);
resolve();
};
Svc.Obs.add(aEventName, handler);
setTimeout(handlerTimeout, 3000);
});
}
/*
* Authentication
*
* Used to sign in an FxA account, takes care of
* the e-mail verification flow.
*
* Usage:
*
* await Authentication.signIn(username, password);
*/
var Authentication = {
async isLoggedIn() {
return !!(await this.getSignedInUser());
},
async isReady() {
let user = await this.getSignedInUser();
if (user) {
LOG("current user " + JSON.stringify(user));
}
return user && user.verified;
},
async getSignedInUser() {
try {
return await fxAccounts.getSignedInUser();
} catch (error) {
LOG("getSignedInUser() failed", error);
throw error;
}
},
async shortWaitForVerification(ms) {
LOG("shortWaitForVerification");
let userData = await this.getSignedInUser();
let timeoutID;
LOG("set a timeout");
let timeoutPromise = new Promise(resolve => {
timeoutID = setTimeout(() => {
LOG(`Warning: no verification after ${ms}ms.`);
resolve();
}, ms);
});
LOG("set a fxAccounts.whenVerified");
await Promise.race([
fxAccounts.whenVerified(userData).finally(() => clearTimeout(timeoutID)),
timeoutPromise,
]);
LOG("done");
return this.isReady();
},
async _confirmUser(uri) {
LOG("Open new tab and load verification page");
let mainWindow = Services.wm.getMostRecentWindow("navigator:browser");
let newtab = mainWindow.gBrowser.addWebTab(uri);
let win = mainWindow.gBrowser.getBrowserForTab(newtab);
win.addEventListener("load", function(e) {
LOG("load");
});
win.addEventListener("loadstart", function(e) {
LOG("loadstart");
});
win.addEventListener("error", function(msg, url, lineNo, columnNo, error) {
var string = msg.toLowerCase();
var substring = "script error";
if (string.indexOf(substring) > -1) {
LOG("Script Error: See Browser Console for Detail");
} else {
var message = [
"Message: " + msg,
"URL: " + url,
"Line: " + lineNo,
"Column: " + columnNo,
"Error object: " + JSON.stringify(error),
].join(" - ");
LOG(message);
}
});
LOG("wait for page to load");
await new Promise(resolve => {
let handlerTimeout = () => {
LOG("timed out ");
resolve();
};
var timer = setTimeout(handlerTimeout, 10000);
win.addEventListener("loadend", function() {
resolve();
clearTimeout(timer);
});
});
LOG("Page Loaded");
let didVerify = await this.shortWaitForVerification(10000);
LOG("remove tab");
mainWindow.gBrowser.removeTab(newtab);
return didVerify;
},
/*
* This whole verification process may be bypassed if the
* account is whitelisted.
*/
async _completeVerification(username) {
LOG("Fetching mail (from restmail) for user " + username);
let restmailURI = `https://www.restmail.net/mail/${encodeURIComponent(
username
)}`;
let triedAlready = new Set();
const tries = 10;
const normalWait = 4000;
for (let i = 0; i < tries; ++i) {
let resp = await fetch(restmailURI);
let messages = await resp.json();
// Sort so that the most recent emails are first.
messages.sort((a, b) => new Date(b.receivedAt) - new Date(a.receivedAt));
for (let m of messages) {
// We look for a link that has a x-link that we haven't yet tried.
if (!m.headers["x-link"] || triedAlready.has(m.headers["x-link"])) {
continue;
}
if (!m.headers["x-verify-code"]) {
continue;
}
let confirmLink = m.headers["x-link"];
triedAlready.add(confirmLink);
LOG("Trying confirmation link " + confirmLink);
try {
if (await this._confirmUser(confirmLink)) {
LOG("confirmation done");
return true;
}
LOG("confirmation failed");
} catch (e) {
LOG(
"Warning: Failed to follow confirmation link: " +
Log.exceptionStr(e)
);
}
}
if (i === 0) {
// first time through after failing we'll do this.
LOG("resendVerificationEmail");
await fxAccounts.resendVerificationEmail();
}
if (await this.shortWaitForVerification(normalWait)) {
return true;
}
}
// One last try.
return this.shortWaitForVerification(normalWait);
},
async signIn(username, password) {
LOG("Login user: " + username);
try {
// Required here since we don't go through the real login page
LOG("Calling FxAccountsConfig.ensureConfigured");
await FxAccountsConfig.ensureConfigured();
let client = new FxAccountsClient();
LOG("Signing in");
let credentials = await client.signIn(username, password, true);
LOG("Signed in, setting up the signed user in fxAccounts");
await fxAccounts._internal.setSignedInUser(credentials);
// If the account is not whitelisted for tests, we need to verify it
if (!credentials.verified) {
LOG("We need to verify the account");
await this._completeVerification(username);
} else {
LOG("Credentials already verified");
}
return true;
} catch (error) {
LOG("signIn() failed", error);
throw error;
}
},
async signOut() {
if (await Authentication.isLoggedIn()) {
// Note: This will clean up the device ID.
await fxAccounts.signOut();
}
},
};
/*
* Sync
*
* Used to trigger sync.
*
* usage:
*
* await Sync.triggerSync();
*/
var Sync = {
getSyncLogsDirectory() {
return OS.Path.join(OS.Constants.Path.profileDir, ...["weave", "logs"]);
},
async init() {
Svc.Obs.add("weave:service:sync:error", this);
Svc.Obs.add("weave:service:setup-complete", this);
Svc.Obs.add("weave:service:tracking-started", this);
// Delay the automatic sync operations, so we can trigger it manually
Weave.Svc.Prefs.set("scheduler.immediateInterval", 7200);
Weave.Svc.Prefs.set("scheduler.idleInterval", 7200);
Weave.Svc.Prefs.set("scheduler.activeInterval", 7200);
Weave.Svc.Prefs.set("syncThreshold", 10000000);
// Wipe all the logs
await this.wipeLogs();
},
observe(subject, topic, data) {
LOG("Event received " + topic);
},
async configureSync() {
// todo, enable all sync engines here
// the addon engine requires kinto creds...
LOG("configuring sync");
console.assert(await Authentication.isReady(), "You are not connected");
await Weave.Service.configure();
if (!Weave.Status.ready) {
await promiseObserver("weave:service:ready");
}
if (Weave.Service.locked) {
await promiseObserver("weave:service:resyncs-finished");
}
},
/*
* triggerSync() runs the whole process of Syncing.
*
* returns 1 on success, 0 on failure.
*/
async triggerSync() {
if (!(await Authentication.isLoggedIn())) {
LOG("Not connected");
return 1;
}
await this.init();
let result = 1;
try {
await this.configureSync();
LOG("Triggering a sync");
await Weave.Service.sync();
// wait a second for things to settle...
await new Promise(resolve => setTimeout(resolve, 1000));
LOG("Sync done");
result = 0;
} catch (error) {
LOG("triggerSync() failed", error);
}
return result;
},
async wipeLogs() {
let outputDirectory = this.getSyncLogsDirectory();
if (!(await OS.File.exists(outputDirectory))) {
return;
}
LOG("Wiping existing Sync logs");
try {
let iterator = new OS.File.DirectoryIterator(outputDirectory);
await iterator.forEach(async entry => {
try {
await OS.File.remove(entry.path);
} catch (error) {
LOG("wipeLogs() could not remove " + entry.path, error);
}
});
} catch (error) {
LOG("wipeLogs() failed", error);
}
},
async getLogs() {
let outputDirectory = this.getSyncLogsDirectory();
let entries = [];
if (await OS.File.exists(outputDirectory)) {
// Iterate through the directory
let iterator = new OS.File.DirectoryIterator(outputDirectory);
await iterator.forEach(async entry => {
let info = await OS.File.stat(entry.path);
entries.push({
path: entry.path,
name: entry.name,
lastModificationDate: info.lastModificationDate,
});
});
entries.sort(function(a, b) {
return b.lastModificationDate - a.lastModificationDate;
});
}
const promises = entries.map(async entry => {
let content = await OS.File.read(entry.path, {
encoding: "utf-8",
});
return {
name: entry.name,
content,
};
});
return Promise.all(promises);
},
};
function initConfig(autoconfig) {
Services.prefs.setCharPref(AUTOCONFIG_PREF, autoconfig);
}
async function triggerSync(username, password, autoconfig) {
initConfig(autoconfig);
await Authentication.signIn(username, password);
var result = await Sync.triggerSync();
await Authentication.signOut();
var logs = {
sync: await Sync.getLogs(),
condprof: [
{
name: "console.txt",
content: dumpLogs(),
},
],
};
return {
result,
logs,
};
}

Просмотреть файл

@ -0,0 +1,14 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
with Files('**'):
BUG_COMPONENT = ('Firefox', 'Services Automation')
EXTRA_COMPONENTS += [
'AutomationComponents.manifest',
]
EXTRA_JS_MODULES['services-automation'] += [
'ServicesAutomation.jsm',
]

Просмотреть файл

@ -13,6 +13,12 @@ DIRS += [
'settings',
]
# The automation dir is only included in nightlies and debug
if not CONFIG['RELEASE_OR_BETA'] or CONFIG['MOZ_DEBUG']:
DIRS += [
'automation'
]
if CONFIG['MOZ_WIDGET_TOOLKIT'] != 'android':
DIRS += [
'fxaccounts',

Просмотреть файл

@ -22,8 +22,11 @@ job-defaults:
using: run-task
checkout: false
tooltool-downloads: public
scopes:
- secrets:get:project/releng/gecko/build/level-{level}/conditioned-profiles
worker:
max-run-time: 5400
taskcluster-proxy: true
max-run-time: 10800
env:
MOZ_FETCHES_DIR: fetches
CONDPROF_ARCHIVES: /builds/worker/archive

Просмотреть файл

@ -28,7 +28,7 @@ import os
from arsenic import get_session
from arsenic.browsers import Firefox
from condprof.util import fresh_profile, LOG, ERROR
from condprof.util import fresh_profile, LOG, ERROR, obfuscate_file, obfuscate
from condprof.scenarii import scenarii
from condprof.client import get_profile, ProfileNotFoundError
from condprof.archiver import Archiver
@ -59,6 +59,16 @@ class ProfileCreator:
async def run(self, headless=True):
LOG("Building %s x %s" % (self.scenario, self.customization_data["name"]))
if self.scenario in self.customization_data.get("ignore_scenario", []):
LOG("Skipping (ignored scenario in that customization)")
return
filter_by_platform = self.customization_data.get("platforms")
if filter_by_platform and self.env.target_platform not in filter_by_platform:
LOG("Skipping (ignored platform in that customization)")
return
with self.env.get_device(2828, verbose=True) as device:
try:
with self.env.get_browser():
@ -83,9 +93,18 @@ class ProfileCreator:
LOG("Archive created at %s" % archive_name)
statinfo = os.stat(archive_name)
LOG("Current size is %d" % statinfo.st_size)
LOG("Extracting logs")
if "logs" in metadata:
logs = metadata.pop("logs")
for prefix, prefixed_logs in logs.items():
for log in prefixed_logs:
content = obfuscate(log["content"])[1]
with open(os.path.join(dir, prefix + "-" + log["name"]), "wb") as f:
f.write(content.encode("utf-8"))
if metadata.get("result", 0) != 0:
LOG("The scenario returned a bad exit code")
raise Exception("scenario error")
raise Exception(metadata.get("result_message", "scenario error"))
self.changelog.append("update", **metadata)
async def build_profile(self, device, headless):
@ -117,7 +136,8 @@ class ProfileCreator:
metadata = Metadata(profile)
LOG("Starting the Gecko app...")
self.env.prepare(logfile=self._log_filename("adb"))
adb_logs = self._log_filename("adb")
self.env.prepare(logfile=adb_logs)
geckodriver_logs = self._log_filename("geckodriver")
LOG("Writing geckodriver logs in %s" % geckodriver_logs)
step = START
@ -146,7 +166,9 @@ class ProfileCreator:
raise
finally:
self.env.stop_browser()
for logfile in (adb_logs, geckodriver_logs):
if os.path.exists(logfile):
obfuscate_file(logfile)
self.env.collect_profile()
# writing metadata

Просмотреть файл

@ -12,14 +12,21 @@ def get_customizations():
yield os.path.join(HERE, f)
def get_customization(path):
if not path.endswith(".json"):
path += ".json"
if not os.path.exists(path):
def find_customization(path_or_name):
if not path_or_name.endswith(".json"):
path_or_name += ".json"
if not os.path.exists(path_or_name):
# trying relative
rpath = os.path.join(HERE, path)
rpath = os.path.join(HERE, path_or_name)
if not os.path.exists(rpath):
raise IOError("Can't find the customization file %r" % path)
path = rpath
return None
path_or_name = rpath
return path_or_name
def get_customization(path_or_name):
path = find_customization(path_or_name)
if path is None:
raise IOError("Can't find the customization file %r" % path_or_name)
with open(path) as f:
return json.loads(f.read())

Просмотреть файл

@ -5,6 +5,7 @@
},
"scenario": {
"heavy": {"max_urls": 50}
}
"heavy": {"max_urls": 100}
},
"ignore_scenario": ["synced"]
}

Просмотреть файл

@ -0,0 +1,12 @@
{
"name": "sync",
"addons":{},
"prefs":{
},
"scenario": {
"synced": {"max_urls": 250}
},
"platforms": ["macosx64", "win32", "win64", "linux64"],
"ignore_scenario": ["heavy", "settled"]
}

Просмотреть файл

@ -3,7 +3,7 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
""" Helpers to build scenarii
"""
from condprof.util import ERROR
from condprof.util import ERROR, LOG
_SUPPORTED_MOBILE_BROWSERS = "fenix", "gecko", "fennec"
@ -56,3 +56,50 @@ class TabSwitcher:
await self.session.switch_to_window(handle)
except Exception:
ERROR("Could not switch to handle %s" % str(handle))
# 10 minutes
_SCRIPT_TIMEOUT = 10 * 60 * 1000
async def execute_async_script(session, script, *args):
# switch to the right context if needed
current_context = await session._request(url="/moz/context", method="GET")
if current_context != "chrome":
LOG("Switching to chrome context")
await session._request(
url="/moz/context", method="POST", data={"context": "chrome"}
)
switch_back = True
else:
switch_back = False
LOG("Setting up script timeout")
await session._request(
url="/timeouts", method="POST", data={"script": _SCRIPT_TIMEOUT}
)
try:
attempts = 0
while True:
LOG("Running triggerSync()")
try:
return await session._request(
url="/execute/async",
method="POST",
data={"script": script, "args": list(args)},
)
except Exception as e:
attempts += 1
LOG("The script failed.")
LOG("Error:" + str(e))
if attempts > 2:
return {
"result": 1,
"result_message": str(e),
"result_exc": e,
"logs": {},
}
finally:
if switch_back:
await session._request(
url="/moz/context", method="POST", data={"context": current_context}
)

Просмотреть файл

@ -29,7 +29,7 @@ from condprof.util import (
get_current_platform,
extract_from_dmg,
) # NOQA
from condprof.customization import get_customizations # NOQA
from condprof.customization import get_customizations, find_customization # NOQA
from condprof.client import read_changelog, ProfileNotFoundError # NOQA
@ -44,12 +44,6 @@ def main(args=sys.argv[1:]):
parser.add_argument(
"--customization", help="Profile customization to use", type=str, default="all"
)
parser.add_argument(
"--fresh-profile",
help="Create a fresh profile",
action="store_true",
default=False,
)
parser.add_argument(
"--visible", help="Don't use headless mode", action="store_true", default=False
)
@ -90,6 +84,14 @@ def main(args=sys.argv[1:]):
args.android = args.firefox is not None and args.firefox.startswith("org.mozilla")
# early checks to avoid extra work
if args.customization != "all":
if find_customization(args.customization) is None:
raise IOError("Cannot find customization %r" % args.customization)
if args.scenario != "all" and args.scenario not in scenarii:
raise IOError("Cannot find scenario %r" % args.scenario)
if not args.android and args.firefox is not None:
LOG("Verifying Desktop Firefox binary")
# we want to verify we do have a firefox binary

Просмотреть файл

@ -2,6 +2,7 @@ from __future__ import absolute_import
from condprof.scenarii.heavy import heavy
from condprof.scenarii.settled import settled
from condprof.scenarii.synced import synced
scenarii = {"heavy": heavy, "settled": settled}
scenarii = {"heavy": heavy, "settled": settled, "synced": synced}

Просмотреть файл

@ -2,7 +2,7 @@ import random
import os
import asyncio
from condprof.util import get_logger
from condprof.util import LOG
from condprof.helpers import TabSwitcher
@ -47,7 +47,7 @@ async def heavy(session, options):
visited = 0
for current, url in enumerate(URL_LIST):
get_logger().visit_url(index=current + 1, total=max, url=url)
LOG("%d/%d %s" % (current + 1, max, url))
retries = 0
while retries < 3:
try:

Просмотреть файл

@ -0,0 +1,17 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
const { triggerSync } = ChromeUtils.import(
"resource://gre/modules/services-automation/ServicesAutomation.jsm"
);
let resolve = arguments[3]; // eslint-disable-line
try {
triggerSync(arguments[0], arguments[1], arguments[2]).then(res => { // eslint-disable-line
resolve(res); // eslint-disable-line
});
} catch (error) {
let res = { logs: {}, result: 1, result_message: error.toString() };
resolve(res);
}

Просмотреть файл

@ -0,0 +1,55 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import asyncio
from condprof.scenarii.heavy import URL_LIST
from condprof.util import LOG, get_credentials
from condprof.helpers import TabSwitcher, execute_async_script
with open(os.path.join(os.path.dirname(__file__), "sync.js")) as f:
SYNC_SCRIPT = f.read()
async def synced(session, options):
username, password = get_credentials()
if username is None:
raise ValueError("The synced scenario needs an fxa username and" " password")
metadata = {}
max = options.get("max_urls", 150)
tabs = TabSwitcher(session, options)
await tabs.create_windows()
visited = 0
for current, url in enumerate(URL_LIST):
LOG("%d/%d %s" % (current + 1, max, url))
retries = 0
while retries < 3:
try:
await asyncio.wait_for(session.get(url), 5)
visited += 1
break
except asyncio.TimeoutError:
retries += 1
if max != -1 and current + 1 == max:
break
# switch to the next tab
await tabs.switch()
# now that we've visited all pages, we want to upload to FXSync
LOG("Syncing profile to FxSync")
LOG("Username is %s, password is %s" % (username, password))
script_res = await execute_async_script(
session, SYNC_SCRIPT, username, password, "https://accounts.stage.mozaws.net"
)
if script_res is None:
script_res = {}
metadata["logs"] = script_res.get("logs", {})
metadata["result"] = script_res.get("result", 0)
metadata["result_message"] = script_res.get("result_message", "SUCCESS")
metadata["visited_url"] = visited
return metadata

Просмотреть файл

@ -4,12 +4,17 @@ import tempfile
import shutil
import responses
import re
import json
from condprof.client import get_profile, ROOT_URL
from condprof.util import _DEFAULT_SERVER
PROFILE = re.compile(ROOT_URL + "/.*/.*tgz")
with open(os.path.join(os.path.dirname(__file__), "profile.tgz"), "rb") as f:
PROFILE_DATA = f.read()
SECRETS = re.compile(_DEFAULT_SERVER + "/.*")
SECRETS_PROXY = re.compile("http://taskcluster/secrets/.*")
class TestClient(unittest.TestCase):
@ -35,6 +40,17 @@ class TestClient(unittest.TestCase):
status=200,
)
secret = {"secret": {"username": "user", "password": "pass"}}
secret = json.dumps(secret)
for pattern in (SECRETS, SECRETS_PROXY):
responses.add(
responses.GET,
pattern,
body=secret,
headers={"content-length": str(len(secret))},
status=200,
)
def tearDown(self):
shutil.rmtree(self.target)
shutil.rmtree(self.download_dir)
@ -52,8 +68,9 @@ class TestClient(unittest.TestCase):
# grabbing a profile should generate two files
self.assertEqual(len(os.listdir(download_dir)), num_elmts + 2)
# we do two network calls when getting a file, a HEAD and a GET
self.assertEqual(len(responses.calls), 2)
# we do at least two network calls when getting a file,
# a HEAD and a GET and possibly a TC secret
self.assertTrue(len(responses.calls) >= 2)
# reseting the response counters
responses.calls.reset()
@ -64,7 +81,8 @@ class TestClient(unittest.TestCase):
# grabbing a profile should not download new stuff
self.assertEqual(len(os.listdir(download_dir)), num_elmts + 2)
# and do a single extra HEAD call
# and do a single extra HEAD call, everything else is cached,
# even the TC secret
self.assertEqual(len(responses.calls), 1)

Просмотреть файл

@ -86,7 +86,23 @@ class TestRunner(unittest.TestCase):
@responses.activate
def test_runner(self):
args = ["--geckodriver", GECKODRIVER, "--firefox", FIREFOX, self.archive_dir]
main(args)
# XXX we want a bunch of assertions here to check
# that the archives dir gets filled correctly
if "FXA_USERNAME" not in os.environ:
os.environ["FXA_USERNAME"] = "me"
if "FXA_PASSWORD" not in os.environ:
os.environ["FXA_PASSWORD"] = "password"
try:
args = [
"--geckodriver",
GECKODRIVER,
"--firefox",
FIREFOX,
self.archive_dir,
]
main(args)
# XXX we want a bunch of assertions here to check
# that the archives dir gets filled correctly
finally:
if os.environ["FXA_USERNAME"] == "me":
del os.environ["FXA_USERNAME"]
if os.environ["FXA_PASSWORD"] == "password":
del os.environ["FXA_PASSWORD"]

Просмотреть файл

@ -21,6 +21,7 @@ from datetime import datetime
import requests
from requests.exceptions import ConnectionError
from requests.packages.urllib3.util.retry import Retry
from condprof import progress
@ -70,7 +71,7 @@ class NullLogger:
traceback.print_exc(file=sys.stdout)
def get_logger():
def _get_logger():
global _LOGGER
if _LOGGER is not None:
return _LOGGER
@ -95,12 +96,12 @@ def get_logger():
def LOG(msg):
msg = "[%s] %s" % (datetime.now().isoformat(), msg)
get_logger().msg(msg)
_get_logger().msg(obfuscate(msg)[1])
def ERROR(msg):
msg = "[%s] %s" % (datetime.now().isoformat(), msg)
get_logger().error(msg)
_get_logger().error(obfuscate(msg)[1])
def fresh_profile(profile, customization_data):
@ -111,12 +112,15 @@ def fresh_profile(profile, customization_data):
new_profile = create_profile(app="firefox")
prefs = customization_data["prefs"]
prefs.update(DEFAULT_PREFS)
LOG("Setting prefs %s" % str(prefs.items()))
new_profile.set_preferences(prefs)
extensions = []
for name, url in customization_data["addons"].items():
LOG("Downloading addon %s" % name)
extension = download_file(url)
extensions.append(extension)
LOG("Installing addons")
new_profile.addons.install(extensions, unpack=True)
new_profile.addons.install(extensions)
shutil.copytree(new_profile.profile, profile)
return profile
@ -279,12 +283,12 @@ def latest_nightly(binary=None):
def write_yml_file(yml_file, yml_data):
get_logger().info("writing %s to %s" % (yml_data, yml_file))
LOG("writing %s to %s" % (yml_data, yml_file))
try:
with open(yml_file, "w") as outfile:
yaml.dump(yml_data, outfile, default_flow_style=False)
except Exception as e:
get_logger().critical("failed to write yaml file, exeption: %s" % e)
ERROR("failed to write yaml file, exeption: %s" % e)
def get_version(firefox):
@ -357,3 +361,70 @@ class BaseEnv:
def stop_browser(self):
pass
_URL = (
"{0}/secrets/v1/secret/project"
"{1}releng{1}gecko{1}build{1}level-{2}{1}conditioned-profiles"
)
_DEFAULT_SERVER = "https://firefox-ci-tc.services.mozilla.com"
def get_tc_secret():
if "TASK_ID" not in os.environ:
raise OSError("Not running in Taskcluster")
session = requests.Session()
retry = Retry(total=5, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504])
http_adapter = requests.adapters.HTTPAdapter(max_retries=retry)
session.mount("https://", http_adapter)
session.mount("http://", http_adapter)
secrets_url = _URL.format(
os.environ.get("TASKCLUSTER_PROXY_URL", _DEFAULT_SERVER),
"%2F",
os.environ.get("MOZ_SCM_LEVEL", "1"),
)
res = session.get(secrets_url)
res.raise_for_status()
return res.json()["secret"]
_CACHED = {}
def obfuscate(text):
username, password = get_credentials()
if username is None:
return False, text
if username not in text and password not in text:
return False, text
text = text.replace(password, "<PASSWORD>")
text = text.replace(username, "<USERNAME>")
return True, text
def obfuscate_file(path):
with open(path) as f:
data = f.read()
hit, data = obfuscate(data)
if not hit:
return
with open(path, "w") as f:
f.write(data)
def get_credentials():
if "creds" in _CACHED:
return _CACHED["creds"]
password = os.environ.get("FXA_PASSWORD")
username = os.environ.get("FXA_USERNAME")
if username is None or password is None:
if "TASK_ID" not in os.environ:
return None, None
try:
secret = get_tc_secret()
except Exception:
return None, None
password = secret["password"]
username = secret["username"]
_CACHED["creds"] = username, password
return username, password