[android] Push only updated files in PushIfNeeded when few files have changed.
BUG=263857 TEST=None NOTRY=True Review URL: https://chromiumcodereview.appspot.com/21307002 git-svn-id: http://src.chromium.org/svn/trunk/src/build@216208 4ff67af0-8c30-449e-8e8b-ad334ec8d88c
This commit is contained in:
craigdh@chromium.org
Родитель
4a8abc7e6f
Коммит
266cfb727a
|
|
@ -184,14 +184,15 @@ def _GetFilesFromRecursiveLsOutput(path, ls_output, re_file, utc_offset=None):
|
|||
return files
|
||||
|
||||
|
||||
def _ComputeFileListHash(md5sum_output):
|
||||
def _ParseMd5SumOutput(md5sum_output):
|
||||
"""Returns a list of tuples from the provided md5sum output.
|
||||
|
||||
Args:
|
||||
md5sum_output: output directly from md5sum binary.
|
||||
|
||||
Returns:
|
||||
List of namedtuples (hash, path).
|
||||
List of namedtuples with attributes |hash| and |path|, where |path| is the
|
||||
absolute path to the file with an Md5Sum of |hash|.
|
||||
"""
|
||||
HashAndPath = collections.namedtuple('HashAndPath', ['hash', 'path'])
|
||||
split_lines = [line.split(' ') for line in md5sum_output]
|
||||
|
|
@ -418,7 +419,8 @@ class AndroidCommands(object):
|
|||
# Check if package is already installed and up to date.
|
||||
if package_name:
|
||||
installed_apk_path = self.GetApplicationPath(package_name)
|
||||
if installed_apk_path and self.CheckMd5Sum(apk_path, installed_apk_path):
|
||||
if (installed_apk_path and
|
||||
not self.GetFilesChanged(apk_path, installed_apk_path)):
|
||||
logging.info('Skipped install: identical %s APK already installed' %
|
||||
package_name)
|
||||
return
|
||||
|
|
@ -738,15 +740,16 @@ class AndroidCommands(object):
|
|||
"""
|
||||
self.RunShellCommand('input keyevent %d' % keycode)
|
||||
|
||||
def CheckMd5Sum(self, local_path, device_path):
|
||||
"""Compares the md5sum of a local path against a device path.
|
||||
def _RunMd5Sum(self, host_path, device_path):
|
||||
"""Gets the md5sum of a host path and device path.
|
||||
|
||||
Args:
|
||||
local_path: Path (file or directory) on the host.
|
||||
host_path: Path (file or directory) on the host.
|
||||
device_path: Path on the device.
|
||||
|
||||
Returns:
|
||||
True if the md5sums match.
|
||||
A tuple containing lists of the host and device md5sum results as
|
||||
created by _ParseMd5SumOutput().
|
||||
"""
|
||||
if not self._md5sum_build_dir:
|
||||
default_build_type = os.environ.get('BUILD_TYPE', 'Debug')
|
||||
|
|
@ -763,73 +766,112 @@ class AndroidCommands(object):
|
|||
|
||||
cmd = (MD5SUM_LD_LIBRARY_PATH + ' ' + self._util_wrapper + ' ' +
|
||||
MD5SUM_DEVICE_PATH + ' ' + device_path)
|
||||
device_hash_tuples = _ComputeFileListHash(
|
||||
device_hash_tuples = _ParseMd5SumOutput(
|
||||
self.RunShellCommand(cmd, timeout_time=2 * 60))
|
||||
assert os.path.exists(local_path), 'Local path not found %s' % local_path
|
||||
assert os.path.exists(host_path), 'Local path not found %s' % host_path
|
||||
md5sum_output = cmd_helper.GetCmdOutput(
|
||||
['%s/md5sum_bin_host' % self._md5sum_build_dir, local_path])
|
||||
host_hash_tuples = _ComputeFileListHash(md5sum_output.splitlines())
|
||||
['%s/md5sum_bin_host' % self._md5sum_build_dir, host_path])
|
||||
host_hash_tuples = _ParseMd5SumOutput(md5sum_output.splitlines())
|
||||
return (host_hash_tuples, device_hash_tuples)
|
||||
|
||||
def GetFilesChanged(self, host_path, device_path):
|
||||
"""Compares the md5sum of a host path against a device path.
|
||||
|
||||
Note: Ignores extra files on the device.
|
||||
|
||||
Args:
|
||||
host_path: Path (file or directory) on the host.
|
||||
device_path: Path on the device.
|
||||
|
||||
Returns:
|
||||
A list of tuples of the form (host_path, device_path) for files whose
|
||||
md5sums do not match.
|
||||
"""
|
||||
host_hash_tuples, device_hash_tuples = self._RunMd5Sum(
|
||||
host_path, device_path)
|
||||
|
||||
# Ignore extra files on the device.
|
||||
if len(device_hash_tuples) > len(host_hash_tuples):
|
||||
host_files = [os.path.relpath(os.path.normpath(p.path),
|
||||
os.path.normpath(local_path)) for p in host_hash_tuples]
|
||||
os.path.normpath(host_path)) for p in host_hash_tuples]
|
||||
|
||||
def _host_has(fname):
|
||||
def HostHas(fname):
|
||||
return any(path in fname for path in host_files)
|
||||
|
||||
hashes_on_device = [h.hash for h in device_hash_tuples if
|
||||
_host_has(h.path)]
|
||||
else:
|
||||
hashes_on_device = [h.hash for h in device_hash_tuples]
|
||||
device_hash_tuples = [h for h in device_hash_tuples if HostHas(h.path)]
|
||||
|
||||
# Compare md5sums between host and device files.
|
||||
hashes_on_host = [h.hash for h in host_hash_tuples]
|
||||
hashes_on_device.sort()
|
||||
hashes_on_host.sort()
|
||||
return hashes_on_device == hashes_on_host
|
||||
# Constructs the target device path from a given host path. Don't use when
|
||||
# only a single file is given as the base name given in device_path may
|
||||
# differ from that in host_path.
|
||||
def HostToDevicePath(host_file_path):
|
||||
return os.path.join(os.path.dirname(device_path), os.path.relpath(
|
||||
host_file_path, os.path.dirname(os.path.normpath(host_path))))
|
||||
|
||||
def PushIfNeeded(self, local_path, device_path):
|
||||
"""Pushes |local_path| to |device_path|.
|
||||
device_hashes = [h.hash for h in device_hash_tuples]
|
||||
return [(t.path, HostToDevicePath(t.path) if os.path.isdir(host_path) else
|
||||
device_path)
|
||||
for t in host_hash_tuples if t.hash not in device_hashes]
|
||||
|
||||
def PushIfNeeded(self, host_path, device_path):
|
||||
"""Pushes |host_path| to |device_path|.
|
||||
|
||||
Works for files and directories. This method skips copying any paths in
|
||||
|test_data_paths| that already exist on the device with the same hash.
|
||||
|
||||
All pushed files can be removed by calling RemovePushedFiles().
|
||||
"""
|
||||
assert os.path.exists(local_path), 'Local path not found %s' % local_path
|
||||
size = int(cmd_helper.GetCmdOutput(['du', '-sb', local_path]).split()[0])
|
||||
MAX_INDIVIDUAL_PUSHES = 50
|
||||
assert os.path.exists(host_path), 'Local path not found %s' % host_path
|
||||
|
||||
def GetHostSize(path):
|
||||
return int(cmd_helper.GetCmdOutput(['du', '-sb', path]).split()[0])
|
||||
|
||||
size = GetHostSize(host_path)
|
||||
self._pushed_files.append(device_path)
|
||||
self._potential_push_size += size
|
||||
|
||||
if self.CheckMd5Sum(local_path, device_path):
|
||||
changed_files = self.GetFilesChanged(host_path, device_path)
|
||||
if not changed_files:
|
||||
return
|
||||
|
||||
self._actual_push_size += size
|
||||
# They don't match, so remove everything first and then create it.
|
||||
if os.path.isdir(local_path):
|
||||
self.RunShellCommand('rm -r %s' % device_path, timeout_time=2 * 60)
|
||||
self.RunShellCommand('mkdir -p %s' % device_path)
|
||||
def Push(host, device):
|
||||
# NOTE: We can't use adb_interface.Push() because it hardcodes a timeout
|
||||
# of 60 seconds which isn't sufficient for a lot of users of this method.
|
||||
push_command = 'push %s %s' % (host, device)
|
||||
self._LogShell(push_command)
|
||||
|
||||
# NOTE: We can't use adb_interface.Push() because it hardcodes a timeout of
|
||||
# 60 seconds which isn't sufficient for a lot of users of this method.
|
||||
push_command = 'push %s %s' % (local_path, device_path)
|
||||
self._LogShell(push_command)
|
||||
# Retry push with increasing backoff if the device is busy.
|
||||
retry = 0
|
||||
while True:
|
||||
output = self._adb.SendCommand(push_command, timeout_time=30 * 60)
|
||||
if _HasAdbPushSucceeded(output):
|
||||
return
|
||||
if retry < 3:
|
||||
retry += 1
|
||||
wait_time = 5 * retry
|
||||
logging.error('Push failed, retrying in %d seconds: %s' %
|
||||
(wait_time, output))
|
||||
time.sleep(wait_time)
|
||||
else:
|
||||
raise Exception('Push failed: %s' % output)
|
||||
|
||||
# Retry push with increasing backoff if the device is busy.
|
||||
retry = 0
|
||||
while True:
|
||||
output = self._adb.SendCommand(push_command, timeout_time=30 * 60)
|
||||
if _HasAdbPushSucceeded(output):
|
||||
return
|
||||
if 'resource busy' in output and retry < 3:
|
||||
retry += 1
|
||||
wait_time = 5 * retry
|
||||
logging.error('Push failed, retrying in %d seconds: %s' %
|
||||
(wait_time, output))
|
||||
time.sleep(wait_time)
|
||||
else:
|
||||
raise Exception('Push failed: %s' % output)
|
||||
diff_size = 0
|
||||
if len(changed_files) <= MAX_INDIVIDUAL_PUSHES:
|
||||
diff_size = sum(GetHostSize(f[0]) for f in changed_files)
|
||||
|
||||
# TODO(craigdh): Replace this educated guess with a heuristic that
|
||||
# approximates the push time for each method.
|
||||
if len(changed_files) > MAX_INDIVIDUAL_PUSHES or diff_size > 0.5 * size:
|
||||
# We're pushing everything, remove everything first and then create it.
|
||||
self._actual_push_size += size
|
||||
if os.path.isdir(host_path):
|
||||
self.RunShellCommand('rm -r %s' % device_path, timeout_time=2 * 60)
|
||||
self.RunShellCommand('mkdir -p %s' % device_path)
|
||||
Push(host_path, device_path)
|
||||
else:
|
||||
for f in changed_files:
|
||||
Push(f[0], f[1])
|
||||
self._actual_push_size += diff_size
|
||||
|
||||
def GetPushSizeInfo(self):
|
||||
"""Get total size of pushes to the device done via PushIfNeeded()
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче