зеркало из https://github.com/mozilla/gecko-dev.git
281 строка
10 KiB
Python
Executable File
281 строка
10 KiB
Python
Executable File
#!/usr/bin/python
|
|
# vim:sts=2 sw=2
|
|
import sys
|
|
import shutil
|
|
import urllib2
|
|
import urllib
|
|
import os
|
|
import traceback
|
|
import time
|
|
if os.name == 'nt':
|
|
from win32file import RemoveDirectory, DeleteFile, \
|
|
GetFileAttributesW, SetFileAttributesW, \
|
|
FILE_ATTRIBUTE_NORMAL, FILE_ATTRIBUTE_DIRECTORY
|
|
from win32api import FindFiles
|
|
|
|
clobber_suffix = '.deleteme'
|
|
|
|
|
|
def ts_to_str(ts):
|
|
if ts is None:
|
|
return None
|
|
return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ts))
|
|
|
|
|
|
def write_file(ts, fn):
|
|
assert isinstance(ts, int)
|
|
f = open(fn, "w")
|
|
f.write(str(ts))
|
|
f.close()
|
|
|
|
|
|
def read_file(fn):
|
|
if not os.path.exists(fn):
|
|
return None
|
|
|
|
data = open(fn).read().strip()
|
|
try:
|
|
return int(data)
|
|
except ValueError:
|
|
return None
|
|
|
|
def rmdirRecursiveWindows(dir):
|
|
"""Windows-specific version of rmdirRecursive that handles
|
|
path lengths longer than MAX_PATH.
|
|
"""
|
|
|
|
dir = os.path.realpath(dir)
|
|
# Make sure directory is writable
|
|
SetFileAttributesW('\\\\?\\' + dir, FILE_ATTRIBUTE_NORMAL)
|
|
|
|
for ffrec in FindFiles('\\\\?\\' + dir + '\\*.*'):
|
|
file_attr = ffrec[0]
|
|
name = ffrec[8]
|
|
if name == '.' or name == '..':
|
|
continue
|
|
full_name = os.path.join(dir, name)
|
|
|
|
if file_attr & FILE_ATTRIBUTE_DIRECTORY:
|
|
rmdirRecursiveWindows(full_name)
|
|
else:
|
|
SetFileAttributesW('\\\\?\\' + full_name, FILE_ATTRIBUTE_NORMAL)
|
|
DeleteFile('\\\\?\\' + full_name)
|
|
RemoveDirectory('\\\\?\\' + dir)
|
|
|
|
def rmdirRecursive(dir):
|
|
"""This is a replacement for shutil.rmtree that works better under
|
|
windows. Thanks to Bear at the OSAF for the code.
|
|
(Borrowed from buildbot.slave.commands)"""
|
|
if os.name == 'nt':
|
|
rmdirRecursiveWindows(dir)
|
|
return
|
|
|
|
if not os.path.exists(dir):
|
|
# This handles broken links
|
|
if os.path.islink(dir):
|
|
os.remove(dir)
|
|
return
|
|
|
|
if os.path.islink(dir):
|
|
os.remove(dir)
|
|
return
|
|
|
|
# Verify the directory is read/write/execute for the current user
|
|
os.chmod(dir, 0700)
|
|
|
|
for name in os.listdir(dir):
|
|
full_name = os.path.join(dir, name)
|
|
# on Windows, if we don't have write permission we can't remove
|
|
# the file/directory either, so turn that on
|
|
if os.name == 'nt':
|
|
if not os.access(full_name, os.W_OK):
|
|
# I think this is now redundant, but I don't have an NT
|
|
# machine to test on, so I'm going to leave it in place
|
|
# -warner
|
|
os.chmod(full_name, 0600)
|
|
|
|
if os.path.isdir(full_name):
|
|
rmdirRecursive(full_name)
|
|
else:
|
|
# Don't try to chmod links
|
|
if not os.path.islink(full_name):
|
|
os.chmod(full_name, 0700)
|
|
os.remove(full_name)
|
|
os.rmdir(dir)
|
|
|
|
|
|
def do_clobber(dir, dryrun=False, skip=None):
|
|
try:
|
|
for f in os.listdir(dir):
|
|
if skip is not None and f in skip:
|
|
print "Skipping", f
|
|
continue
|
|
clobber_path = f + clobber_suffix
|
|
if os.path.isfile(f):
|
|
print "Removing", f
|
|
if not dryrun:
|
|
if os.path.exists(clobber_path):
|
|
os.unlink(clobber_path)
|
|
# Prevent repeated moving.
|
|
if f.endswith(clobber_suffix):
|
|
os.unlink(f)
|
|
else:
|
|
shutil.move(f, clobber_path)
|
|
os.unlink(clobber_path)
|
|
elif os.path.isdir(f):
|
|
print "Removing %s/" % f
|
|
if not dryrun:
|
|
if os.path.exists(clobber_path):
|
|
rmdirRecursive(clobber_path)
|
|
# Prevent repeated moving.
|
|
if f.endswith(clobber_suffix):
|
|
rmdirRecursive(f)
|
|
else:
|
|
shutil.move(f, clobber_path)
|
|
rmdirRecursive(clobber_path)
|
|
except:
|
|
print "Couldn't clobber properly, bailing out."
|
|
sys.exit(1)
|
|
|
|
|
|
def getClobberDates(clobberURL, branch, buildername, builddir, slave, master):
|
|
params = dict(branch=branch, buildername=buildername,
|
|
builddir=builddir, slave=slave, master=master)
|
|
url = "%s?%s" % (clobberURL, urllib.urlencode(params))
|
|
print "Checking clobber URL: %s" % url
|
|
# The timeout arg was added to urlopen() at Python 2.6
|
|
# Deprecate this test when esr17 reaches EOL
|
|
if sys.version_info[:2] < (2, 6):
|
|
data = urllib2.urlopen(url).read().strip()
|
|
else:
|
|
data = urllib2.urlopen(url, timeout=30).read().strip()
|
|
|
|
retval = {}
|
|
try:
|
|
for line in data.split("\n"):
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
builddir, builder_time, who = line.split(":")
|
|
builder_time = int(builder_time)
|
|
retval[builddir] = (builder_time, who)
|
|
return retval
|
|
except ValueError:
|
|
print "Error parsing response from server"
|
|
print data
|
|
raise
|
|
|
|
if __name__ == "__main__":
|
|
from optparse import OptionParser
|
|
parser = OptionParser(
|
|
"%prog [options] clobberURL branch buildername builddir slave master")
|
|
parser.add_option("-n", "--dry-run", dest="dryrun", action="store_true",
|
|
default=False, help="don't actually delete anything")
|
|
parser.add_option("-t", "--periodic", dest="period", type="float",
|
|
default=None, help="hours between periodic clobbers")
|
|
parser.add_option('-s', '--skip', help='do not delete this file/directory',
|
|
action='append', dest='skip', default=['last-clobber'])
|
|
parser.add_option('-d', '--dir', help='clobber this directory',
|
|
dest='dir', default='.', type='string')
|
|
parser.add_option('-v', '--verbose', help='be more verbose',
|
|
dest='verbose', action='store_true', default=False)
|
|
|
|
options, args = parser.parse_args()
|
|
if len(args) != 6:
|
|
parser.error("Incorrect number of arguments")
|
|
|
|
if options.period:
|
|
periodicClobberTime = options.period * 3600
|
|
else:
|
|
periodicClobberTime = None
|
|
|
|
clobberURL, branch, builder, my_builddir, slave, master = args
|
|
|
|
try:
|
|
server_clobber_dates = getClobberDates(
|
|
clobberURL, branch, builder, my_builddir, slave, master)
|
|
except:
|
|
if options.verbose:
|
|
traceback.print_exc()
|
|
print "Error contacting server"
|
|
sys.exit(1)
|
|
|
|
if options.verbose:
|
|
print "Server gave us", server_clobber_dates
|
|
|
|
now = int(time.time())
|
|
|
|
# Add ourself to the server_clobber_dates if it's not set
|
|
# This happens when this slave has never been clobbered
|
|
if my_builddir not in server_clobber_dates:
|
|
server_clobber_dates[my_builddir] = None, ""
|
|
|
|
root_dir = os.path.abspath(options.dir)
|
|
|
|
for builddir, (server_clobber_date, who) in server_clobber_dates.items():
|
|
builder_dir = os.path.join(root_dir, builddir)
|
|
if not os.path.isdir(builder_dir):
|
|
print "%s doesn't exist, skipping" % builder_dir
|
|
continue
|
|
os.chdir(builder_dir)
|
|
|
|
our_clobber_date = read_file("last-clobber")
|
|
|
|
clobber = False
|
|
clobberType = None
|
|
|
|
print "%s:Our last clobber date: " % builddir, ts_to_str(our_clobber_date)
|
|
print "%s:Server clobber date: " % builddir, ts_to_str(server_clobber_date)
|
|
|
|
# If we don't have a last clobber date, then this is probably a fresh build.
|
|
# We should only do a forced server clobber if we know when our last clobber
|
|
# was, and if the server date is more recent than that.
|
|
if server_clobber_date is not None and our_clobber_date is not None:
|
|
# If the server is giving us a clobber date, compare the server's idea of
|
|
# the clobber date to our last clobber date
|
|
if server_clobber_date > our_clobber_date:
|
|
# If the server's clobber date is greater than our last clobber date,
|
|
# then we should clobber.
|
|
clobber = True
|
|
clobberType = "forced"
|
|
# We should also update our clobber date to match the server's
|
|
our_clobber_date = server_clobber_date
|
|
if who:
|
|
print "%s:Server is forcing a clobber, initiated by %s" % (builddir, who)
|
|
else:
|
|
print "%s:Server is forcing a clobber" % builddir
|
|
|
|
if not clobber:
|
|
# Disable periodic clobbers for builders that aren't my_builddir
|
|
if builddir != my_builddir:
|
|
continue
|
|
|
|
# Next, check if more than the periodicClobberTime period has passed since
|
|
# our last clobber
|
|
if our_clobber_date is None:
|
|
# We've never been clobbered
|
|
# Set our last clobber time to now, so that we'll clobber
|
|
# properly after periodicClobberTime
|
|
clobberType = "purged"
|
|
our_clobber_date = now
|
|
write_file(our_clobber_date, "last-clobber")
|
|
elif periodicClobberTime and now > our_clobber_date + periodicClobberTime:
|
|
# periodicClobberTime has passed since our last clobber
|
|
clobber = True
|
|
clobberType = "periodic"
|
|
# Update our clobber date to now
|
|
our_clobber_date = now
|
|
print "%s:More than %s seconds have passed since our last clobber" % (builddir, periodicClobberTime)
|
|
|
|
if clobber:
|
|
# Finally, perform a clobber if we're supposed to
|
|
print "%s:Clobbering..." % builddir
|
|
do_clobber(builder_dir, options.dryrun, options.skip)
|
|
write_file(our_clobber_date, "last-clobber")
|
|
|
|
# If this is the build dir for the current job, display the clobber type in TBPL.
|
|
# Note in the case of purged clobber, we output the clobber type even though no
|
|
# clobber was performed this time.
|
|
if clobberType and builddir == my_builddir:
|
|
print "TinderboxPrint: %s clobber" % clobberType
|