mozilla
/
pjs
зеркало из https://github.com/mozilla/pjs.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Backed out changeset f2c04d62dc9c

This commit is contained in:
Kyle Huey 2010-08-09 17:55:33 -07:00
Родитель 6670578a98
Коммит ff48898ceb
17 изменённых файлов: 81 добавлений и 426 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

5
build/pymake/.hg_archival.txt
Просмотреть файл

@ -1,5 +1,2 @@
repo: f5ab154deef2ffa97f1b2139589ae4a1962090a4
node: e50be3234e397322410b29a9e226365e23cb322e
branch: default
latesttag: null
latesttagdistance: 255
node: 7ae0b4af32617677698f9de3ab76bcb154bbf085

10
build/pymake/make.py
Просмотреть файл

@ -10,10 +10,8 @@ import sys, os
import pymake.command, pymake.process
import gc
gc.disable()
if __name__ == '__main__':
gc.disable()
pymake.command.main(sys.argv[1:], os.environ, os.getcwd(), cb=sys.exit)
pymake.process.ParallelContext.spin()
assert False, "Not reached"
pymake.command.main(sys.argv[1:], os.environ, os.getcwd(), cb=sys.exit)
pymake.process.ParallelContext.spin()
assert False, "Not reached"

76
build/pymake/pymake/builtins.py
Просмотреть файл

@ -1,68 +1,10 @@
# Basic commands implemented in Python
import sys, os, shutil, time
from getopt import getopt, GetoptError
"""
Implicit variables; perhaps in the future this will also include some implicit
rules, at least match-anything cancellation rules.
"""
from process import PythonException
__all__ = ["rm", "sleep", "touch"]
def rm(args):
"""
Emulate most of the behavior of rm(1).
Only supports the -r (--recursive) and -f (--force) arguments.
"""
try:
opts, args = getopt(args, "rRf", ["force", "recursive"])
except GetoptError, e:
raise PythonException, ("rm: %s" % e, 1)
force = False
recursive = False
for o, a in opts:
if o in ('-f', '--force'):
force = True
elif o in ('-r', '-R', '--recursive'):
recursive = True
for f in args:
if os.path.isdir(f):
if not recursive:
raise PythonException, ("rm: cannot remove '%s': Is a directory" % f, 1)
else:
shutil.rmtree(f, force)
elif os.path.exists(f):
try:
os.unlink(f)
except:
if not force:
raise PythonException, ("rm: failed to remove '%s': %s" % (f, sys.exc_info()[0]), 1)
elif not force:
raise PythonException, ("rm: cannot remove '%s': No such file or directory" % f, 1)
def sleep(args):
"""
Emulate the behavior of sleep(1).
"""
total = 0
values = {'s': 1, 'm': 60, 'h': 3600, 'd': 86400}
for a in args:
multiplier = 1
for k, v in values.iteritems():
if a.endswith(k):
a = a[:-1]
multiplier = v
break
try:
f = float(a)
total += f * multiplier
except ValueError:
raise PythonException, ("sleep: invalid time interval '%s'" % a, 1)
time.sleep(total)
def touch(args):
"""
Emulate the behavior of touch(1).
"""
for f in args:
if os.path.exists(f):
os.utime(f, None)
else:
open(f, 'w').close()
variables = {
'RM': 'rm -f',
'.LIBPATTERNS': 'lib%.so lib%.a',
'.PYMAKE': '1',
}

9
build/pymake/pymake/command.py
Просмотреть файл

@ -106,8 +106,7 @@ class _MakeContext(object):
makeoverrides=self.overrides,
workdir=self.workdir,
context=self.context, env=self.env, makelevel=self.makelevel,
targets=self.targets, keepgoing=self.options.keepgoing,
silent=self.options.silent)
targets=self.targets, keepgoing=self.options.keepgoing)
self.restarts += 1
@ -187,8 +186,6 @@ def main(args, env, cwd, cb):
dest="printdir")
op.add_option('--no-print-directory', action="store_false",
dest="printdir", default=True)
op.add_option('-s', '--silent', action="store_true",
dest="silent", default=False)
options, arguments1 = op.parse_args(parsemakeflags(env))
options, arguments2 = op.parse_args(args, values=options)
@ -211,10 +208,6 @@ def main(args, env, cwd, cb):
if options.printdir:
shortflags.append('w')
if options.silent:
shortflags.append('s')
options.printdir = False
loglevel = logging.WARNING
if options.verbose:
loglevel = logging.DEBUG

62
build/pymake/pymake/data.py
Просмотреть файл

@ -3,7 +3,7 @@ A representation of makefile data structures.
"""
import logging, re, os, sys
import parserdata, parser, functions, process, util, implicit
import parserdata, parser, functions, process, util, builtins
from cStringIO import StringIO
_log = logging.getLogger('pymake.data')
@ -1142,18 +1142,17 @@ def splitcommand(command):
def findmodifiers(command):
"""
Find any of +-@% prefixed on the command.
@returns (command, isHidden, isRecursive, ignoreErrors, isNative)
Find any of +-@ prefixed on the command.
@returns (command, isHidden, isRecursive, ignoreErrors)
"""
isHidden = False
isRecursive = False
ignoreErrors = False
isNative = False
realcommand = command.lstrip(' \t\n@+-%')
realcommand = command.lstrip(' \t\n@+-')
modset = set(command[:-len(realcommand)])
return realcommand, '@' in modset, '+' in modset, '-' in modset, '%' in modset
return realcommand, '@' in modset, '+' in modset, '-' in modset
class _CommandWrapper(object):
def __init__(self, cline, ignoreErrors, loc, context, **kwargs):
@ -1174,32 +1173,6 @@ class _CommandWrapper(object):
self.usercb = cb
process.call(self.cline, loc=self.loc, cb=self._cb, context=self.context, **self.kwargs)
class _NativeWrapper(_CommandWrapper):
def __init__(self, cline, ignoreErrors, loc, context,
pycommandpath, **kwargs):
_CommandWrapper.__init__(self, cline, ignoreErrors, loc, context,
**kwargs)
# get the module and method to call
parts, badchar = process.clinetoargv(cline)
if parts is None:
raise DataError("native command '%s': shell metacharacter '%s' in command line" % (cline, badchar), self.loc)
if len(parts) < 2:
raise DataError("native command '%s': no method name specified" % cline, self.loc)
if pycommandpath:
self.pycommandpath = re.split('[%s\s]+' % os.pathsep,
pycommandpath)
else:
self.pycommandpath = None
self.module = parts[0]
self.method = parts[1]
self.cline_list = parts[2:]
def __call__(self, cb):
self.usercb = cb
process.call_native(self.module, self.method, self.cline_list,
loc=self.loc, cb=self._cb, context=self.context,
pycommandpath=self.pycommandpath, **self.kwargs)
def getcommandsforrule(rule, target, makefile, prerequisites, stem):
v = Variables(parent=target.variables)
setautomaticvariables(v, makefile, target, prerequisites)
@ -1211,22 +1184,13 @@ def getcommandsforrule(rule, target, makefile, prerequisites, stem):
for c in rule.commands:
cstring = c.resolvestr(makefile, v)
for cline in splitcommand(cstring):
cline, isHidden, isRecursive, ignoreErrors, isNative = findmodifiers(cline)
if isHidden or makefile.silent:
cline, isHidden, isRecursive, ignoreErrors = findmodifiers(cline)
if isHidden:
echo = None
else:
echo = "%s$ %s" % (c.loc, cline)
if not isNative:
yield _CommandWrapper(cline, ignoreErrors=ignoreErrors, env=env, cwd=makefile.workdir, loc=c.loc, context=makefile.context,
echo=echo)
else:
f, s, e = v.get("PYCOMMANDPATH", True)
if e:
e = e.resolvestr(makefile, v, ["PYCOMMANDPATH"])
yield _NativeWrapper(cline, ignoreErrors=ignoreErrors,
env=env, cwd=makefile.workdir,
loc=c.loc, context=makefile.context,
echo=echo, pycommandpath=e)
yield _CommandWrapper(cline, ignoreErrors=ignoreErrors, env=env, cwd=makefile.workdir, loc=c.loc, context=makefile.context,
echo=echo)
class Rule(object):
"""
@ -1371,8 +1335,7 @@ class Makefile(object):
def __init__(self, workdir=None, env=None, restarts=0, make=None,
makeflags='', makeoverrides='',
makelevel=0, context=None, targets=(), keepgoing=False,
silent=False):
makelevel=0, context=None, targets=(), keepgoing=False):
self.defaulttarget = None
if env is None:
@ -1386,7 +1349,6 @@ class Makefile(object):
self.exportedvars = {}
self._targets = {}
self.keepgoing = keepgoing
self.silent = silent
self._patternvariables = [] # of (pattern, variables)
self.implicitrules = []
self.parsingfinished = False
@ -1406,8 +1368,6 @@ class Makefile(object):
self.variables.set('MAKE_RESTARTS', Variables.FLAVOR_SIMPLE,
Variables.SOURCE_AUTOMATIC, restarts > 0 and str(restarts) or '')
self.variables.set('.PYMAKE', Variables.FLAVOR_SIMPLE,
Variables.SOURCE_MAKEFILE, "1")
if make is not None:
self.variables.set('MAKE', Variables.FLAVOR_SIMPLE,
Variables.SOURCE_MAKEFILE, make)
@ -1432,7 +1392,7 @@ class Makefile(object):
self.variables.set('MAKECMDGOALS', Variables.FLAVOR_SIMPLE,
Variables.SOURCE_AUTOMATIC, ' '.join(targets))
for vname, val in implicit.variables.iteritems():
for vname, val in builtins.variables.iteritems():
self.variables.set(vname,
Variables.FLAVOR_SIMPLE,
Variables.SOURCE_IMPLICIT, val)

13
build/pymake/pymake/implicit.py
Просмотреть файл

@ -1,13 +0,0 @@
"""
Implicit variables; perhaps in the future this will also include some implicit
rules, at least match-anything cancellation rules.
"""
variables = {
'RM': '%pymake.builtins rm -f',
'SLEEP': '%pymake.builtins sleep',
'TOUCH': '%pymake.builtins touch',
'.LIBPATTERNS': 'lib%.so lib%.a',
'.PYMAKE': '1',
}

222
build/pymake/pymake/process.py
Просмотреть файл

@ -3,9 +3,7 @@ Skipping shell invocations is good, when possible. This wrapper around subproces
parsing command lines into argv and making sure that no shell magic is being used.
"""
#TODO: ship pyprocessing?
from multiprocessing import Pool, Condition
import subprocess, shlex, re, logging, sys, traceback, os, imp
import subprocess, shlex, re, logging, sys, traceback, os
import command, util
if sys.platform=='win32':
import win32process
@ -81,11 +79,6 @@ def call(cline, env, cwd, loc, cb, context, echo):
context.call(argv, executable=executable, shell=False, env=env, cwd=cwd, cb=cb, echo=echo)
def call_native(module, method, argv, env, cwd, loc, cb, context, echo,
pycommandpath=None):
context.call_native(module, method, argv, env=env, cwd=cwd, cb=cb,
echo=echo, pycommandpath=pycommandpath)
def statustoresult(status):
"""
Convert the status returned from waitpid into a prettier numeric result.
@ -96,127 +89,17 @@ def statustoresult(status):
return status >>8
class Job(object):
"""
A single job to be executed on the process pool.
"""
done = False # set to true when the job completes
def __init__(self):
self.exitcode = -127
def notify(self, condition, result):
condition.acquire()
self.done = True
self.exitcode = result
condition.notify()
condition.release()
def get_callback(self, condition):
return lambda result: self.notify(condition, result)
class PopenJob(Job):
"""
A job that executes a command using subprocess.Popen.
"""
def __init__(self, argv, executable, shell, env, cwd):
Job.__init__(self)
self.argv = argv
self.executable = executable
self.shell = shell
self.env = env
self.cwd = cwd
def run(self):
try:
p = subprocess.Popen(self.argv, executable=self.executable, shell=self.shell, env=self.env, cwd=self.cwd)
return p.wait()
except OSError, e:
print >>sys.stderr, e
return -127
class PythonException(Exception):
def __init__(self, message, exitcode):
Exception.__init__(self)
self.message = message
self.exitcode = exitcode
def __str__(self):
return self.message
def load_module_recursive(module, path):
"""
Emulate the behavior of __import__, but allow
passing a custom path to search for modules.
"""
bits = module.split('.')
for i, bit in enumerate(bits):
dotname = '.'.join(bits[:i+1])
try:
f, path, desc = imp.find_module(bit, path)
m = imp.load_module(dotname, f, path, desc)
if f is None:
path = m.__path__
except ImportError:
return
class PythonJob(Job):
"""
A job that calls a Python method.
"""
def __init__(self, module, method, argv, env, cwd, pycommandpath=None):
self.module = module
self.method = method
self.argv = argv
self.env = env
self.cwd = cwd
self.pycommandpath = pycommandpath or []
def run(self):
oldenv = os.environ
try:
os.chdir(self.cwd)
os.environ = self.env
if self.module not in sys.modules:
load_module_recursive(self.module,
sys.path + self.pycommandpath)
if self.module not in sys.modules:
print >>sys.stderr, "No module named '%s'" % self.module
return -127
m = sys.modules[self.module]
if self.method not in m.__dict__:
print >>sys.stderr, "No method named '%s' in module %s" % (method, module)
return -127
m.__dict__[self.method](self.argv)
except PythonException, e:
print >>sys.stderr, e
return e.exitcode
except:
print >>sys.stderr, sys.exc_info()[1]
return -127
finally:
os.environ = oldenv
return 0
def job_runner(job):
"""
Run a job. Called in a Process pool.
"""
return job.run()
class ParallelContext(object):
"""
Manages the parallel execution of processes.
"""
_allcontexts = set()
_condition = Condition()
def __init__(self, jcount):
self.jcount = jcount
self.exit = False
self.pool = Pool(processes=jcount)
self.pending = [] # list of (cb, args, kwargs)
self.running = [] # list of (subprocess, cb)
@ -224,8 +107,6 @@ class ParallelContext(object):
def finish(self):
assert len(self.pending) == 0 and len(self.running) == 0, "pending: %i running: %i" % (len(self.pending), len(self.running))
self.pool.close()
self.pool.join()
self._allcontexts.remove(self)
def run(self):
@ -238,19 +119,16 @@ class ParallelContext(object):
self.pending.append((cb, args, kwargs))
def _docall(self, argv, executable, shell, env, cwd, cb, echo):
if echo is not None:
print echo
job = PopenJob(argv, executable=executable, shell=shell, env=env, cwd=cwd)
self.pool.apply_async(job_runner, args=(job,), callback=job.get_callback(ParallelContext._condition))
self.running.append((job, cb))
if echo is not None:
print echo
try:
p = subprocess.Popen(argv, executable=executable, shell=shell, env=env, cwd=cwd)
except OSError, e:
print >>sys.stderr, e
cb(-127)
return
def _docallnative(self, module, method, argv, env, cwd, cb, echo,
pycommandpath=None):
if echo is not None:
print echo
job = PythonJob(module, method, argv, env, cwd, pycommandpath)
self.pool.apply_async(job_runner, args=(job,), callback=job.get_callback(ParallelContext._condition))
self.running.append((job, cb))
self.running.append((p, cb))
def call(self, argv, shell, env, cwd, cb, echo, executable=None):
"""
@ -259,43 +137,24 @@ class ParallelContext(object):
self.defer(self._docall, argv, executable, shell, env, cwd, cb, echo)
def call_native(self, module, method, argv, env, cwd, cb,
echo, pycommandpath=None):
"""
Asynchronously call the native function
"""
if sys.platform == 'win32':
@staticmethod
def _waitany():
return win32process.WaitForAnyProcess([p for c in ParallelContext._allcontexts for p, cb in c.running])
self.defer(self._docallnative, module, method, argv, env, cwd, cb,
echo, pycommandpath)
@staticmethod
def _comparepid(pid, process):
return pid == process
@staticmethod
def _waitany():
def _checkdone():
jobs = []
for c in ParallelContext._allcontexts:
for i in xrange(0, len(c.running)):
if c.running[i][0].done:
jobs.append(c.running[i])
for j in jobs:
if j in c.running:
c.running.remove(j)
return jobs
else:
@staticmethod
def _waitany():
return os.waitpid(-1, 0)
# We must acquire the lock, and then check to see if any jobs have
# finished. If we don't check after acquiring the lock it's possible
# that all outstanding jobs will have completed before we wait and we'll
# wait for notifications that have already occurred.
ParallelContext._condition.acquire()
jobs = _checkdone()
@staticmethod
def _comparepid(pid, process):
return pid == process.pid
if jobs == []:
ParallelContext._condition.wait()
jobs = _checkdone()
ParallelContext._condition.release()
return jobs
@staticmethod
def spin():
"""
@ -307,10 +166,39 @@ class ParallelContext(object):
for c in clist:
c.run()
# In python 2.4, subprocess instances wait on child processes under the hood when they are created... this
# unfortunate behavior means that before using os.waitpid, we need to check the status using .poll()
# see http://bytes.com/groups/python/675403-os-wait-losing-child
found = False
for c in clist:
for i in xrange(0, len(c.running)):
p, cb = c.running[i]
result = p.poll()
if result != None:
del c.running[i]
cb(result)
found = True
break
if found: break
if found: continue
dowait = util.any((len(c.running) for c in ParallelContext._allcontexts))
if dowait:
for job, cb in ParallelContext._waitany():
cb(job.exitcode)
pid, status = ParallelContext._waitany()
result = statustoresult(status)
for c in ParallelContext._allcontexts:
for i in xrange(0, len(c.running)):
p, cb = c.running[i]
if ParallelContext._comparepid(pid, p):
del c.running[i]
cb(result)
found = True
break
if found: break
else:
assert any(len(c.pending) for c in ParallelContext._allcontexts)

10
build/pymake/tests/native-environment.mk
Просмотреть файл

@ -1,10 +0,0 @@
#T gmake skip
export EXPECTED := some data
CMD = %pycmd writeenvtofile
PYCOMMANDPATH = $(TESTPATH)
all:
$(CMD) results EXPECTED
test "$$(cat results)" = "$(EXPECTED)"
@echo TEST-PASS

21
build/pymake/tests/native-pycommandpath-sep.mk
Просмотреть файл

@ -1,21 +0,0 @@
#T gmake skip
EXPECTED := some data
# verify that we can load native command modules from
# multiple directories in PYCOMMANDPATH separated by the native
# path separator
ifdef __WIN32__
PS:=;
else
PS:=:
endif
CMD = %pycmd writetofile
CMD2 = %pymod writetofile
PYCOMMANDPATH = $(TESTPATH)$(PS)$(TESTPATH)/subdir
all:
$(CMD) results $(EXPECTED)
test "$$(cat results)" = "$(EXPECTED)"
$(CMD2) results2 $(EXPECTED)
test "$$(cat results2)" = "$(EXPECTED)"
@echo TEST-PASS

15
build/pymake/tests/native-pycommandpath.mk
Просмотреть файл

@ -1,15 +0,0 @@
#T gmake skip
EXPECTED := some data
# verify that we can load native command modules from
# multiple space-separated directories in PYCOMMANDPATH
CMD = %pycmd writetofile
CMD2 = %pymod writetofile
PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir
all:
$(CMD) results $(EXPECTED)
test "$$(cat results)" = "$(EXPECTED)"
$(CMD2) results2 $(EXPECTED)
test "$$(cat results2)" = "$(EXPECTED)"
@echo TEST-PASS

10
build/pymake/tests/native-simple.mk
Просмотреть файл

@ -1,10 +0,0 @@
ifndef TOUCH
TOUCH = touch
endif
all: testfile
test -f testfile
@echo TEST-PASS
testfile:
$(TOUCH) $@

21
build/pymake/tests/parallel-native.mk
Просмотреть файл

@ -1,21 +0,0 @@
#T commandline: ['-j2']
# ensure that calling python commands doesn't block other targets
ifndef SLEEP
SLEEP := sleep
endif
PRINTF = printf "$@:0:" >>results
EXPECTED = target2:0:target1:0:
all:: target1 target2
cat results
test "$$(cat results)" = "$(EXPECTED)"
@echo TEST-PASS
target1:
$(SLEEP) 0.1
$(PRINTF)
target2:
$(PRINTF)

9
build/pymake/tests/pycmd.py
Просмотреть файл

@ -1,9 +0,0 @@
import os
def writetofile(args):
with open(args[0], 'w') as f:
f.write(' '.join(args[1:]))
def writeenvtofile(args):
with open(args[0], 'w') as f:
f.write(os.environ[args[1]])

7
build/pymake/tests/rm-fail.mk
Просмотреть файл

@ -1,7 +0,0 @@
#T returncode: 2
all:
mkdir newdir
test -d newdir
touch newdir/newfile
$(RM) newdir
@echo TEST-PASS

13
build/pymake/tests/rm.mk
Просмотреть файл

@ -1,13 +0,0 @@
all:
# $(RM) defaults to -f
$(RM) nosuchfile
touch newfile
test -f newfile
$(RM) newfile
test ! -f newfile
mkdir newdir
test -d newdir
touch newdir/newfile
$(RM) -r newdir
test ! -d newdir
@echo TEST-PASS

1
build/pymake/tests/runtests.py
Просмотреть файл

@ -1,4 +1,3 @@
#!/usr/bin/env python
"""
Run the test(s) listed on the command line. If a directory is listed, the script will recursively
walk the directory for files named .mk and run each.

3
build/pymake/tests/subdir/pymod.py
Просмотреть файл

@ -1,3 +0,0 @@
def writetofile(args):
with open(args[0], 'w') as f:
f.write(' '.join(args[1:]))