зеркало из https://github.com/mozilla/gecko-dev.git
Bug 935987 - Part 2: Add preprocessed files to mozpack.files; r=gps
--HG-- extra : rebase_source : fe32f92b22aecc82ea1b6d95a5ee43e274a9f8be
This commit is contained in:
Brian O'Keefe
Родитель
88d7f1342f
Коммит
49217e22e3
|
|
@ -8,6 +8,9 @@ import re
|
|||
import shutil
|
||||
import stat
|
||||
import uuid
|
||||
import mozbuild.makeutil as makeutil
|
||||
from mozbuild.preprocessor import Preprocessor
|
||||
from mozbuild.util import FileAvoidWrite
|
||||
from mozpack.executables import (
|
||||
is_executable,
|
||||
may_strip,
|
||||
|
|
@ -40,6 +43,10 @@ class Dest(object):
|
|||
self.path = path
|
||||
self.mode = None
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self.path
|
||||
|
||||
def read(self, length=-1):
|
||||
if self.mode != 'r':
|
||||
self.file = open(self.path, 'rb')
|
||||
|
|
@ -67,6 +74,36 @@ class BaseFile(object):
|
|||
their own copy function, or rely on BaseFile.copy using the open() member
|
||||
function and/or the path property.
|
||||
'''
|
||||
@staticmethod
|
||||
def is_older(first, second):
|
||||
'''
|
||||
Compares the modification time of two files, and returns whether the
|
||||
``first`` file is older than the ``second`` file.
|
||||
'''
|
||||
# os.path.getmtime returns a result in seconds with precision up to
|
||||
# the microsecond. But microsecond is too precise because
|
||||
# shutil.copystat only copies milliseconds, and seconds is not
|
||||
# enough precision.
|
||||
return int(os.path.getmtime(first) * 1000) \
|
||||
<= int(os.path.getmtime(second) * 1000)
|
||||
|
||||
@staticmethod
|
||||
def any_newer(dest, inputs):
|
||||
'''
|
||||
Compares the modification time of ``dest`` to multiple input files, and
|
||||
returns whether any of the ``inputs`` is newer (has a later mtime) than
|
||||
``dest``.
|
||||
'''
|
||||
# os.path.getmtime returns a result in seconds with precision up to
|
||||
# the microsecond. But microsecond is too precise because
|
||||
# shutil.copystat only copies milliseconds, and seconds is not
|
||||
# enough precision.
|
||||
dest_mtime = int(os.path.getmtime(dest) * 1000)
|
||||
for input in inputs:
|
||||
if dest_mtime < int(os.path.getmtime(input) * 1000):
|
||||
return True
|
||||
return False
|
||||
|
||||
def copy(self, dest, skip_if_older=True):
|
||||
'''
|
||||
Copy the BaseFile content to the destination given as a string or a
|
||||
|
|
@ -85,12 +122,7 @@ class BaseFile(object):
|
|||
if not dest.exists():
|
||||
can_skip_content_check = True
|
||||
elif getattr(self, 'path', None) and getattr(dest, 'path', None):
|
||||
# os.path.getmtime returns a result in seconds with precision up to
|
||||
# the microsecond. But microsecond is too precise because
|
||||
# shutil.copystat only copies milliseconds, and seconds is not
|
||||
# enough precision.
|
||||
if skip_if_older and int(os.path.getmtime(self.path) * 1000) \
|
||||
<= int(os.path.getmtime(dest.path) * 1000):
|
||||
if skip_if_older and BaseFile.is_older(self.path, dest.path):
|
||||
return False
|
||||
elif os.path.getsize(self.path) != os.path.getsize(dest.path):
|
||||
can_skip_content_check = True
|
||||
|
|
@ -292,6 +324,76 @@ class ExistingFile(BaseFile):
|
|||
dest.path)
|
||||
|
||||
|
||||
class PreprocessedFile(BaseFile):
|
||||
'''
|
||||
File class for a file that is preprocessed. PreprocessedFile.copy() runs
|
||||
the preprocessor on the file to create the output.
|
||||
'''
|
||||
def __init__(self, path, depfile_path, marker, defines, extra_depends=None):
|
||||
self.path = path
|
||||
self.depfile = depfile_path
|
||||
self.marker = marker
|
||||
self.defines = defines
|
||||
self.extra_depends = list(extra_depends or [])
|
||||
|
||||
def copy(self, dest, skip_if_older=True):
|
||||
'''
|
||||
Invokes the preprocessor to create the destination file.
|
||||
'''
|
||||
if isinstance(dest, basestring):
|
||||
dest = Dest(dest)
|
||||
else:
|
||||
assert isinstance(dest, Dest)
|
||||
|
||||
# We have to account for the case where the destination exists and is a
|
||||
# symlink to something. Since we know the preprocessor is certainly not
|
||||
# going to create a symlink, we can just remove the existing one. If the
|
||||
# destination is not a symlink, we leave it alone, since we're going to
|
||||
# overwrite its contents anyway.
|
||||
# If symlinks aren't supported at all, we can skip this step.
|
||||
if hasattr(os, 'symlink'):
|
||||
if os.path.islink(dest.path):
|
||||
os.remove(dest.path)
|
||||
|
||||
pp_deps = set(self.extra_depends)
|
||||
|
||||
# If a dependency file was specified, and it exists, add any
|
||||
# dependencies from that file to our list.
|
||||
if self.depfile and os.path.exists(self.depfile):
|
||||
target = mozpack.path.normpath(dest.name)
|
||||
with open(self.depfile, 'rb') as fileobj:
|
||||
for rule in makeutil.read_dep_makefile(fileobj):
|
||||
if target in rule.targets():
|
||||
pp_deps.update(rule.dependencies())
|
||||
|
||||
skip = False
|
||||
if dest.exists() and skip_if_older:
|
||||
# If a dependency file was specified, and it doesn't exist,
|
||||
# assume that the preprocessor needs to be rerun. That will
|
||||
# regenerate the dependency file.
|
||||
if self.depfile and not os.path.exists(self.depfile):
|
||||
skip = False
|
||||
else:
|
||||
skip = not BaseFile.any_newer(dest.path, pp_deps)
|
||||
|
||||
if skip:
|
||||
return False
|
||||
|
||||
deps_out = None
|
||||
if self.depfile:
|
||||
deps_out = FileAvoidWrite(self.depfile)
|
||||
pp = Preprocessor(defines=self.defines, marker=self.marker)
|
||||
|
||||
with open(self.path, 'rU') as input:
|
||||
pp.processFile(input=input, output=dest, depfile=deps_out)
|
||||
|
||||
dest.close()
|
||||
if self.depfile:
|
||||
deps_out.close()
|
||||
|
||||
return True
|
||||
|
||||
|
||||
class GeneratedFile(BaseFile):
|
||||
'''
|
||||
File class for content with no previous existence on the filesystem.
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from mozpack.files import (
|
|||
JarFinder,
|
||||
ManifestFile,
|
||||
MinifiedProperties,
|
||||
PreprocessedFile,
|
||||
XPTFile,
|
||||
)
|
||||
from mozpack.mozjar import (
|
||||
|
|
@ -327,6 +328,128 @@ class TestAbsoluteSymlinkFile(TestWithTmpDir):
|
|||
link = os.readlink(dest)
|
||||
self.assertEqual(link, source)
|
||||
|
||||
class TestPreprocessedFile(TestWithTmpDir):
|
||||
def test_preprocess(self):
|
||||
'''
|
||||
Test that copying the file invokes the preprocessor
|
||||
'''
|
||||
src = self.tmppath('src')
|
||||
dest = self.tmppath('dest')
|
||||
|
||||
with open(src, 'wb') as tmp:
|
||||
tmp.write('#ifdef FOO\ntest\n#endif')
|
||||
|
||||
f = PreprocessedFile(src, depfile_path=None, marker='#', defines={'FOO': True})
|
||||
self.assertTrue(f.copy(dest))
|
||||
|
||||
self.assertEqual('test\n', open(dest, 'rb').read())
|
||||
|
||||
def test_preprocess_file_no_write(self):
|
||||
'''
|
||||
Test various conditions where PreprocessedFile.copy is expected not to
|
||||
write in the destination file.
|
||||
'''
|
||||
src = self.tmppath('src')
|
||||
dest = self.tmppath('dest')
|
||||
depfile = self.tmppath('depfile')
|
||||
|
||||
with open(src, 'wb') as tmp:
|
||||
tmp.write('#ifdef FOO\ntest\n#endif')
|
||||
|
||||
# Initial copy
|
||||
f = PreprocessedFile(src, depfile_path=depfile, marker='#', defines={'FOO': True})
|
||||
self.assertTrue(f.copy(dest))
|
||||
|
||||
# Ensure subsequent copies won't trigger writes
|
||||
self.assertFalse(f.copy(DestNoWrite(dest)))
|
||||
self.assertEqual('test\n', open(dest, 'rb').read())
|
||||
|
||||
# When the source file is older than the destination file, even with
|
||||
# different content, no copy should occur.
|
||||
with open(src, 'wb') as tmp:
|
||||
tmp.write('#ifdef FOO\nfooo\n#endif')
|
||||
time = os.path.getmtime(dest) - 1
|
||||
os.utime(src, (time, time))
|
||||
self.assertFalse(f.copy(DestNoWrite(dest)))
|
||||
self.assertEqual('test\n', open(dest, 'rb').read())
|
||||
|
||||
# skip_if_older=False is expected to force a copy in this situation.
|
||||
self.assertTrue(f.copy(dest, skip_if_older=False))
|
||||
self.assertEqual('fooo\n', open(dest, 'rb').read())
|
||||
|
||||
def test_preprocess_file_dependencies(self):
|
||||
'''
|
||||
Test that the preprocess runs if the dependencies of the source change
|
||||
'''
|
||||
src = self.tmppath('src')
|
||||
dest = self.tmppath('dest')
|
||||
incl = self.tmppath('incl')
|
||||
deps = self.tmppath('src.pp')
|
||||
|
||||
with open(src, 'wb') as tmp:
|
||||
tmp.write('#ifdef FOO\ntest\n#endif')
|
||||
|
||||
with open(incl, 'wb') as tmp:
|
||||
tmp.write('foo bar')
|
||||
|
||||
# Initial copy
|
||||
f = PreprocessedFile(src, depfile_path=deps, marker='#', defines={'FOO': True})
|
||||
self.assertTrue(f.copy(dest))
|
||||
|
||||
# Update the source so it #includes the include file.
|
||||
with open(src, 'wb') as tmp:
|
||||
tmp.write('#include incl\n')
|
||||
time = os.path.getmtime(dest) + 1
|
||||
os.utime(src, (time, time))
|
||||
self.assertTrue(f.copy(dest))
|
||||
self.assertEqual('foo bar', open(dest, 'rb').read())
|
||||
|
||||
# If one of the dependencies changes, the file should be updated. The
|
||||
# mtime of the dependency is set after the destination file, to avoid
|
||||
# both files having the same time.
|
||||
with open(incl, 'wb') as tmp:
|
||||
tmp.write('quux')
|
||||
time = os.path.getmtime(dest) + 1
|
||||
os.utime(incl, (time, time))
|
||||
self.assertTrue(f.copy(dest))
|
||||
self.assertEqual('quux', open(dest, 'rb').read())
|
||||
|
||||
# Perform one final copy to confirm that we don't run the preprocessor
|
||||
# again. We update the mtime of the destination so it's newer than the
|
||||
# input files. This would "just work" if we weren't changing
|
||||
time = os.path.getmtime(incl) + 1
|
||||
os.utime(dest, (time, time))
|
||||
self.assertFalse(f.copy(DestNoWrite(dest)))
|
||||
|
||||
def test_replace_symlink(self):
|
||||
'''
|
||||
Test that if the destination exists, and is a symlink, the target of
|
||||
the symlink is not overwritten by the preprocessor output.
|
||||
'''
|
||||
if not self.symlink_supported:
|
||||
return
|
||||
|
||||
source = self.tmppath('source')
|
||||
dest = self.tmppath('dest')
|
||||
pp_source = self.tmppath('pp_in')
|
||||
deps = self.tmppath('deps')
|
||||
|
||||
with open(source, 'a'):
|
||||
pass
|
||||
|
||||
os.symlink(source, dest)
|
||||
self.assertTrue(os.path.islink(dest))
|
||||
|
||||
with open(pp_source, 'wb') as tmp:
|
||||
tmp.write('#define FOO\nPREPROCESSED')
|
||||
|
||||
f = PreprocessedFile(pp_source, depfile_path=deps, marker='#',
|
||||
defines={'FOO': True})
|
||||
self.assertTrue(f.copy(dest))
|
||||
|
||||
self.assertEqual('PREPROCESSED', open(dest, 'rb').read())
|
||||
self.assertFalse(os.path.islink(dest))
|
||||
self.assertEqual('', open(source, 'rb').read())
|
||||
|
||||
class TestExistingFile(TestWithTmpDir):
|
||||
def test_required_missing_dest(self):
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче