зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1528892 - Make preprocessor output more reproducible. r=froydnj
On CI, Windows builds start from different directories on every build, except when sccache is enabled. This affects many build types, such as l10n repacks, and the preprocessor likes to put full paths in its output, which means it includes those different directories, making the builds non reproducible. This changes the preprocessor to replace the source and object directories with generic strings. Differential Revision: https://phabricator.services.mozilla.com/D20421 --HG-- extra : moz-landing-system : lando
This commit is contained in:
Mike Hommey
Родитель
3b5df70b17
Коммит
de6692d10a
|
|
@ -45,6 +45,15 @@ __all__ = [
|
|||
]
|
||||
|
||||
|
||||
def path_starts_with(path, prefix):
|
||||
if os.altsep:
|
||||
prefix = prefix.replace(os.altsep, os.sep)
|
||||
path = path.replace(os.altsep, os.sep)
|
||||
prefix = [os.path.normcase(p) for p in prefix.split(os.sep)]
|
||||
path = [os.path.normcase(p) for p in path.split(os.sep)]
|
||||
return path[:len(prefix)] == prefix
|
||||
|
||||
|
||||
class Expression:
|
||||
def __init__(self, expression_string):
|
||||
"""
|
||||
|
|
@ -280,6 +289,15 @@ class Preprocessor:
|
|||
'LINE': 0,
|
||||
'DIRECTORY': os.path.abspath('.')}.iteritems():
|
||||
self.context[k] = v
|
||||
try:
|
||||
# Can import globally because of bootstrapping issues.
|
||||
from buildconfig import topsrcdir, topobjdir
|
||||
except ImportError:
|
||||
# Allow this script to still work independently of a configured objdir.
|
||||
topsrcdir = topobjdir = None
|
||||
self.topsrcdir = topsrcdir
|
||||
self.topobjdir = topobjdir
|
||||
self.curdir = '.'
|
||||
self.actionLevel = 0
|
||||
self.disableLevel = 0
|
||||
# ifStates can be
|
||||
|
|
@ -747,7 +765,7 @@ class Preprocessor:
|
|||
if filters:
|
||||
args = self.applyFilters(args)
|
||||
if not os.path.isabs(args):
|
||||
args = os.path.join(self.context['DIRECTORY'], args)
|
||||
args = os.path.join(self.curdir, args)
|
||||
args = open(args, 'rU')
|
||||
except Preprocessor.Error:
|
||||
raise
|
||||
|
|
@ -757,15 +775,22 @@ class Preprocessor:
|
|||
oldFile = self.context['FILE']
|
||||
oldLine = self.context['LINE']
|
||||
oldDir = self.context['DIRECTORY']
|
||||
oldCurdir = self.curdir
|
||||
self.noteLineInfo()
|
||||
|
||||
if args.isatty():
|
||||
# we're stdin, use '-' and '' for file and dir
|
||||
self.context['FILE'] = '-'
|
||||
self.context['DIRECTORY'] = ''
|
||||
self.curdir = '.'
|
||||
else:
|
||||
abspath = os.path.abspath(args.name)
|
||||
self.curdir = os.path.dirname(abspath)
|
||||
self.includes.add(abspath)
|
||||
if self.topobjdir and path_starts_with(abspath, self.topobjdir):
|
||||
abspath = '$OBJDIR' + abspath[len(self.topobjdir):]
|
||||
elif self.topsrcdir and path_starts_with(abspath, self.topsrcdir):
|
||||
abspath = '$SRCDIR' + abspath[len(self.topsrcdir):]
|
||||
self.context['FILE'] = abspath
|
||||
self.context['DIRECTORY'] = os.path.dirname(abspath)
|
||||
self.context['LINE'] = 0
|
||||
|
|
@ -780,6 +805,7 @@ class Preprocessor:
|
|||
self.checkLineNumbers = oldCheckLineNumbers
|
||||
self.context['LINE'] = oldLine
|
||||
self.context['DIRECTORY'] = oldDir
|
||||
self.curdir = oldCurdir
|
||||
def do_includesubst(self, args):
|
||||
args = self.filter_substitution(args)
|
||||
self.do_include(args)
|
||||
|
|
|
|||
|
|
@ -148,8 +148,8 @@ class TestBuild(unittest.TestCase):
|
|||
|
||||
def validate(self, config):
|
||||
self.maxDiff = None
|
||||
test_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||||
'data', 'build') + os.sep
|
||||
test_path = os.sep.join(('$SRCDIR', 'python', 'mozbuild', 'mozbuild',
|
||||
'test', 'backend', 'data', 'build')) + os.sep
|
||||
|
||||
# We want unicode instances out of the files, because having plain str
|
||||
# makes assertEqual diff output in case of error extra verbose because
|
||||
|
|
|
|||
|
|
@ -567,53 +567,67 @@ class TestPreprocessor(unittest.TestCase):
|
|||
|
||||
def test_include_line(self):
|
||||
files = {
|
||||
'test.js': '\n'.join([
|
||||
'srcdir/test.js': '\n'.join([
|
||||
'#define foo foobarbaz',
|
||||
'#include @inc@',
|
||||
'@bar@',
|
||||
'',
|
||||
]),
|
||||
'bar.js': '\n'.join([
|
||||
'srcdir/bar.js': '\n'.join([
|
||||
'#define bar barfoobaz',
|
||||
'@foo@',
|
||||
'',
|
||||
]),
|
||||
'foo.js': '\n'.join([
|
||||
'srcdir/foo.js': '\n'.join([
|
||||
'bazfoobar',
|
||||
'#include bar.js',
|
||||
'bazbarfoo',
|
||||
'',
|
||||
]),
|
||||
'baz.js': 'baz\n',
|
||||
'f.js': '\n'.join([
|
||||
'objdir/baz.js': 'baz\n',
|
||||
'srcdir/f.js': '\n'.join([
|
||||
'#include foo.js',
|
||||
'#filter substitution',
|
||||
'#define inc bar.js',
|
||||
'#include test.js',
|
||||
'#include baz.js',
|
||||
'#include ../objdir/baz.js',
|
||||
'fin',
|
||||
'',
|
||||
]),
|
||||
}
|
||||
|
||||
preprocessed = ('//@line 1 "$SRCDIR/foo.js"\n'
|
||||
'bazfoobar\n'
|
||||
'//@line 2 "$SRCDIR/bar.js"\n'
|
||||
'@foo@\n'
|
||||
'//@line 3 "$SRCDIR/foo.js"\n'
|
||||
'bazbarfoo\n'
|
||||
'//@line 2 "$SRCDIR/bar.js"\n'
|
||||
'foobarbaz\n'
|
||||
'//@line 3 "$SRCDIR/test.js"\n'
|
||||
'barfoobaz\n'
|
||||
'//@line 1 "$OBJDIR/baz.js"\n'
|
||||
'baz\n'
|
||||
'//@line 6 "$SRCDIR/f.js"\n'
|
||||
'fin\n').replace('DIR/', 'DIR' + os.sep)
|
||||
|
||||
# Try with separate srcdir/objdir
|
||||
with MockedOpen(files):
|
||||
self.pp.do_include('f.js')
|
||||
self.assertEqual(self.pp.out.getvalue(),
|
||||
('//@line 1 "CWD/foo.js"\n'
|
||||
'bazfoobar\n'
|
||||
'//@line 2 "CWD/bar.js"\n'
|
||||
'@foo@\n'
|
||||
'//@line 3 "CWD/foo.js"\n'
|
||||
'bazbarfoo\n'
|
||||
'//@line 2 "CWD/bar.js"\n'
|
||||
'foobarbaz\n'
|
||||
'//@line 3 "CWD/test.js"\n'
|
||||
'barfoobaz\n'
|
||||
'//@line 1 "CWD/baz.js"\n'
|
||||
'baz\n'
|
||||
'//@line 6 "CWD/f.js"\n'
|
||||
'fin\n').replace('CWD/',
|
||||
os.getcwd() + os.path.sep))
|
||||
self.pp.topsrcdir = os.path.abspath('srcdir')
|
||||
self.pp.topobjdir = os.path.abspath('objdir')
|
||||
self.pp.do_include('srcdir/f.js')
|
||||
self.assertEqual(self.pp.out.getvalue(), preprocessed)
|
||||
|
||||
# Try again with relative objdir
|
||||
self.setUp()
|
||||
files['srcdir/objdir/baz.js'] = files['objdir/baz.js']
|
||||
del files['objdir/baz.js']
|
||||
files['srcdir/f.js'] = files['srcdir/f.js'].replace('../', '')
|
||||
with MockedOpen(files):
|
||||
self.pp.topsrcdir = os.path.abspath('srcdir')
|
||||
self.pp.topobjdir = os.path.abspath('srcdir/objdir')
|
||||
self.pp.do_include('srcdir/f.js')
|
||||
self.assertEqual(self.pp.out.getvalue(), preprocessed)
|
||||
|
||||
def test_include_missing_file(self):
|
||||
with MockedOpen({'f': '#include foo\n'}):
|
||||
|
|
|
|||
|
|
@ -188,8 +188,7 @@ class PreprocessorOutputWrapper(object):
|
|||
self._pp = preprocessor
|
||||
|
||||
def write(self, str):
|
||||
file = os.path.normpath(os.path.abspath(self._pp.context['FILE']))
|
||||
with errors.context(file, self._pp.context['LINE']):
|
||||
with errors.context(self._pp.context['FILE'], self._pp.context['LINE']):
|
||||
self._parser.handle_line(str)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
import unittest
|
||||
import mozunit
|
||||
import os
|
||||
from buildconfig import topobjdir
|
||||
from mozpack.packager import (
|
||||
preprocess_manifest,
|
||||
CallDeque,
|
||||
|
|
@ -43,7 +44,7 @@ baz@SUFFIX@
|
|||
|
||||
|
||||
class TestPreprocessManifest(unittest.TestCase):
|
||||
MANIFEST_PATH = os.path.join(os.path.abspath(os.curdir), 'manifest')
|
||||
MANIFEST_PATH = os.path.join('$OBJDIR', 'manifest')
|
||||
|
||||
EXPECTED_LOG = [
|
||||
((MANIFEST_PATH, 2), 'add', '', 'bar/*'),
|
||||
|
|
@ -68,6 +69,11 @@ class TestPreprocessManifest(unittest.TestCase):
|
|||
self.log.append(args)
|
||||
|
||||
self.sink = MockSink()
|
||||
self.cwd = os.getcwd()
|
||||
os.chdir(topobjdir)
|
||||
|
||||
def tearDown(self):
|
||||
os.chdir(self.cwd)
|
||||
|
||||
def test_preprocess_manifest(self):
|
||||
with MockedOpen({'manifest': MANIFEST}):
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче