refactor library building/caching
This commit is contained in:
Alon Zakai
Родитель
d259c5e17e
Коммит
96a9c24f81
|
|
@ -164,7 +164,7 @@ class RunnerCore(unittest.TestCase):
|
|||
# Link all files
|
||||
if len(additional_files) + len(libraries) > 0:
|
||||
shutil.move(filename + '.o', filename + '.o.alone')
|
||||
do_link([filename + '.o.alone'] + map(lambda f: f + '.o', additional_files) + libraries,
|
||||
Building.link([filename + '.o.alone'] + map(lambda f: f + '.o', additional_files) + libraries,
|
||||
filename + '.o')
|
||||
if not os.path.exists(filename + '.o'):
|
||||
print "Failed to link LLVM binaries:\n\n", output
|
||||
|
|
@ -242,6 +242,27 @@ class RunnerCore(unittest.TestCase):
|
|||
limit_size(''.join([a.rstrip()+'\n' for a in difflib.unified_diff(value.split('\n'), string.split('\n'), fromfile='expected', tofile='actual')]))
|
||||
))
|
||||
|
||||
library_cache = {}
|
||||
|
||||
def get_library(self, name, generated_libs, configure=['./configure'], configure_args=[], make=['make'], make_args=['-j', '2'], cache=True):
|
||||
build_dir = self.get_build_dir()
|
||||
output_dir = self.get_dir()
|
||||
|
||||
cache_name = name + '|' + COMPILER
|
||||
if self.library_cache is not None:
|
||||
if cache and self.library_cache.get(cache_name):
|
||||
print >> sys.stderr, '<load build from cache> ',
|
||||
bc_file = os.path.join(output_dir, 'lib' + name + '.bc')
|
||||
f = open(bc_file, 'wb')
|
||||
f.write(self.library_cache[cache_name])
|
||||
f.close()
|
||||
return bc_file
|
||||
|
||||
print >> sys.stderr, '<building and saving into cache> ',
|
||||
|
||||
return Building.build_library(name, build_dir, output_dir, generated_libs, configure, configure_args, make, make_args, self.library_cache, cache_name)
|
||||
|
||||
|
||||
###################################################################################################
|
||||
|
||||
if 'benchmark' not in str(sys.argv):
|
||||
|
|
@ -3081,7 +3102,7 @@ if 'benchmark' not in str(sys.argv):
|
|||
def get_freetype(self):
|
||||
Settings.INIT_STACK = 1 # TODO: Investigate why this is necessary
|
||||
|
||||
return get_library('freetype', self.get_build_dir(), self.get_dir(), os.path.join('objs', '.libs', 'libfreetype.a.bc'))
|
||||
return self.get_library('freetype', os.path.join('objs', '.libs', 'libfreetype.a.bc'))
|
||||
|
||||
def test_freetype(self):
|
||||
if Settings.QUANTUM_SIZE == 1: return self.skip('TODO: Figure out and try to fix')
|
||||
|
|
@ -3159,7 +3180,7 @@ if 'benchmark' not in str(sys.argv):
|
|||
|
||||
self.do_run(open(path_from_root('tests', 'zlib', 'example.c'), 'r').read(),
|
||||
open(path_from_root('tests', 'zlib', 'ref.txt'), 'r').read(),
|
||||
libraries=[get_library('zlib', self.get_build_dir(), self.get_dir(), os.path.join('libz.a.bc'), make_args=['libz.a'])],
|
||||
libraries=[self.get_library('zlib', os.path.join('libz.a.bc'), make_args=['libz.a'])],
|
||||
includes=[path_from_root('tests', 'zlib')],
|
||||
force_c=True)
|
||||
|
||||
|
|
@ -3178,7 +3199,7 @@ if 'benchmark' not in str(sys.argv):
|
|||
|
||||
self.do_run(open(path_from_root('tests', 'bullet', 'Demos', 'HelloWorld', 'HelloWorld.cpp'), 'r').read(),
|
||||
open(path_from_root('tests', 'bullet', 'output.txt'), 'r').read(),
|
||||
libraries=[get_library('bullet', self.get_build_dir(), self.get_dir(), [os.path.join('src', '.libs', 'libBulletCollision.a.bc'),
|
||||
libraries=[self.get_library('bullet', [os.path.join('src', '.libs', 'libBulletCollision.a.bc'),
|
||||
os.path.join('src', '.libs', 'libBulletDynamics.a.bc'),
|
||||
os.path.join('src', '.libs', 'libLinearMath.a.bc')],
|
||||
configure_args=['--disable-demos','--disable-dependency-tracking'])],
|
||||
|
|
@ -3239,11 +3260,11 @@ if 'benchmark' not in str(sys.argv):
|
|||
)
|
||||
src.close()
|
||||
|
||||
#fontconfig = get_library('fontconfig', self.get_build_dir(), self.get_dir(), [os.path.join('src', '.libs', 'libfontconfig.a')]) # Used in file, but not needed, mostly
|
||||
#fontconfig = self.get_library('fontconfig', [os.path.join('src', '.libs', 'libfontconfig.a')]) # Used in file, but not needed, mostly
|
||||
|
||||
freetype = self.get_freetype()
|
||||
|
||||
poppler = get_library('poppler', self.get_build_dir(), self.get_dir(),
|
||||
poppler = self.get_library('poppler',
|
||||
[os.path.join('poppler', '.libs', 'libpoppler.so.13.0.0.bc'),
|
||||
os.path.join('goo', '.libs', 'libgoo.a.bc'),
|
||||
os.path.join('fofi', '.libs', 'libfofi.a.bc'),
|
||||
|
|
@ -3255,7 +3276,7 @@ if 'benchmark' not in str(sys.argv):
|
|||
# Combine libraries
|
||||
|
||||
combined = os.path.join(self.get_building_dir(), 'combined.bc')
|
||||
do_link([freetype, poppler], combined)
|
||||
Building.link([freetype, poppler], combined)
|
||||
|
||||
self.do_ll_run(combined,
|
||||
lambda: map(ord, open(path_from_root('tests', 'poppler', 'ref.ppm'), 'r').read()).__str__().replace(' ', ''),
|
||||
|
|
@ -3284,7 +3305,7 @@ if 'benchmark' not in str(sys.argv):
|
|||
)
|
||||
open(filename, 'w').write(src)
|
||||
|
||||
lib = get_library('openjpeg', self.get_build_dir(), self.get_dir(),
|
||||
lib = self.get_library('openjpeg',
|
||||
[os.path.join('bin', 'libopenjpeg.so.1.4.0.bc'),
|
||||
os.path.sep.join('codec/CMakeFiles/j2k_to_image.dir/index.c.o'.split('/')),
|
||||
os.path.sep.join('codec/CMakeFiles/j2k_to_image.dir/convert.c.o'.split('/')),
|
||||
|
|
|
|||
|
|
@ -214,61 +214,43 @@ Settings = Dummy() # A global singleton. Not pretty, but nicer than passing |, s
|
|||
|
||||
# Building
|
||||
|
||||
COMPILER = CLANG
|
||||
LLVM_OPTS = False
|
||||
COMPILER_TEST_OPTS = []
|
||||
class Building:
|
||||
COMPILER = CLANG
|
||||
LLVM_OPTS = False
|
||||
COMPILER_TEST_OPTS = []
|
||||
|
||||
GlobalCache = {}
|
||||
@staticmethod
|
||||
def build_library(name, build_dir, output_dir, generated_libs, configure=['./configure'], configure_args=[], make=['make'], make_args=['-j', '2'], cache=None, cache_name=None):
|
||||
''' Build a library into a .bc file. We build the .bc file once and cache it for all our tests. (We cache in
|
||||
memory since the test directory is destroyed and recreated for each test. Note that we cache separately
|
||||
for different compilers) '''
|
||||
|
||||
def get_library(name, build_dir, output_dir, generated_libs, configure=['./configure'], configure_args=[], make=['make'], make_args=['-j', '2'], cache=True, build_subdir=None):
|
||||
''' Build a library into a .bc file. We build the .bc file once and cache it for all our tests. (We cache in
|
||||
memory since the test directory is destroyed and recreated for each test. Note that we cache separately
|
||||
for different compilers) '''
|
||||
if type(generated_libs) is not list: generated_libs = [generated_libs]
|
||||
|
||||
if type(generated_libs) is not list: generated_libs = [generated_libs]
|
||||
if build_subdir and configure.startswith('./'):
|
||||
configure = '.' + configure
|
||||
temp_dir = build_dir
|
||||
project_dir = os.path.join(temp_dir, name)
|
||||
shutil.copytree(path_from_root('tests', name), project_dir) # Useful in debugging sometimes to comment this out
|
||||
os.chdir(project_dir)
|
||||
env = os.environ.copy()
|
||||
env['RANLIB'] = env['AR'] = env['CXX'] = env['CC'] = env['LIBTOOL'] = EMMAKEN
|
||||
env['EMMAKEN_COMPILER'] = COMPILER
|
||||
env['EMSCRIPTEN_TOOLS'] = path_from_root('tools')
|
||||
env['CFLAGS'] = env['EMMAKEN_CFLAGS'] = ' '.join(COMPILER_OPTS + COMPILER_TEST_OPTS) # Normal CFLAGS is ignored by some configure's.
|
||||
if configure: # Useful in debugging sometimes to comment this out (and the lines below up to and including the |make| call)
|
||||
env['EMMAKEN_JUST_CONFIGURE'] = '1'
|
||||
Popen(configure + configure_args, stdout=open(os.path.join(output_dir, 'configure'), 'w'),
|
||||
stderr=open(os.path.join(output_dir, 'configure_err'), 'w'), env=env).communicate()[0]
|
||||
del env['EMMAKEN_JUST_CONFIGURE']
|
||||
Popen(make + make_args, stdout=open(os.path.join(output_dir, 'make'), 'w'),
|
||||
stderr=open(os.path.join(output_dir, 'make_err'), 'w'), env=env).communicate()[0]
|
||||
bc_file = os.path.join(project_dir, 'bc.bc')
|
||||
Building.link(map(lambda lib: os.path.join(project_dir, lib), generated_libs), bc_file)
|
||||
if cache is not None:
|
||||
cache[cache_name] = open(bc_file, 'rb').read()
|
||||
return bc_file
|
||||
|
||||
if GlobalCache is not None:
|
||||
cache_name = name + '|' + COMPILER
|
||||
if cache and GlobalCache.get(cache_name):
|
||||
print >> sys.stderr, '<load build from cache> ',
|
||||
bc_file = os.path.join(output_dir, 'lib' + name + '.bc')
|
||||
f = open(bc_file, 'wb')
|
||||
f.write(GlobalCache[cache_name])
|
||||
f.close()
|
||||
return bc_file
|
||||
|
||||
temp_dir = build_dir
|
||||
project_dir = os.path.join(temp_dir, name)
|
||||
shutil.copytree(path_from_root('tests', name), project_dir) # Useful in debugging sometimes to comment this out
|
||||
os.chdir(project_dir)
|
||||
if build_subdir:
|
||||
try:
|
||||
os.mkdir('cbuild')
|
||||
except:
|
||||
pass
|
||||
os.chdir(os.path.join(project_dir, 'cbuild'))
|
||||
env = os.environ.copy()
|
||||
env['RANLIB'] = env['AR'] = env['CXX'] = env['CC'] = env['LIBTOOL'] = EMMAKEN
|
||||
env['EMMAKEN_COMPILER'] = COMPILER
|
||||
env['EMSCRIPTEN_TOOLS'] = path_from_root('tools')
|
||||
env['CFLAGS'] = env['EMMAKEN_CFLAGS'] = ' '.join(COMPILER_OPTS + COMPILER_TEST_OPTS) # Normal CFLAGS is ignored by some configure's.
|
||||
if configure: # Useful in debugging sometimes to comment this out (and the lines below up to and including the |make| call)
|
||||
env['EMMAKEN_JUST_CONFIGURE'] = '1'
|
||||
Popen(configure + configure_args, stdout=open(os.path.join(output_dir, 'configure'), 'w'),
|
||||
stderr=open(os.path.join(output_dir, 'configure_err'), 'w'), env=env).communicate()[0]
|
||||
del env['EMMAKEN_JUST_CONFIGURE']
|
||||
Popen(make + make_args, stdout=open(os.path.join(output_dir, 'make'), 'w'),
|
||||
stderr=open(os.path.join(output_dir, 'make_err'), 'w'), env=env).communicate()[0]
|
||||
bc_file = os.path.join(project_dir, 'bc.bc')
|
||||
do_link(map(lambda lib: os.path.join(project_dir, 'cbuild', lib) if build_subdir else os.path.join(project_dir, lib), generated_libs), bc_file)
|
||||
if cache and GlobalCache is not None:
|
||||
print >> sys.stderr, '<save build into cache> ',
|
||||
GlobalCache[cache_name] = open(bc_file, 'rb').read()
|
||||
return bc_file
|
||||
|
||||
def do_link(files, target):
|
||||
output = Popen([LLVM_LINK] + files + ['-o', target], stdout=PIPE, stderr=STDOUT).communicate()[0]
|
||||
assert output is None or 'Could not open input file' not in output, 'Linking error: ' + output
|
||||
@staticmethod
|
||||
def link(files, target):
|
||||
output = Popen([LLVM_LINK] + files + ['-o', target], stdout=PIPE, stderr=STDOUT).communicate()[0]
|
||||
assert output is None or 'Could not open input file' not in output, 'Linking error: ' + output
|
||||
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче