diff --git a/em-config.py b/em-config.py index 0f3767913..2708d18ca 100755 --- a/em-config.py +++ b/em-config.py @@ -16,17 +16,17 @@ is found, or exits with 1 if the variable does not exist. import sys import re -from tools import shared +from tools import config def main(): if len(sys.argv) != 2 or \ not re.match(r"^[\w\W_][\w\W_\d]*$", sys.argv[1]) or \ - not hasattr(shared, sys.argv[1]): + not hasattr(config, sys.argv[1]): print('Usage: em-config VAR_NAME', file=sys.stderr) exit(1) - print(getattr(shared, sys.argv[1])) + print(getattr(config, sys.argv[1])) return 0 diff --git a/emcc.py b/emcc.py index 51555e5d3..3d8660992 100755 --- a/emcc.py +++ b/emcc.py @@ -49,6 +49,7 @@ from tools.toolchain_profiler import ToolchainProfiler from tools import js_manipulation from tools import wasm2c from tools import webassembly +from tools import config if __name__ == '__main__': ToolchainProfiler.record_process_start() @@ -654,8 +655,8 @@ def backend_binaryen_passes(): def make_js_executable(script): src = open(script).read() - cmd = shared.shlex_join(shared.JS_ENGINE) - if not os.path.isabs(shared.JS_ENGINE[0]): + cmd = shared.shlex_join(config.JS_ENGINE) + if not os.path.isabs(config.JS_ENGINE[0]): # TODO: use whereis etc. And how about non-*NIX? cmd = '/usr/bin/env -S ' + cmd logger.debug('adding `#!` to JavaScript file: %s' % cmd) @@ -926,7 +927,7 @@ There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR P # warnings are properly printed during arg parse. newargs = diagnostics.capture_warnings(newargs) - if not shared.CONFIG_FILE: + if not config.config_file: diagnostics.warning('deprecated', 'Specifying EM_CONFIG as a python literal is deprecated. Please use a file instead.') for i in range(len(newargs)): @@ -1947,10 +1948,10 @@ There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR P CXX = [shared.CLANG_CXX] CC = [shared.CLANG_CC] - if shared.COMPILER_WRAPPER: - logger.debug('using compiler wrapper: %s', shared.COMPILER_WRAPPER) - CXX.insert(0, shared.COMPILER_WRAPPER) - CC.insert(0, shared.COMPILER_WRAPPER) + if config.COMPILER_WRAPPER: + logger.debug('using compiler wrapper: %s', config.COMPILER_WRAPPER) + CXX.insert(0, config.COMPILER_WRAPPER) + CC.insert(0, config.COMPILER_WRAPPER) if 'EMMAKEN_COMPILER' in os.environ: diagnostics.warning('deprecated', '`EMMAKEN_COMPILER` is deprecated.\n' @@ -2437,7 +2438,7 @@ def parse_args(newargs): elif check_arg('--extern-post-js'): options.extern_post_js += open(consume_arg()).read() + '\n' elif check_arg('--compiler-wrapper'): - shared.COMPILER_WRAPPER = consume_arg() + config.COMPILER_WRAPPER = consume_arg() elif check_flag('--post-link'): options.post_link = True elif check_arg('--oformat'): @@ -2604,7 +2605,7 @@ def parse_args(newargs): if os.path.exists(path): exit_with_error('File ' + optarg + ' passed to --generate-config already exists!') else: - shared.generate_config(optarg) + config.generate_config(optarg) should_exit = True # Record USE_PTHREADS setting because it controls whether --shared-memory is passed to lld elif arg == '-pthread': diff --git a/tests/clang_native.py b/tests/clang_native.py index 3032c4824..d980777e3 100644 --- a/tests/clang_native.py +++ b/tests/clang_native.py @@ -5,7 +5,8 @@ import logging import os -from tools.shared import MACOS, WINDOWS, path_from_root, PIPE, run_process, CLANG_CC, CLANG_CXX +from tools.shared import PIPE, run_process, CLANG_CC, CLANG_CXX +from tools.utils import MACOS, WINDOWS, path_from_root from tools import building logger = logging.getLogger('clang_native') diff --git a/tests/jsrun.py b/tests/jsrun.py index 780ba6e7c..f421c80fd 100644 --- a/tests/jsrun.py +++ b/tests/jsrun.py @@ -9,6 +9,7 @@ import sys from subprocess import Popen, PIPE, CalledProcessError from tools import shared +from tools.shared import config WORKING_ENGINES = {} # Holds all configured engines and whether they work: maps path -> True/False @@ -90,7 +91,7 @@ def run_js(filename, engine=None, args=[], stdin=None, stdout=PIPE, stderr=None, cwd=None, full_output=False, assert_returncode=0, skip_check=False): if not engine: - engine = shared.JS_ENGINES[0] + engine = config.JS_ENGINES[0] # We used to support True here but we no longer do. Assert here just in case. assert(type(assert_returncode) == int) diff --git a/tests/runner.py b/tests/runner.py index 363f2544f..c3c0cfe6d 100755 --- a/tests/runner.py +++ b/tests/runner.py @@ -55,12 +55,13 @@ import clang_native import jsrun import parallel_testsuite from jsrun import NON_ZERO -from tools.shared import EM_CONFIG, TEMP_DIR, EMCC, EMXX, DEBUG +from tools.config import EM_CONFIG +from tools.shared import TEMP_DIR, EMCC, EMXX, DEBUG from tools.shared import EMSCRIPTEN_TEMP_DIR -from tools.shared import MACOS, WINDOWS from tools.shared import EM_BUILD_VERBOSE from tools.shared import asstr, get_canonical_temp_dir, try_delete -from tools.shared import asbytes, Settings +from tools.shared import asbytes, Settings, config +from tools.utils import MACOS, WINDOWS from tools import shared, line_endings, building @@ -232,24 +233,24 @@ def chdir(dir): @contextlib.contextmanager def js_engines_modify(replacements): - """A context manager that updates shared.JS_ENGINES.""" - original = shared.JS_ENGINES - shared.JS_ENGINES = replacements + """A context manager that updates config.JS_ENGINES.""" + original = config.JS_ENGINES + config.JS_ENGINES = replacements try: yield finally: - shared.JS_ENGINES = original + config.JS_ENGINES = original @contextlib.contextmanager def wasm_engines_modify(replacements): - """A context manager that updates shared.WASM_ENGINES.""" - original = shared.WASM_ENGINES - shared.WASM_ENGINES = replacements + """A context manager that updates config.WASM_ENGINES.""" + original = config.WASM_ENGINES + config.WASM_ENGINES = replacements try: yield finally: - shared.WASM_ENGINES = original + config.WASM_ENGINES = original def ensure_dir(dirname): @@ -989,7 +990,7 @@ class RunnerCore(unittest.TestCase, metaclass=RunnerMeta): def filtered_js_engines(self, js_engines=None): if js_engines is None: - js_engines = shared.JS_ENGINES + js_engines = config.JS_ENGINES for engine in js_engines: assert type(engine) == list for engine in self.banned_js_engines: @@ -1046,7 +1047,7 @@ class RunnerCore(unittest.TestCase, metaclass=RunnerMeta): if self.get_setting('STANDALONE_WASM'): # TODO once standalone wasm support is more stable, apply use_all_engines # like with js engines, but for now as we bring it up, test in all of them - wasm_engines = shared.WASM_ENGINES + wasm_engines = config.WASM_ENGINES if len(wasm_engines) == 0: logger.warning('no wasm engine was found to run the standalone part of this test') engines += wasm_engines @@ -1656,8 +1657,8 @@ def build_library(name, def check_js_engines(): - working_engines = list(filter(jsrun.check_engine, shared.JS_ENGINES)) - if len(working_engines) < len(shared.JS_ENGINES): + working_engines = list(filter(jsrun.check_engine, config.JS_ENGINES)) + if len(working_engines) < len(config.JS_ENGINES): print('Not all the JS engines in JS_ENGINES appears to work.') exit(1) diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 02f5ae23e..4fc8b3fe6 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -18,8 +18,8 @@ if __name__ == '__main__': import clang_native import jsrun import runner -from tools.shared import run_process, path_from_root, SPIDERMONKEY_ENGINE, LLVM_ROOT, V8_ENGINE, PIPE, try_delete, EMCC -from tools import shared, building +from tools.shared import run_process, path_from_root, PIPE, try_delete, EMCC, config +from tools import building # standard arguments for timing: # 0: no runtime, just startup @@ -188,7 +188,7 @@ class EmscriptenBenchmarker(Benchmarker): self.filename = filename self.old_env = os.environ os.environ = self.env.copy() - llvm_root = self.env.get('LLVM') or LLVM_ROOT + llvm_root = self.env.get('LLVM') or config.LLVM_ROOT if lib_builder: env_init = self.env.copy() # Note that we need to pass in all the flags here because some build @@ -356,11 +356,11 @@ benchmarkers = [ # NativeBenchmarker('gcc', 'gcc', 'g++') ] -if V8_ENGINE and V8_ENGINE in shared.JS_ENGINES: +if config.V8_ENGINE and config.V8_ENGINE in config.JS_ENGINES: # avoid the baseline compiler running, because it adds a lot of noise # (the nondeterministic time it takes to get to the full compiler ends up # mattering as much as the actual benchmark) - aot_v8 = V8_ENGINE + ['--no-liftoff'] + aot_v8 = config.V8_ENGINE + ['--no-liftoff'] default_v8_name = os.environ.get('EMBENCH_NAME') or 'v8' benchmarkers += [ EmscriptenBenchmarker(default_v8_name, aot_v8), @@ -372,7 +372,7 @@ if V8_ENGINE and V8_ENGINE in shared.JS_ENGINES: # CheerpBenchmarker('cheerp-v8-wasm', aot_v8), ] -if SPIDERMONKEY_ENGINE and SPIDERMONKEY_ENGINE in shared.JS_ENGINES: +if config.SPIDERMONKEY_ENGINE and config.SPIDERMONKEY_ENGINE in config.JS_ENGINES: # TODO: ensure no baseline compiler is used, see v8 benchmarkers += [ # EmscriptenBenchmarker('sm', SPIDERMONKEY_ENGINE), @@ -382,7 +382,7 @@ if SPIDERMONKEY_ENGINE and SPIDERMONKEY_ENGINE in shared.JS_ENGINES: # CheerpBenchmarker('cheerp-sm-wasm', SPIDERMONKEY_ENGINE), ] -if shared.NODE_JS and shared.NODE_JS in shared.JS_ENGINES: +if config.NODE_JS and config.NODE_JS in config.JS_ENGINES: benchmarkers += [ # EmscriptenBenchmarker('Node.js', shared.NODE_JS), ] @@ -408,7 +408,7 @@ class benchmark(runner.RunnerCore): fingerprint.append('sm: ' + [line for line in run_process(['hg', 'tip'], stdout=PIPE).stdout.splitlines() if 'changeset' in line][0]) except Exception: pass - fingerprint.append('llvm: ' + LLVM_ROOT) + fingerprint.append('llvm: ' + config.LLVM_ROOT) print('Running Emscripten benchmarks... [ %s ]' % ' | '.join(fingerprint)) # avoid depending on argument reception from the commandline diff --git a/tests/test_browser.py b/tests/test_browser.py index 7759a7079..b00b4932b 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -25,8 +25,8 @@ from runner import no_wasm_backend, create_test_file, parameterized, ensure_dir from tools import building from tools import shared from tools import system_libs -from tools.shared import PYTHON, EMCC, WINDOWS, FILE_PACKAGER, PIPE, V8_ENGINE -from tools.shared import try_delete +from tools.shared import PYTHON, EMCC, WINDOWS, FILE_PACKAGER, PIPE +from tools.shared import try_delete, config def test_chunked_synchronous_xhr_server(support_byte_ranges, chunkSize, data, checksum, port): @@ -4887,7 +4887,7 @@ window.close = function() { # test that we can allocate in the 2-4GB range, if we enable growth and # set the max appropriately self.emcc_args += ['-O2', '-s', 'ALLOW_MEMORY_GROWTH', '-s', 'MAXIMUM_MEMORY=4GB'] - self.do_run_in_out_file_test('tests', 'browser', 'test_4GB.cpp', js_engines=[V8_ENGINE]) + self.do_run_in_out_file_test('tests', 'browser', 'test_4GB.cpp', js_engines=[config.V8_ENGINE]) @no_firefox('no 4GB support yet') def test_zzz_zzz_2GB_fail(self): @@ -4900,7 +4900,7 @@ window.close = function() { # test that growth doesn't go beyond 2GB without the max being set for that, # and that we can catch an allocation failure exception for that self.emcc_args += ['-O2', '-s', 'ALLOW_MEMORY_GROWTH', '-s', 'MAXIMUM_MEMORY=2GB'] - self.do_run_in_out_file_test('tests', 'browser', 'test_2GB_fail.cpp', js_engines=[V8_ENGINE]) + self.do_run_in_out_file_test('tests', 'browser', 'test_2GB_fail.cpp', js_engines=[config.V8_ENGINE]) @no_firefox('no 4GB support yet') def test_zzz_zzz_4GB_fail(self): @@ -4913,7 +4913,7 @@ window.close = function() { # test that we properly report an allocation error that would overflow over # 4GB. self.emcc_args += ['-O2', '-s', 'ALLOW_MEMORY_GROWTH', '-s', 'MAXIMUM_MEMORY=4GB', '-s', 'ABORTING_MALLOC=0'] - self.do_run_in_out_file_test('tests', 'browser', 'test_4GB_fail.cpp', js_engines=[V8_ENGINE]) + self.do_run_in_out_file_test('tests', 'browser', 'test_4GB_fail.cpp', js_engines=[config.V8_ENGINE]) @unittest.skip("only run this manually, to test for race conditions") @parameterized({ diff --git a/tests/test_core.py b/tests/test_core.py index a80b813e1..2cee76587 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -20,8 +20,9 @@ if __name__ == '__main__': raise Exception('do not run this file directly; do something like: tests/runner.py') from tools.shared import try_delete, PIPE -from tools.shared import NODE_JS, V8_ENGINE, JS_ENGINES, SPIDERMONKEY_ENGINE, PYTHON, EMCC, EMAR, WINDOWS, MACOS, LLVM_ROOT -from tools import shared, building +from tools.shared import PYTHON, EMCC, EMAR +from tools.utils import WINDOWS, MACOS +from tools import shared, building, config from runner import RunnerCore, path_from_root, requires_native_clang from runner import skip_if, no_wasm_backend, needs_dlfcn, no_windows, is_slow_test, create_test_file, parameterized from runner import js_engines_modify, wasm_engines_modify, env_modify, with_env_modify, disabled @@ -33,24 +34,24 @@ import clang_native def wasm_simd(f): def decorated(self): - if not V8_ENGINE or V8_ENGINE not in JS_ENGINES: + if not config.V8_ENGINE or config.V8_ENGINE not in config.JS_ENGINES: self.skipTest('wasm simd only supported in d8 for now') if not self.is_wasm(): self.skipTest('wasm2js only supports MVP for now') self.emcc_args.append('-msimd128') self.emcc_args.append('-fno-lax-vector-conversions') - with js_engines_modify([V8_ENGINE + ['--experimental-wasm-simd']]): + with js_engines_modify([config.V8_ENGINE + ['--experimental-wasm-simd']]): f(self) return decorated def bleeding_edge_wasm_backend(f): def decorated(self): - if not V8_ENGINE or V8_ENGINE not in JS_ENGINES: + if not config.V8_ENGINE or config.V8_ENGINE not in config.JS_ENGINES: self.skipTest('only works in d8 for now') if not self.is_wasm(): self.skipTest('wasm2js only supports MVP for now') - with js_engines_modify([V8_ENGINE]): + with js_engines_modify([config.V8_ENGINE]): f(self) return decorated @@ -61,7 +62,7 @@ def also_with_wasm_bigint(f): f(self) if self.get_setting('WASM'): self.set_setting('WASM_BIGINT', 1) - with js_engines_modify([NODE_JS + ['--experimental-wasm-bigint']]): + with js_engines_modify([config.NODE_JS + ['--experimental-wasm-bigint']]): f(self) return decorated @@ -90,10 +91,10 @@ def with_both_exception_handling(f): # Wasm EH is currently supported only in wasm backend and V8 if not self.get_setting('WASM'): self.skipTest('wasm2js does not support wasm exceptions') - if not V8_ENGINE or V8_ENGINE not in JS_ENGINES: + if not config.V8_ENGINE or config.V8_ENGINE not in config.JS_ENGINES: self.skipTest('d8 required to run wasm eh tests') self.emcc_args.append('-fwasm-exceptions') - with js_engines_modify([V8_ENGINE + ['--experimental-wasm-eh']]): + with js_engines_modify([config.V8_ENGINE + ['--experimental-wasm-eh']]): f(self) else: self.set_setting('DISABLE_EXCEPTION_CATCHING', 0) @@ -128,7 +129,7 @@ def also_with_noderawfs(func): func(self) print('noderawfs') self.emcc_args = orig_args + ['-s', 'NODERAWFS=1', '-DNODERAWFS'] - with js_engines_modify([NODE_JS]): + with js_engines_modify([config.NODE_JS]): func(self) return decorated @@ -156,8 +157,8 @@ def also_with_standalone_wasm(wasm2c=False, impure=False): # when it sees an i64 on the ffi. self.set_setting('WASM_BIGINT', 1) # if we are impure, disallow all wasm engines - with wasm_engines_modify([] if impure else shared.WASM_ENGINES): - with js_engines_modify([NODE_JS + ['--experimental-wasm-bigint']]): + with wasm_engines_modify([] if impure else config.WASM_ENGINES): + with js_engines_modify([config.NODE_JS + ['--experimental-wasm-bigint']]): func(self) if wasm2c: print('wasm2c') @@ -179,7 +180,7 @@ def node_pthreads(f): self.skipTest('asan ends up using atomics that are not yet supported in node 12') if self.get_setting('MINIMAL_RUNTIME'): self.skipTest('node pthreads not yet supported with MINIMAL_RUNTIME') - with js_engines_modify([NODE_JS + ['--experimental-wasm-threads', '--experimental-wasm-bulk-memory']]): + with js_engines_modify([config.NODE_JS + ['--experimental-wasm-threads', '--experimental-wasm-bulk-memory']]): f(self) return decorated @@ -398,7 +399,7 @@ class TestCoreBase(RunnerCore): self.set_setting('WASM_BIGINT', 1) self.emcc_args += ['-fexceptions'] self.do_run_in_out_file_test('tests', 'core', 'test_i64_invoke_bigint.cpp', - js_engines=[NODE_JS + ['--experimental-wasm-bigint']]) + js_engines=[config.NODE_JS + ['--experimental-wasm-bigint']]) def test_vararg_copy(self): self.do_run_in_out_file_test('tests', 'va_arg', 'test_va_copy.c') @@ -1796,7 +1797,7 @@ int main() { self.do_run(src, 'got null') def test_emscripten_get_now(self): - self.banned_js_engines = [V8_ENGINE] # timer limitations in v8 shell + self.banned_js_engines = [config.V8_ENGINE] # timer limitations in v8 shell # needs to flush stdio streams self.set_setting('EXIT_RUNTIME', 1) @@ -2443,7 +2444,7 @@ The current type of b is: 9 self.do_run_in_out_file_test('tests', 'core', 'test_copyop.cpp') def test_memcpy_memcmp(self): - self.banned_js_engines = [V8_ENGINE] # Currently broken under V8_ENGINE but not node + self.banned_js_engines = [config.V8_ENGINE] # Currently broken under V8_ENGINE but not node def check(result, err): result = result.replace('\n \n', '\n') # remove extra node output @@ -2808,7 +2809,7 @@ The current type of b is: 9 def test_dlfcn_data_and_fptr(self): # Failing under v8 since: https://chromium-review.googlesource.com/712595 if self.is_wasm(): - self.banned_js_engines = [V8_ENGINE] + self.banned_js_engines = [config.V8_ENGINE] self.prep_dlfcn_lib() create_test_file('liblib.cpp', r''' @@ -4740,7 +4741,7 @@ Pass: 0.000012 0.000012''') self.do_run_in_out_file_test('tests', 'core', 'test_langinfo.c') def test_files(self): - self.banned_js_engines = [SPIDERMONKEY_ENGINE] # closure can generate variables called 'gc', which pick up js shell stuff + self.banned_js_engines = [config.SPIDERMONKEY_ENGINE] # closure can generate variables called 'gc', which pick up js shell stuff if self.maybe_closure(): # Use closure here, to test we don't break FS stuff self.emcc_args = [x for x in self.emcc_args if x != '-g'] # ensure we test --closure 1 --memory-init-file 1 (-g would disable closure) elif '-O3' in self.emcc_args and not self.is_wasm(): @@ -4824,13 +4825,13 @@ Module = { self.do_runf(path_from_root('tests', 'fs', 'test_getdents64.cpp'), '..') def test_getdents64_special_cases(self): - self.banned_js_engines = [V8_ENGINE] # https://bugs.chromium.org/p/v8/issues/detail?id=6881 + self.banned_js_engines = [config.V8_ENGINE] # https://bugs.chromium.org/p/v8/issues/detail?id=6881 src = path_from_root('tests', 'fs', 'test_getdents64_special_cases.cpp') out = path_from_root('tests', 'fs', 'test_getdents64_special_cases.out') self.do_run_from_file(src, out, assert_identical=True) def test_getcwd_with_non_ascii_name(self): - self.banned_js_engines = [V8_ENGINE] # https://bugs.chromium.org/p/v8/issues/detail?id=6881 + self.banned_js_engines = [config.V8_ENGINE] # https://bugs.chromium.org/p/v8/issues/detail?id=6881 src = path_from_root('tests', 'fs', 'test_getcwd_with_non_ascii_name.cpp') out = path_from_root('tests', 'fs', 'test_getcwd_with_non_ascii_name.out') self.do_run_from_file(src, out, assert_identical=True) @@ -4848,7 +4849,7 @@ Module = { self.emcc_args = orig_compiler_opts + ['-D' + fs] if fs == 'NODEFS': self.emcc_args += ['-lnodefs.js'] - self.do_runf(path_from_root('tests', 'stdio', 'test_fgetc_ungetc.c'), 'success', js_engines=[NODE_JS]) + self.do_runf(path_from_root('tests', 'stdio', 'test_fgetc_ungetc.c'), 'success', js_engines=[config.NODE_JS]) def test_fgetc_unsigned(self): src = r''' @@ -5006,7 +5007,7 @@ main( int argv, char ** argc ) { @no_minimal_runtime('MINIMAL_RUNTIME does not have getValue() and setValue() (TODO add it to a JS library function to get it in)') def test_utf(self): - self.banned_js_engines = [SPIDERMONKEY_ENGINE] # only node handles utf well + self.banned_js_engines = [config.SPIDERMONKEY_ENGINE] # only node handles utf well self.set_setting('EXPORTED_FUNCTIONS', ['_main', '_malloc']) self.set_setting('EXTRA_EXPORTED_RUNTIME_METHODS', ['getValue', 'setValue', 'UTF8ToString', 'stringToUTF8']) self.do_run_in_out_file_test('tests', 'core', 'test_utf.c') @@ -5117,11 +5118,11 @@ main( int argv, char ** argc ) { def test_fs_nodefs_home(self): self.set_setting('FORCE_FILESYSTEM', 1) self.emcc_args += ['-lnodefs.js'] - self.do_runf(path_from_root('tests', 'fs', 'test_nodefs_home.c'), 'success', js_engines=[NODE_JS]) + self.do_runf(path_from_root('tests', 'fs', 'test_nodefs_home.c'), 'success', js_engines=[config.NODE_JS]) def test_fs_nodefs_nofollow(self): self.emcc_args += ['-lnodefs.js'] - self.do_runf(path_from_root('tests', 'fs', 'test_nodefs_nofollow.c'), 'success', js_engines=[NODE_JS]) + self.do_runf(path_from_root('tests', 'fs', 'test_nodefs_nofollow.c'), 'success', js_engines=[config.NODE_JS]) def test_fs_trackingdelegate(self): src = path_from_root('tests', 'fs', 'test_trackingdelegate.c') @@ -5206,12 +5207,12 @@ main( int argv, char ** argc ) { self.emcc_args = orig_compiler_opts + ['-D' + fs] if fs == 'NODEFS': self.emcc_args += ['-lnodefs.js'] - self.do_run_in_out_file_test('tests', 'unistd', 'access.c', js_engines=[NODE_JS]) + self.do_run_in_out_file_test('tests', 'unistd', 'access.c', js_engines=[config.NODE_JS]) # Node.js fs.chmod is nearly no-op on Windows if not WINDOWS: self.emcc_args = orig_compiler_opts self.emcc_args += ['-s', 'NODERAWFS=1'] - self.do_run_in_out_file_test('tests', 'unistd', 'access.c', js_engines=[NODE_JS]) + self.do_run_in_out_file_test('tests', 'unistd', 'access.c', js_engines=[config.NODE_JS]) def test_unistd_curdir(self): self.uses_es6 = True @@ -5245,14 +5246,14 @@ main( int argv, char ** argc ) { self.emcc_args = orig_compiler_opts + ['-D' + fs] if fs == 'NODEFS': self.emcc_args += ['-lnodefs.js'] - self.do_run_in_out_file_test('tests', 'unistd', 'truncate.c', js_engines=[NODE_JS]) + self.do_run_in_out_file_test('tests', 'unistd', 'truncate.c', js_engines=[config.NODE_JS]) @no_windows("Windows throws EPERM rather than EACCES or EINVAL") @unittest.skipIf(WINDOWS or os.geteuid() == 0, "Root access invalidates this test by being able to write on readonly files") def test_unistd_truncate_noderawfs(self): self.uses_es6 = True self.set_setting('NODERAWFS') - self.do_run_in_out_file_test('tests', 'unistd', 'truncate.c', js_engines=[NODE_JS]) + self.do_run_in_out_file_test('tests', 'unistd', 'truncate.c', js_engines=[config.NODE_JS]) def test_unistd_swab(self): self.do_run_in_out_file_test('tests', 'unistd', 'swab.c') @@ -5290,7 +5291,7 @@ main( int argv, char ** argc ) { self.emcc_args += ['-DNO_SYMLINK=1'] if MACOS: continue - self.do_runf(path_from_root('tests', 'unistd', 'unlink.c'), 'success', js_engines=[NODE_JS]) + self.do_runf(path_from_root('tests', 'unistd', 'unlink.c'), 'success', js_engines=[config.NODE_JS]) # Several differences/bugs on non-linux including https://github.com/nodejs/node/issues/18014 if not WINDOWS and not MACOS: self.emcc_args = orig_compiler_opts + ['-DNODERAWFS'] @@ -5298,7 +5299,7 @@ main( int argv, char ** argc ) { if os.geteuid() == 0: self.emcc_args += ['-DSKIP_ACCESS_TESTS'] self.emcc_args += ['-s', 'NODERAWFS=1'] - self.do_runf(path_from_root('tests', 'unistd', 'unlink.c'), 'success', js_engines=[NODE_JS]) + self.do_runf(path_from_root('tests', 'unistd', 'unlink.c'), 'success', js_engines=[config.NODE_JS]) def test_unistd_links(self): self.clear() @@ -5313,7 +5314,7 @@ main( int argv, char ** argc ) { self.emcc_args = orig_compiler_opts + ['-D' + fs] if fs == 'NODEFS': self.emcc_args += ['-lnodefs.js'] - self.do_run_in_out_file_test('tests', 'unistd', 'links.c', js_engines=[NODE_JS]) + self.do_run_in_out_file_test('tests', 'unistd', 'links.c', js_engines=[config.NODE_JS]) @no_windows('Skipping NODEFS test, since it would require administrative privileges.') def test_unistd_symlink_on_nodefs(self): @@ -5321,7 +5322,7 @@ main( int argv, char ** argc ) { # test expects /, but Windows gives \ as path slashes. # Calling readlink() on a non-link gives error 22 EINVAL on Unix, but simply error 0 OK on Windows. self.emcc_args += ['-lnodefs.js'] - self.do_run_in_out_file_test('tests', 'unistd', 'symlink_on_nodefs.c', js_engines=[NODE_JS]) + self.do_run_in_out_file_test('tests', 'unistd', 'symlink_on_nodefs.c', js_engines=[config.NODE_JS]) def test_unistd_sleep(self): self.do_run_in_out_file_test('tests', 'unistd', 'sleep.c') @@ -5344,13 +5345,13 @@ main( int argv, char ** argc ) { self.emcc_args = orig_compiler_opts + ['-D' + fs] if fs == 'NODEFS': self.emcc_args += ['-lnodefs.js'] - self.do_run_in_out_file_test('tests', 'unistd', 'misc.c', js_engines=[NODE_JS]) + self.do_run_in_out_file_test('tests', 'unistd', 'misc.c', js_engines=[config.NODE_JS]) # i64s in the API, which we'd need to legalize for JS, so in standalone mode # all we can test is wasm VMs @also_with_standalone_wasm(wasm2c=True) def test_posixtime(self): - self.banned_js_engines = [V8_ENGINE] # v8 lacks monotonic time + self.banned_js_engines = [config.V8_ENGINE] # v8 lacks monotonic time self.do_run_in_out_file_test('tests', 'core', 'test_posixtime.c') def test_uname(self): @@ -5596,7 +5597,7 @@ int main(void) { @no_asan('depends on the specifics of memory size, which for asan we are forced to increase') def test_dlmalloc_inline(self): - self.banned_js_engines = [NODE_JS] # slower, and fail on 64-bit + self.banned_js_engines = [config.NODE_JS] # slower, and fail on 64-bit # needed with typed arrays self.set_setting('INITIAL_MEMORY', 128 * 1024 * 1024) @@ -5606,7 +5607,7 @@ int main(void) { @no_asan('depends on the specifics of memory size, which for asan we are forced to increase') def test_dlmalloc(self): - self.banned_js_engines = [NODE_JS] # slower, and fail on 64-bit + self.banned_js_engines = [config.NODE_JS] # slower, and fail on 64-bit # needed with typed arrays self.set_setting('INITIAL_MEMORY', 128 * 1024 * 1024) @@ -6529,8 +6530,8 @@ return malloc(size); self.assertIsNotNone(full_aborter) self.assertIsNotNone(short_aborter) print('full:', full_aborter, 'short:', short_aborter) - if SPIDERMONKEY_ENGINE and os.path.exists(SPIDERMONKEY_ENGINE[0]): - output = self.run_js('test_demangle_stacks.js', engine=SPIDERMONKEY_ENGINE, assert_returncode=NON_ZERO) + if config.SPIDERMONKEY_ENGINE and os.path.exists(config.SPIDERMONKEY_ENGINE[0]): + output = self.run_js('test_demangle_stacks.js', engine=config.SPIDERMONKEY_ENGINE, assert_returncode=NON_ZERO) # we may see the full one, if -g, or the short one if not if ' ' + short_aborter + ' ' not in output and ' ' + full_aborter + ' ' not in output: # stack traces may also be ' name ' or 'name@' etc @@ -7031,7 +7032,7 @@ someweirdtext building.emcc('src.cpp', self.get_emcc_args(), js_filename) - LLVM_DWARFDUMP = os.path.join(LLVM_ROOT, 'llvm-dwarfdump') + LLVM_DWARFDUMP = os.path.join(config.LLVM_ROOT, 'llvm-dwarfdump') out = self.run_process([LLVM_DWARFDUMP, wasm_filename, '-all'], stdout=PIPE).stdout # parse the sections @@ -7168,7 +7169,7 @@ someweirdtext @no_wasm2js('symbol names look different wasm2js backtraces') def test_emscripten_log(self): - self.banned_js_engines = [V8_ENGINE] # v8 doesn't support console.log + self.banned_js_engines = [config.V8_ENGINE] # v8 doesn't support console.log self.emcc_args += ['-s', 'DEMANGLE_SUPPORT=1'] if '-g' not in self.emcc_args: self.emcc_args.append('-g') @@ -7248,7 +7249,7 @@ someweirdtext # needs to flush stdio streams self.set_setting('EXIT_RUNTIME', 1) self.set_setting('ASYNCIFY', 1) - self.banned_js_engines = [SPIDERMONKEY_ENGINE, V8_ENGINE] # needs setTimeout which only node has + self.banned_js_engines = [config.SPIDERMONKEY_ENGINE, config.V8_ENGINE] # needs setTimeout which only node has src = r''' #include @@ -7693,12 +7694,12 @@ NODEFS is no longer included by default; build with -lnodefs.js js = open('test_hello_world.js').read() assert ('require(' in js) == ('node' in self.get_setting('ENVIRONMENT')), 'we should have require() calls only if node js specified' - for engine in JS_ENGINES: + for engine in config.JS_ENGINES: print(engine) # set us to test in just this engine - self.banned_js_engines = [e for e in JS_ENGINES if e != engine] + self.banned_js_engines = [e for e in config.JS_ENGINES if e != engine] # tell the compiler to build with just that engine - if engine == NODE_JS: + if engine == config.NODE_JS: right = 'node' wrong = 'shell' else: @@ -7766,7 +7767,7 @@ NODEFS is no longer included by default; build with -lnodefs.js }) def test_minimal_runtime_hello_world(self, args): # TODO: Support for non-Node.js shells has not yet been added to MINIMAL_RUNTIME - self.banned_js_engines = [V8_ENGINE, SPIDERMONKEY_ENGINE] + self.banned_js_engines = [config.V8_ENGINE, config.SPIDERMONKEY_ENGINE] self.emcc_args = ['-s', 'MINIMAL_RUNTIME=1'] + args self.set_setting('MINIMAL_RUNTIME', 1) self.maybe_closure() diff --git a/tests/test_interactive.py b/tests/test_interactive.py index 19f8e1ef5..f6440ddb1 100644 --- a/tests/test_interactive.py +++ b/tests/test_interactive.py @@ -13,7 +13,8 @@ if __name__ == '__main__': from runner import parameterized from runner import BrowserCore, path_from_root -from tools.shared import EMCC, WINDOWS, which +from tools.shared import EMCC, WINDOWS +from tools.utils import which class interactive(BrowserCore): diff --git a/tests/test_other.py b/tests/test_other.py index 3358edf99..ab9defb8e 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -28,20 +28,19 @@ from subprocess import PIPE, STDOUT if __name__ == '__main__': raise Exception('do not run this file directly; do something like: tests/runner.py other') -from tools.shared import try_delete -from tools.shared import EMCC, EMXX, EMAR, EMRANLIB, PYTHON, FILE_PACKAGER, WINDOWS, LLVM_ROOT, EM_BUILD_VERBOSE +from tools.shared import try_delete, config +from tools.shared import EMCC, EMXX, EMAR, EMRANLIB, PYTHON, FILE_PACKAGER, WINDOWS, EM_BUILD_VERBOSE from tools.shared import CLANG_CC, CLANG_CXX, LLVM_AR, LLVM_DWARFDUMP -from tools.shared import NODE_JS, JS_ENGINES, WASM_ENGINES, V8_ENGINE from runner import RunnerCore, path_from_root, is_slow_test, ensure_dir, disabled, make_executable from runner import env_modify, no_mac, no_windows, requires_native_clang, chdir, with_env_modify, create_test_file, parameterized from runner import js_engines_modify, NON_ZERO -from tools import shared, building +from tools import shared, building, utils import jsrun import clang_native from tools import line_endings from tools import webassembly -scons_path = shared.which('scons') +scons_path = utils.which('scons') emmake = shared.bat_suffix(path_from_root('emmake')) emcmake = shared.bat_suffix(path_from_root('emcmake')) emconfigure = shared.bat_suffix(path_from_root('emconfigure')) @@ -464,7 +463,7 @@ f.close() self.assertExists('a.out.js') if wasm: self.assertExists('a.out.wasm') - for engine in JS_ENGINES: + for engine in config.JS_ENGINES: print(' engine', engine) out = self.run_js('a.out.js', engine=engine) self.assertContained('hello, world!', out) @@ -490,8 +489,8 @@ f.close() def test_emar_em_config_flag(self): # Test that the --em-config flag is accepted but not passed down do llvm-ar. # We expand this in case the EM_CONFIG is ~/.emscripten (default) - config = os.path.expanduser(shared.EM_CONFIG) - proc = self.run_process([EMAR, '--em-config', config, '-version'], stdout=PIPE, stderr=PIPE) + conf = os.path.expanduser(config.EM_CONFIG) + proc = self.run_process([EMAR, '--em-config', conf, '-version'], stdout=PIPE, stderr=PIPE) self.assertEqual(proc.stderr, "") self.assertContained('LLVM', proc.stdout) @@ -533,7 +532,7 @@ f.close() for generator in generators: conf = configurations[generator] - if not shared.which(conf['build'][0]): + if not utils.which(conf['build'][0]): # Use simple test if applicable print('Skipping %s test for CMake support; build tool found found: %s.' % (generator, conf['build'][0])) continue @@ -559,7 +558,7 @@ f.close() # Run through node, if CMake produced a .js file. if output_file.endswith('.js'): - ret = self.run_process(NODE_JS + [tempdirname + '/' + output_file], stdout=PIPE).stdout + ret = self.run_process(config.NODE_JS + [tempdirname + '/' + output_file], stdout=PIPE).stdout self.assertTextDataIdentical(open(cmakelistsdir + '/out.txt').read().strip(), ret.strip()) # Test that the various CMAKE_xxx_COMPILE_FEATURES that are advertised for the Emscripten toolchain match with the actual language features that Clang supports. @@ -583,7 +582,7 @@ f.close() # Tests that it's possible to pass C++11 or GNU++11 build modes to CMake by building code that # needs C++11 (embind) def test_cmake_with_embind_cpp11_mode(self): - if WINDOWS and not shared.which('ninja'): + if WINDOWS and not utils.which('ninja'): self.skipTest('Skipping cmake test on windows since ninja not found') for args in [[], ['-DNO_GNU_EXTENSIONS=1']]: self.clear() @@ -597,7 +596,7 @@ f.close() print(str(build)) self.run_process(build) - out = self.run_process(NODE_JS + ['cmake_with_emval.js'], stdout=PIPE).stdout + out = self.run_process(config.NODE_JS + ['cmake_with_emval.js'], stdout=PIPE).stdout if '-DNO_GNU_EXTENSIONS=1' in args: self.assertContained('Hello! __STRICT_ANSI__: 1, __cplusplus: 201103', out) else: @@ -1017,8 +1016,8 @@ int f() { def test_stdin(self): def run_test(): - for engine in JS_ENGINES: - if engine == V8_ENGINE: + for engine in config.JS_ENGINES: + if engine == config.V8_ENGINE: continue # no stdin support in v8 shell engine[0] = os.path.normpath(engine[0]) print(engine, file=sys.stderr) @@ -1061,7 +1060,7 @@ int f() { ''') create_test_file('my_test.input', 'abc') building.emcc('main.cpp', ['--embed-file', 'my_test.input'], output_filename='a.out.js') - self.assertContained('zyx', self.run_process(JS_ENGINES[0] + ['a.out.js'], stdout=PIPE, stderr=PIPE).stdout) + self.assertContained('zyx', self.run_process(config.JS_ENGINES[0] + ['a.out.js'], stdout=PIPE, stderr=PIPE).stdout) def test_abspaths(self): # Includes with absolute paths are generally dangerous, things like -I/usr/.. will get to system local headers, not our portable ones. @@ -1448,7 +1447,7 @@ int f() { def test_libpng(self): shutil.copyfile(path_from_root('tests', 'pngtest.png'), 'pngtest.png') building.emcc(path_from_root('tests', 'pngtest.c'), ['--embed-file', 'pngtest.png', '-s', 'USE_LIBPNG=1'], output_filename='a.out.js') - self.assertContained('TESTS PASSED', self.run_process(JS_ENGINES[0] + ['a.out.js'], stdout=PIPE, stderr=PIPE).stdout) + self.assertContained('TESTS PASSED', self.run_process(config.JS_ENGINES[0] + ['a.out.js'], stdout=PIPE, stderr=PIPE).stdout) def test_libjpeg(self): shutil.copyfile(path_from_root('tests', 'screenshot.jpg'), 'screenshot.jpg') @@ -1457,16 +1456,16 @@ int f() { def test_bullet(self): building.emcc(path_from_root('tests', 'bullet_hello_world.cpp'), ['-s', 'USE_BULLET=1'], output_filename='a.out.js') - self.assertContained('BULLET RUNNING', self.run_process(JS_ENGINES[0] + ['a.out.js'], stdout=PIPE, stderr=PIPE).stdout) + self.assertContained('BULLET RUNNING', self.run_process(config.JS_ENGINES[0] + ['a.out.js'], stdout=PIPE, stderr=PIPE).stdout) def test_vorbis(self): # This will also test if ogg compiles, because vorbis depends on ogg building.emcc(path_from_root('tests', 'vorbis_test.c'), ['-s', 'USE_VORBIS=1'], output_filename='a.out.js') - self.assertContained('ALL OK', self.run_process(JS_ENGINES[0] + ['a.out.js'], stdout=PIPE, stderr=PIPE).stdout) + self.assertContained('ALL OK', self.run_process(config.JS_ENGINES[0] + ['a.out.js'], stdout=PIPE, stderr=PIPE).stdout) def test_bzip2(self): building.emcc(path_from_root('tests', 'bzip2_test.c'), ['-s', 'USE_BZIP2=1'], output_filename='a.out.js') - self.assertContained("usage: unzcrash filename", self.run_process(JS_ENGINES[0] + ['a.out.js'], stdout=PIPE, stderr=PIPE).stdout) + self.assertContained("usage: unzcrash filename", self.run_process(config.JS_ENGINES[0] + ['a.out.js'], stdout=PIPE, stderr=PIPE).stdout) def test_freetype(self): # copy the Liberation Sans Bold truetype file located in the @@ -1492,7 +1491,7 @@ int f() { ' +****+ +****\n' + \ ' +****+ +****\n' + \ ' **** ****' - self.assertContained(expectedOutput, self.run_process(JS_ENGINES[0] + ['a.out.js'], stdout=PIPE, stderr=PIPE).stdout) + self.assertContained(expectedOutput, self.run_process(config.JS_ENGINES[0] + ['a.out.js'], stdout=PIPE, stderr=PIPE).stdout) def test_link_memcpy(self): # memcpy can show up *after* optimizations, so after our opportunity to link in libc, so it must be special-cased @@ -1798,10 +1797,10 @@ int f() { # test calling optimizer if not acorn: print(' js') - output = self.run_process(NODE_JS + [path_from_root('tools', 'js-optimizer.js'), input] + passes, stdin=PIPE, stdout=PIPE).stdout + output = self.run_process(config.NODE_JS + [path_from_root('tools', 'js-optimizer.js'), input] + passes, stdin=PIPE, stdout=PIPE).stdout else: print(' acorn') - output = self.run_process(NODE_JS + [path_from_root('tools', 'acorn-optimizer.js'), input] + passes, stdin=PIPE, stdout=PIPE).stdout + output = self.run_process(config.NODE_JS + [path_from_root('tools', 'acorn-optimizer.js'), input] + passes, stdin=PIPE, stdout=PIPE).stdout def check_js(js, expected): # print >> sys.stderr, 'chak\n==========================\n', js, '\n===========================\n' @@ -2064,7 +2063,7 @@ int f() { def test_emconfig(self): output = self.run_process([emconfig, 'LLVM_ROOT'], stdout=PIPE).stdout.strip() - self.assertEqual(output, LLVM_ROOT) + self.assertEqual(output, config.LLVM_ROOT) # EMSCRIPTEN_ROOT is kind of special since it should always report the locaton of em-config # itself (its not configurable via the config file but driven by the location for arg0) output = self.run_process([emconfig, 'EMSCRIPTEN_ROOT'], stdout=PIPE).stdout.strip() @@ -2364,7 +2363,7 @@ void wakaw::Cm::RasterBase::merbine1::merbine1::merbine1::merbine1 "/" ''') # Run the test and confirm the output is as expected. - out = self.run_js('testrun.js', engine=NODE_JS + ['--experimental-wasm-bigint']) + out = self.run_js('testrun.js', engine=config.NODE_JS + ['--experimental-wasm-bigint']) self.assertContained('''\ input = 0xaabbccdd11223344 low = 5678 @@ -6256,10 +6255,10 @@ mergeInto(LibraryManager.library, { src = open('a.out.js').read() envs = ['web', 'worker', 'node', 'shell'] for env in envs: - for engine in JS_ENGINES: - if engine == V8_ENGINE: + for engine in config.JS_ENGINES: + if engine == config.V8_ENGINE: continue # ban v8, weird failures - actual = 'NODE' if engine == NODE_JS else 'SHELL' + actual = 'NODE' if engine == config.NODE_JS else 'SHELL' print(env, actual, engine) module = {'ENVIRONMENT': env} if env != actual: @@ -6296,7 +6295,7 @@ mergeInto(LibraryManager.library, { self.assertContained('|world|', self.run_js('a.out.js')) self.run_process([EMCC, 'src.cpp', '--pre-js', 'pre.js', '-s', 'EXTRA_EXPORTED_RUNTIME_METHODS=["ENV"]', '-s', 'MODULARIZE=1']) - output = self.run_process(NODE_JS + ['-e', 'require("./a.out.js")();'], stdout=PIPE, stderr=PIPE) + output = self.run_process(config.NODE_JS + ['-e', 'require("./a.out.js")();'], stdout=PIPE, stderr=PIPE) self.assertContained('|world|', output.stdout) def test_warn_no_filesystem(self): @@ -6935,7 +6934,7 @@ int main() { self.assertContained('hello, world!', self.run_js('out.js')) # verify a standalone wasm if standalone: - for engine in WASM_ENGINES: + for engine in config.WASM_ENGINES: print(engine) self.assertContained('hello, world!', self.run_js('out.wasm', engine=engine)) @@ -7016,12 +7015,12 @@ int main() { self.assertContained('no native wasm support detected', out) def test_jsrun(self): - print(NODE_JS) + print(config.NODE_JS) jsrun.WORKING_ENGINES = {} # Test that engine check passes - self.assertTrue(jsrun.check_engine(NODE_JS)) + self.assertTrue(jsrun.check_engine(config.NODE_JS)) # Run it a second time (cache hit) - self.assertTrue(jsrun.check_engine(NODE_JS)) + self.assertTrue(jsrun.check_engine(config.NODE_JS)) # Test that engine check fails bogus_engine = ['/fake/inline4'] @@ -7029,14 +7028,14 @@ int main() { self.assertFalse(jsrun.check_engine(bogus_engine)) # Test the other possible way (list vs string) to express an engine - if type(NODE_JS) is list: - engine2 = NODE_JS[0] + if type(config.NODE_JS) is list: + engine2 = config.NODE_JS[0] else: - engine2 = [NODE_JS] + engine2 = [config.NODE_JS] self.assertTrue(jsrun.check_engine(engine2)) # Test that self.run_js requires the engine - self.run_js(path_from_root('tests', 'hello_world.js'), NODE_JS) + self.run_js(path_from_root('tests', 'hello_world.js'), config.NODE_JS) caught_exit = 0 try: self.run_js(path_from_root('tests', 'hello_world.js'), bogus_engine) @@ -7658,7 +7657,7 @@ T6:(else) !ASSERTIONS""", output) def test_node_js_run_from_different_directory(self): ensure_dir('subdir') self.run_process([EMCC, path_from_root('tests', 'hello_world.c'), '-o', os.path.join('subdir', 'a.js'), '-O3']) - ret = self.run_process(NODE_JS + [os.path.join('subdir', 'a.js')], stdout=PIPE).stdout + ret = self.run_process(config.NODE_JS + [os.path.join('subdir', 'a.js')], stdout=PIPE).stdout self.assertContained('hello, world!', ret) # Tests that a pthreads + modularize build can be run in node js @@ -7679,13 +7678,13 @@ test_module().then((test_module_instance) => { self.run_process([EMCC, path_from_root('tests', 'hello_world.c'), '-o', os.path.join('subdir', 'module.js'), '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2', '-s', 'MODULARIZE=1', '-s', 'EXPORT_NAME=test_module', '-s', 'ENVIRONMENT=worker,node']) # run the module - ret = self.run_process(NODE_JS + ['--experimental-wasm-threads'] + [os.path.join('subdir', moduleLoader)], stdout=PIPE).stdout + ret = self.run_process(config.NODE_JS + ['--experimental-wasm-threads'] + [os.path.join('subdir', moduleLoader)], stdout=PIPE).stdout self.assertContained('hello, world!', ret) @no_windows('node system() does not seem to work, see https://github.com/emscripten-core/emscripten/pull/10547') def test_node_js_system(self): self.run_process([EMCC, '-DENV_NODE', path_from_root('tests', 'system.c'), '-o', 'a.js', '-O3']) - ret = self.run_process(NODE_JS + ['a.js'], stdout=PIPE).stdout + ret = self.run_process(config.NODE_JS + ['a.js'], stdout=PIPE).stdout self.assertContained('OK', ret) def test_is_bitcode(self): @@ -8076,7 +8075,7 @@ int main () { print(' '.join(args)) self.run_process(args) - ret = self.run_process(NODE_JS + ['a.js'], stdout=PIPE).stdout + ret = self.run_process(config.NODE_JS + ['a.js'], stdout=PIPE).stdout self.assertTextDataIdentical('Sum of numbers from 1 to 1000: 500500 (expected 500500)', ret.strip()) check_size('a.js', 150000) @@ -8115,7 +8114,7 @@ int main () { self.assertIn(b'emscripten_metadata', open('out.wasm', 'rb').read()) # make sure wasm executes correctly - ret = self.run_process(NODE_JS + ['a.out.js'], stdout=PIPE).stdout + ret = self.run_process(config.NODE_JS + ['a.out.js'], stdout=PIPE).stdout self.assertContained('hello, world!\n', ret) @parameterized({ @@ -8606,7 +8605,7 @@ int main(void) { self.do_smart_test(path_from_root('tests', 'other', 'test_asan_pthread_stubs.c'), emcc_args=['-fsanitize=address', '-sALLOW_MEMORY_GROWTH=1', '-sINITIAL_MEMORY=314572800']) def test_proxy_to_pthread_stack(self): - with js_engines_modify([NODE_JS + ['--experimental-wasm-threads', '--experimental-wasm-bulk-memory']]): + with js_engines_modify([config.NODE_JS + ['--experimental-wasm-threads', '--experimental-wasm-bulk-memory']]): self.do_smart_test(path_from_root('tests', 'other', 'test_proxy_to_pthread_stack.c'), ['success'], emcc_args=['-s', 'USE_PTHREADS', '-s', 'PROXY_TO_PTHREAD', '-s', 'TOTAL_STACK=1048576']) @@ -8676,7 +8675,7 @@ int main(void) { def test_INCOMING_MODULE_JS_API(self): def test(args): self.run_process([EMCC, path_from_root('tests', 'hello_world.c'), '-O3', '--closure', '1'] + args) - for engine in JS_ENGINES: + for engine in config.JS_ENGINES: self.assertContained('hello, world!', self.run_js('a.out.js', engine=engine)) with open('a.out.js') as f: # ignore \r which on windows can increase the size @@ -8977,7 +8976,7 @@ Module.arguments has been replaced with plain arguments_ (the initial value can js = f.read() with open('a.out.js', 'w') as f: f.write('var WebAssembly = null;\n' + js) - for engine in JS_ENGINES: + for engine in config.JS_ENGINES: self.assertContained('hello, world!', self.run_js('a.out.js', engine=engine)) def test_empty_output_extension(self): @@ -9164,7 +9163,7 @@ int main() { self.run_process([EMCC, path_from_root('tests', 'other', 'test_standalone_syscalls.cpp'), '-o', 'test.wasm']) with open(path_from_root('tests', 'other', 'test_standalone_syscalls.out')) as f: expected = f.read() - for engine in WASM_ENGINES: + for engine in config.WASM_ENGINES: self.assertContained(expected, self.run_js('test.wasm', engine)) @requires_native_clang @@ -9364,7 +9363,7 @@ int main() { self.add_pre_run('console.log("calling foo"); Module["_foo"]();') create_test_file('foo.c', '#include \nint foo() { puts("foo called"); return 3; }') self.build('foo.c') - err = self.expect_fail(NODE_JS + ['foo.js'], stdout=PIPE) + err = self.expect_fail(config.NODE_JS + ['foo.js'], stdout=PIPE) self.assertContained('native function `foo` called before runtime initialization', err) def test_native_call_after_exit(self): @@ -9373,7 +9372,7 @@ int main() { self.add_on_exit('console.log("calling main again"); Module["_main"]();') create_test_file('foo.c', '#include \nint main() { puts("foo called"); return 0; }') self.build('foo.c') - err = self.expect_fail(NODE_JS + ['foo.js'], stdout=PIPE) + err = self.expect_fail(config.NODE_JS + ['foo.js'], stdout=PIPE) self.assertContained('native function `main` called after runtime exit', err) def test_metadce_wasm2js_i64(self): diff --git a/tests/test_sanity.py b/tests/test_sanity.py index a5e3be9c4..d0d0c92ad 100644 --- a/tests/test_sanity.py +++ b/tests/test_sanity.py @@ -14,9 +14,10 @@ from subprocess import PIPE, STDOUT from runner import RunnerCore, path_from_root, env_modify, chdir from runner import create_test_file, ensure_dir, make_executable -from tools.shared import NODE_JS, PYTHON, EMCC, SPIDERMONKEY_ENGINE, V8_ENGINE -from tools.shared import CONFIG_FILE, EM_CONFIG, LLVM_ROOT, CANONICAL_TEMP_DIR -from tools.shared import try_delete +from tools.config import config_file, EM_CONFIG +from tools.shared import PYTHON, EMCC +from tools.shared import CANONICAL_TEMP_DIR +from tools.shared import try_delete, config from tools.shared import EXPECTED_LLVM_VERSION, Cache from tools import shared, system_libs @@ -25,27 +26,27 @@ commands = [[EMCC], [PYTHON, path_from_root('tests', 'runner.py'), 'blahblah']] def restore(): - shutil.copyfile(CONFIG_FILE + '_backup', CONFIG_FILE) + shutil.copyfile(config_file + '_backup', config_file) # restore the config file and set it up for our uses def restore_and_set_up(): restore() - with open(CONFIG_FILE, 'a') as f: + with open(config_file, 'a') as f: # make LLVM_ROOT sensitive to the LLVM env var, as we test that - f.write('LLVM_ROOT = "%s"\n' % LLVM_ROOT) + f.write('LLVM_ROOT = "%s"\n' % config.LLVM_ROOT) # unfreeze the cache, so we can test that f.write('FROZEN_CACHE = False\n') # wipe the config and sanity files, creating a blank slate def wipe(): - try_delete(CONFIG_FILE) + try_delete(config_file) try_delete(SANITY_FILE) def add_to_config(content): - with open(CONFIG_FILE, 'a') as f: + with open(config_file, 'a') as f: f.write('\n' + content + '\n') @@ -54,7 +55,7 @@ def get_basic_config(): LLVM_ROOT = "%s" BINARYEN_ROOT = "%s" NODE_JS = %s -''' % (LLVM_ROOT, shared.BINARYEN_ROOT, NODE_JS) +''' % (config.LLVM_ROOT, config.BINARYEN_ROOT, config.NODE_JS) def make_fake_wasm_opt(filename, version): @@ -122,15 +123,15 @@ class sanity(RunnerCore): # the sanity checks here del os.environ['EMCC_SKIP_SANITY_CHECK'] - assert os.path.exists(CONFIG_FILE), 'To run these tests, we need a (working!) %s file to already exist' % EM_CONFIG - shutil.copyfile(CONFIG_FILE, CONFIG_FILE + '_backup') + assert os.path.exists(config_file), 'To run these tests, we need a (working!) %s file to already exist' % EM_CONFIG + shutil.copyfile(config_file, config_file + '_backup') print() print('Running sanity checks.') - print('WARNING: This will modify %s, and in theory can break it although it should be restored properly. A backup will be saved in %s_backup' % (CONFIG_FILE, CONFIG_FILE)) + print('WARNING: This will modify %s, and in theory can break it although it should be restored properly. A backup will be saved in %s_backup' % (config_file, config_file)) print() print('>>> the original settings file is:') - print(open(CONFIG_FILE).read().strip()) + print(open(config_file).read().strip()) print('<<<') print() @@ -198,7 +199,7 @@ class sanity(RunnerCore): finally: shutil.rmtree(temp_bin) - default_config = shared.embedded_config + default_config = config.embedded_config self.assertContained('Welcome to Emscripten!', output) self.assertContained('This is the first time any of the Emscripten tools has been run.', output) self.assertContained('A settings file has been copied to %s, at absolute path: %s' % (default_config, default_config), output) @@ -255,7 +256,7 @@ class sanity(RunnerCore): # Fake a different llvm version restore_and_set_up() - with open(CONFIG_FILE, 'a') as f: + with open(config_file, 'a') as f: f.write('LLVM_ROOT = "' + self.in_dir('fake') + '"') real_version_x, real_version_y = (int(x) for x in EXPECTED_LLVM_VERSION.split('.')) @@ -302,7 +303,7 @@ class sanity(RunnerCore): # Fake a different node version restore_and_set_up() - with open(CONFIG_FILE, 'a') as f: + with open(config_file, 'a') as f: f.write('NODE_JS = "' + self.in_dir('fake', 'nodejs') + '"') ensure_dir('fake') @@ -321,7 +322,7 @@ echo "%s" else %s $@ fi -''' % (version, NODE_JS)) +''' % (version, config.NODE_JS)) f.close() make_executable(self.in_dir('fake', 'nodejs')) if not succeed: @@ -380,7 +381,7 @@ fi self.assertNotContained(SANITY_FAIL_MESSAGE, output) # emcc should also check sanity if the file is outdated - open(CONFIG_FILE, 'a').write('# extra stuff\n') + open(config_file, 'a').write('# extra stuff\n') output = self.check_working(EMCC) self.assertContained(SANITY_MESSAGE, output) self.assertNotContained(SANITY_FAIL_MESSAGE, output) @@ -509,14 +510,14 @@ fi fd, custom_config_filename = tempfile.mkstemp(prefix='.emscripten_config_') - orig_config = open(CONFIG_FILE, 'r').read() + orig_config = open(config_file, 'r').read() # Move the ~/.emscripten to a custom location. with os.fdopen(fd, "w") as f: f.write(get_basic_config()) # Make a syntax error in the original config file so that attempting to access it would fail. - open(CONFIG_FILE, 'w').write('asdfasdfasdfasdf\n\'\'\'' + orig_config) + open(config_file, 'w').write('asdfasdfasdfasdf\n\'\'\'' + orig_config) temp_dir = tempfile.mkdtemp(prefix='emscripten_temp_') @@ -605,11 +606,11 @@ fi ensure_dir(test_path) with env_modify({'EM_IGNORE_SANITY': '1'}): - jsengines = [('d8', V8_ENGINE), - ('d8_g', V8_ENGINE), - ('js', SPIDERMONKEY_ENGINE), - ('node', NODE_JS), - ('nodejs', NODE_JS)] + jsengines = [('d8', config.V8_ENGINE), + ('d8_g', config.V8_ENGINE), + ('js', config.SPIDERMONKEY_ENGINE), + ('node', config.NODE_JS), + ('nodejs', config.NODE_JS)] for filename, engine in jsengines: if type(engine) is list: engine = engine[0] @@ -655,7 +656,7 @@ fi Cache.erase() def make_fake(report): - with open(CONFIG_FILE, 'a') as f: + with open(config_file, 'a') as f: f.write('LLVM_ROOT = "' + self.in_dir('fake', 'bin') + '"\n') # BINARYEN_ROOT needs to exist in the config, even though this test # doesn't actually use it. @@ -679,11 +680,11 @@ fi # with no binaryen root, an error is shown restore_and_set_up() - open(CONFIG_FILE, 'a').write('\nBINARYEN_ROOT = ""\n') - self.check_working([EMCC, path_from_root('tests', 'hello_world.c')], 'BINARYEN_ROOT is set to empty value in %s' % CONFIG_FILE) + open(config_file, 'a').write('\nBINARYEN_ROOT = ""\n') + self.check_working([EMCC, path_from_root('tests', 'hello_world.c')], 'BINARYEN_ROOT is set to empty value in %s' % config_file) - open(CONFIG_FILE, 'a').write('\ndel BINARYEN_ROOT\n') - self.check_working([EMCC, path_from_root('tests', 'hello_world.c')], 'BINARYEN_ROOT is not defined in %s' % CONFIG_FILE) + open(config_file, 'a').write('\ndel BINARYEN_ROOT\n') + self.check_working([EMCC, path_from_root('tests', 'hello_world.c')], 'BINARYEN_ROOT is not defined in %s' % config_file) def test_embuilder_force(self): restore_and_set_up() @@ -698,14 +699,14 @@ fi # the --lto flag makes us build wasm-bc self.do([EMCC, '--clear-cache']) self.run_process([EMBUILDER, 'build', 'libemmalloc']) - self.assertExists(os.path.join(shared.CACHE, 'wasm')) + self.assertExists(os.path.join(config.CACHE, 'wasm')) self.do([EMCC, '--clear-cache']) self.run_process([EMBUILDER, 'build', 'libemmalloc', '--lto']) - self.assertExists(os.path.join(shared.CACHE, 'wasm-lto')) + self.assertExists(os.path.join(config.CACHE, 'wasm-lto')) def test_binaryen_version(self): restore_and_set_up() - with open(CONFIG_FILE, 'a') as f: + with open(config_file, 'a') as f: f.write('\nBINARYEN_ROOT = "' + self.in_dir('fake') + '"') make_fake_wasm_opt(self.in_dir('fake', 'bin', 'wasm-opt'), 'foo') diff --git a/tests/test_sockets.py b/tests/test_sockets.py index 30d09ecef..b3189a6c4 100644 --- a/tests/test_sockets.py +++ b/tests/test_sockets.py @@ -25,12 +25,12 @@ except Exception: pass import clang_native from runner import BrowserCore, no_windows, chdir -from tools import shared -from tools.shared import PYTHON, EMCC, NODE_JS, path_from_root, WINDOWS, run_process, JS_ENGINES, CLANG_CC +from tools import shared, config +from tools.shared import PYTHON, EMCC, path_from_root, WINDOWS, run_process, CLANG_CC npm_checked = False -NPM = os.path.join(os.path.dirname(NODE_JS[0]), 'npm.cmd' if WINDOWS else 'npm') +NPM = os.path.join(os.path.dirname(config.NODE_JS[0]), 'npm.cmd' if WINDOWS else 'npm') def clean_processes(processes): @@ -113,7 +113,7 @@ class CompiledServerHarness(): # the ws module is installed global npm_checked if not npm_checked: - child = run_process(NODE_JS + ['-e', 'require("ws");'], check=False) + child = run_process(config.NODE_JS + ['-e', 'require("ws");'], check=False) assert child.returncode == 0, '"ws" node module not found. you may need to run npm install' npm_checked = True @@ -121,7 +121,7 @@ class CompiledServerHarness(): proc = run_process([EMCC, '-Werror', path_from_root('tests', self.filename), '-o', 'server.js', '-DSOCKK=%d' % self.listen_port] + self.args) print('Socket server build: out:', proc.stdout or '', '/ err:', proc.stderr or '') - process = Popen(NODE_JS + ['server.js']) + process = Popen(config.NODE_JS + ['server.js']) self.processes.append(process) def __exit__(self, *args, **kwargs): @@ -148,7 +148,7 @@ class BackgroundServerProcess(): def NodeJsWebSocketEchoServerProcess(): - return BackgroundServerProcess(NODE_JS + [path_from_root('tests', 'websocket', 'nodejs_websocket_echo_server.js')]) + return BackgroundServerProcess(config.NODE_JS + [path_from_root('tests', 'websocket', 'nodejs_websocket_echo_server.js')]) def PythonTcpEchoServerProcess(port): @@ -403,7 +403,7 @@ class sockets(BrowserCore): # note: you may need to run this manually yourself, if npm is not in the path, or if you need a version that is not in the path self.run_process([NPM, 'install', path_from_root('tests', 'sockets', 'p2p')]) - broker = Popen(NODE_JS + [path_from_root('tests', 'sockets', 'p2p', 'broker', 'p2p-broker.js')]) + broker = Popen(config.NODE_JS + [path_from_root('tests', 'sockets', 'p2p', 'broker', 'p2p-broker.js')]) expected = '1' self.run_browser(host_outfile, '.', ['/report_result?' + e for e in expected]) @@ -413,7 +413,7 @@ class sockets(BrowserCore): def test_nodejs_sockets_echo(self): # This test checks that sockets work when the client code is run in Node.js # Run with ./runner.py sockets.test_nodejs_sockets_echo - if NODE_JS not in JS_ENGINES: + if config.NODE_JS not in config.JS_ENGINES: self.skipTest('node is not present') sockets_include = '-I' + path_from_root('tests', 'sockets') diff --git a/tools/building.py b/tools/building.py index 6e79b661d..19b5f5083 100644 --- a/tools/building.py +++ b/tools/building.py @@ -20,16 +20,18 @@ from . import diagnostics from . import response_file from . import shared from . import webassembly +from . import config from .toolchain_profiler import ToolchainProfiler from .shared import Settings, CLANG_CC, CLANG_CXX, PYTHON -from .shared import LLVM_NM, EMCC, EMAR, EMXX, EMRANLIB, NODE_JS, WASM_LD, LLVM_AR +from .shared import LLVM_NM, EMCC, EMAR, EMXX, EMRANLIB, WASM_LD, LLVM_AR from .shared import LLVM_LINK, LLVM_OBJCOPY from .shared import try_delete, run_process, check_call, exit_with_error from .shared import configuration, path_from_root, EXPECTED_BINARYEN_VERSION -from .shared import asmjs_mangle, DEBUG, WINDOWS, JAVA -from .shared import EM_BUILD_VERBOSE, TEMP_DIR, print_compiler_stage, BINARYEN_ROOT +from .shared import asmjs_mangle, DEBUG +from .shared import EM_BUILD_VERBOSE, TEMP_DIR, print_compiler_stage from .shared import CANONICAL_TEMP_DIR, LLVM_DWARFDUMP, demangle_c_symbol_name, asbytes -from .shared import get_emscripten_temp_dir, exe_suffix, which, is_c_symbol, shlex_join +from .shared import get_emscripten_temp_dir, exe_suffix, is_c_symbol +from .utils import which, WINDOWS logger = logging.getLogger('building') @@ -289,10 +291,10 @@ def handle_cmake_toolchain(args, env): # Append the Emscripten toolchain file if the user didn't specify one. if not has_substr(args, '-DCMAKE_TOOLCHAIN_FILE'): args.append('-DCMAKE_TOOLCHAIN_FILE=' + path_from_root('cmake', 'Modules', 'Platform', 'Emscripten.cmake')) - node_js = NODE_JS + node_js = config.NODE_JS if not has_substr(args, '-DCMAKE_CROSSCOMPILING_EMULATOR'): - node_js = NODE_JS[0].replace('"', '\"') + node_js = config.NODE_JS[0].replace('"', '\"') args.append('-DCMAKE_CROSSCOMPILING_EMULATOR="%s"' % node_js) # On Windows specify MinGW Makefiles or ninja if we have them and no other @@ -333,7 +335,7 @@ def configure(args, stdout=None, stderr=None, env=None, cflags=[], **kwargs): stdout = None if EM_BUILD_VERBOSE >= 1: stderr = None - print('configure: ' + shlex_join(args), file=sys.stderr) + print('configure: ' + shared.shlex_join(args), file=sys.stderr) run_process(args, stdout=stdout, stderr=stderr, env=env, **kwargs) @@ -869,7 +871,7 @@ def acorn_optimizer(filename, passes, extra_info=None, return_output=False): with open(temp, 'a') as f: f.write('// EXTRA_INFO: ' + extra_info) filename = temp - cmd = NODE_JS + [optimizer, filename] + passes + cmd = config.NODE_JS + [optimizer, filename] + passes # Keep JS code comments intact through the acorn optimization pass so that JSDoc comments # will be carried over to a later Closure run. if Settings.USE_CLOSURE_COMPILER: @@ -899,7 +901,7 @@ def eval_ctors(js_file, binary_file, debug_info=False): # noqa def get_closure_compiler(): # First check if the user configured a specific CLOSURE_COMPILER in thier settings - if shared.CLOSURE_COMPILER: + if config.CLOSURE_COMPILER: return shared.CLOSURE_COMPILER # Otherwise use the one installed vai npm @@ -942,7 +944,7 @@ def closure_compiler(filename, pretty=True, advanced=True, extra_closure_args=No # Closure compiler expects JAVA_HOME to be set *and* java.exe to be in the PATH in order # to enable use the java backend. Without this it will only try the native and JavaScript # versions of the compiler. - java_bin = os.path.dirname(JAVA) + java_bin = os.path.dirname(config.JAVA) if java_bin: def add_to_path(dirname): env['PATH'] = env['PATH'] + os.pathsep + dirname @@ -1569,7 +1571,7 @@ def check_binaryen(bindir): def get_binaryen_bin(): assert Settings.WASM, 'non wasm builds should not ask for binaryen' global binaryen_checked - rtn = os.path.join(BINARYEN_ROOT, 'bin') + rtn = os.path.join(config.BINARYEN_ROOT, 'bin') if not binaryen_checked: check_binaryen(rtn) binaryen_checked = True diff --git a/tools/cache.py b/tools/cache.py index aa2759bbf..64054f32b 100644 --- a/tools/cache.py +++ b/tools/cache.py @@ -6,7 +6,7 @@ import os import shutil import logging -from . import tempfiles, filelock +from . import tempfiles, filelock, config, utils logger = logging.getLogger('cache') @@ -74,7 +74,7 @@ class Cache(object): logger.debug('PID %s released multiprocess file lock to Emscripten cache at %s' % (str(os.getpid()), self.dirname)) def ensure(self): - shared.safe_ensure_dirs(self.dirname) + utils.safe_ensure_dirs(self.dirname) def erase(self): self.acquire_cache_lock() @@ -110,7 +110,7 @@ class Cache(object): if os.path.exists(cachename) and not force: return cachename # it doesn't exist yet, create it - if shared.FROZEN_CACHE: + if config.FROZEN_CACHE: # it's ok to build small .txt marker files like "vanilla" if not shortname.endswith('.txt'): raise Exception('FROZEN_CACHE disallows building system libs: %s' % shortname) @@ -124,7 +124,7 @@ class Cache(object): self.ensure() temp = creator() if os.path.normcase(temp) != os.path.normcase(cachename): - shared.safe_ensure_dirs(os.path.dirname(cachename)) + utils.safe_ensure_dirs(os.path.dirname(cachename)) shutil.copyfile(temp, cachename) logger.info(' - ok') finally: diff --git a/tools/config.py b/tools/config.py new file mode 100644 index 000000000..36dc4b3a2 --- /dev/null +++ b/tools/config.py @@ -0,0 +1,283 @@ +# Copyright 2020 The Emscripten Authors. All rights reserved. +# Emscripten is available under two separate licenses, the MIT license and the +# University of Illinois/NCSA Open Source License. Both these licenses can be +# found in the LICENSE file. + +import os +import sys +import logging +from .utils import path_from_root, exit_with_error, __rootpath__, which + +logger = logging.getLogger('shared') + +# The following class can be overridden by the config file and/or +# environment variables. Specifically any variable whose name +# is in ALL_UPPER_CASE is condifered a valid config file key. +# See parse_config_file below. +EMSCRIPTEN_ROOT = __rootpath__ +NODE_JS = None +BINARYEN_ROOT = None +SPIDERMONKEY_ENGINE = None +V8_ENGINE = None +LLVM_ROOT = None +LLVM_ADD_VERSION = None +CLANG_ADD_VERSION = None +CLOSURE_COMPILER = None +JAVA = None +JS_ENGINE = None +JS_ENGINES = None +WASMER = None +WASMTIME = None +WASM_ENGINES = [] +FROZEN_CACHE = None +CACHE = None +PORTS = None +COMPILER_WRAPPER = None + + +def listify(x): + if type(x) is not list: + return [x] + return x + + +def fix_js_engine(old, new): + if old is None: + return + global JS_ENGINES + JS_ENGINES = [new if x == old else x for x in JS_ENGINES] + return new + + +def root_is_writable(): + return os.access(__rootpath__, os.W_OK) + + +def normalize_config_settings(): + global CACHE, PORTS, JAVA, LLVM_ADD_VERSION, CLANG_ADD_VERSION + global NODE_JS, V8_ENGINE, JS_ENGINE, JS_ENGINES, SPIDERMONKEY_ENGINE, WASM_ENGINES + + # EM_CONFIG stuff + if not JS_ENGINES: + JS_ENGINES = [NODE_JS] + if not JS_ENGINE: + JS_ENGINE = JS_ENGINES[0] + + # Engine tweaks + if SPIDERMONKEY_ENGINE: + new_spidermonkey = SPIDERMONKEY_ENGINE + if '-w' not in str(new_spidermonkey): + new_spidermonkey += ['-w'] + SPIDERMONKEY_ENGINE = fix_js_engine(SPIDERMONKEY_ENGINE, new_spidermonkey) + NODE_JS = fix_js_engine(NODE_JS, listify(NODE_JS)) + V8_ENGINE = fix_js_engine(V8_ENGINE, listify(V8_ENGINE)) + JS_ENGINE = fix_js_engine(JS_ENGINE, listify(JS_ENGINE)) + JS_ENGINES = [listify(engine) for engine in JS_ENGINES] + WASM_ENGINES = [listify(engine) for engine in WASM_ENGINES] + if not CACHE: + if root_is_writable(): + CACHE = path_from_root('cache') + else: + # Use the legacy method of putting the cache in the user's home directory + # if the emscripten root is not writable. + # This is useful mostly for read-only installation and perhaps could + # be removed in the future since such installations should probably be + # setting a specific cache location. + logger.debug('Using home-directory for emscripten cache due to read-only root') + CACHE = os.path.expanduser(os.path.join('~', '.emscripten_cache')) + if not PORTS: + PORTS = os.path.join(CACHE, 'ports') + + if JAVA is None: + logger.debug('JAVA not defined in ' + config_file_location() + ', using "java"') + JAVA = 'java' + + # Tools/paths + if LLVM_ADD_VERSION is None: + LLVM_ADD_VERSION = os.getenv('LLVM_ADD_VERSION') + + if CLANG_ADD_VERSION is None: + CLANG_ADD_VERSION = os.getenv('CLANG_ADD_VERSION') + + +def parse_config_file(): + """Parse the emscripten config file using python's exec. + + Also check EM_ environment variables to override specific config keys. + """ + config = {} + config_text = open(config_file, 'r').read() if config_file else EM_CONFIG + try: + exec(config_text, config) + except Exception as e: + exit_with_error('Error in evaluating %s (at %s): %s, text: %s', EM_CONFIG, config_file, str(e), config_text) + + CONFIG_KEYS = ( + 'NODE_JS', + 'BINARYEN_ROOT', + 'SPIDERMONKEY_ENGINE', + 'V8_ENGINE', + 'LLVM_ROOT', + 'LLVM_ADD_VERSION', + 'CLANG_ADD_VERSION', + 'CLOSURE_COMPILER', + 'JAVA', + 'JS_ENGINE', + 'JS_ENGINES', + 'WASMER', + 'WASMTIME', + 'WASM_ENGINES', + 'FROZEN_CACHE', + 'CACHE', + 'PORTS', + 'COMPILER_WRAPPER', + ) + + # Only propagate certain settings from the config file. + for key in CONFIG_KEYS: + env_var = 'EM_' + key + env_value = os.environ.get(env_var) + if env_value is not None: + globals()[key] = env_value + elif key in config: + globals()[key] = config[key] + + # Certain keys are mandatory + for key in ('LLVM_ROOT', 'NODE_JS', 'BINARYEN_ROOT'): + if key not in config: + exit_with_error('%s is not defined in %s', key, config_file_location()) + if not globals()[key]: + exit_with_error('%s is set to empty value in %s', key, config_file_location()) + + if not NODE_JS: + exit_with_error('NODE_JS is not defined in %s', config_file_location()) + + normalize_config_settings() + + +# Returns the location of the emscripten config file. +def config_file_location(): + # Handle the case where there is no config file at all (i.e. If EM_CONFIG is passed as python code + # direclty on the command line). + if not config_file: + return '' + + return config_file + + +def generate_config(path, first_time=False): + # Note: repr is used to ensure the paths are escaped correctly on Windows. + # The full string is replaced so that the template stays valid Python. + config_file = open(path_from_root('tools', 'settings_template.py')).read().splitlines() + config_file = config_file[3:] # remove the initial comment + config_file = '\n'.join(config_file) + # autodetect some default paths + config_file = config_file.replace('\'{{{ EMSCRIPTEN_ROOT }}}\'', repr(__rootpath__)) + llvm_root = os.path.dirname(which('llvm-dis') or '/usr/bin/llvm-dis') + config_file = config_file.replace('\'{{{ LLVM_ROOT }}}\'', repr(llvm_root)) + + node = which('nodejs') or which('node') or 'node' + config_file = config_file.replace('\'{{{ NODE }}}\'', repr(node)) + + abspath = os.path.abspath(os.path.expanduser(path)) + # write + with open(abspath, 'w') as f: + f.write(config_file) + + if first_time: + print(''' +============================================================================== +Welcome to Emscripten! + +This is the first time any of the Emscripten tools has been run. + +A settings file has been copied to %s, at absolute path: %s + +It contains our best guesses for the important paths, which are: + + LLVM_ROOT = %s + NODE_JS = %s + EMSCRIPTEN_ROOT = %s + +Please edit the file if any of those are incorrect. + +This command will now exit. When you are done editing those paths, re-run it. +============================================================================== +''' % (path, abspath, llvm_root, node, __rootpath__), file=sys.stderr) + + +# Emscripten configuration is done through the --em-config command line option +# or the EM_CONFIG environment variable. If the specified string value contains +# newline or semicolon-separated definitions, then these definitions will be +# used to configure Emscripten. Otherwise, the string is understood to be a +# path to a settings file that contains the required definitions. +# The search order from the config file is as follows: +# 1. Specified on the command line (--em-config) +# 2. Specified via EM_CONFIG environment variable +# 3. Local .emscripten file, if found +# 4. Local .emscripten file, as used by `emsdk --embedded` (two levels above, +# see below) +# 5. User home directory config (~/.emscripten), if found. + +embedded_config = path_from_root('.emscripten') +# For compatibility with `emsdk --embedded` mode also look two levels up. The +# layout of the emsdk puts emcc two levels below emsdk. For exmaple: +# - emsdk/upstream/emscripten/emcc +# - emsdk/emscipten/1.38.31/emcc +# However `emsdk --embedded` stores the config file in the emsdk root. +# Without this check, when emcc is run from within the emsdk in embedded mode +# and the user forgets to first run `emsdk_env.sh` (which sets EM_CONFIG) emcc +# will not see any config file at all and fall back to creating a new/emtpy +# one. +# We could remove this special case if emsdk were to write its embedded config +# file into the emscripten directory itself. +# See: https://github.com/emscripten-core/emsdk/pull/367 +emsdk_root = os.path.dirname(os.path.dirname(path_from_root())) +emsdk_embedded_config = os.path.join(emsdk_root, '.emscripten') +user_home_config = os.path.expanduser('~/.emscripten') + +if '--em-config' in sys.argv: + EM_CONFIG = sys.argv[sys.argv.index('--em-config') + 1] + # And now remove it from sys.argv + skip = False + newargs = [] + for arg in sys.argv: + if not skip and arg != '--em-config': + newargs += [arg] + elif arg == '--em-config': + skip = True + elif skip: + skip = False + sys.argv = newargs + if not os.path.isfile(EM_CONFIG): + if EM_CONFIG.startswith('-'): + exit_with_error('Passed --em-config without an argument. Usage: --em-config /path/to/.emscripten or --em-config LLVM_ROOT=/path;...') + if '=' not in EM_CONFIG: + exit_with_error('File ' + EM_CONFIG + ' passed to --em-config does not exist!') + else: + EM_CONFIG = EM_CONFIG.replace(';', '\n') + '\n' +elif 'EM_CONFIG' in os.environ: + EM_CONFIG = os.environ['EM_CONFIG'] +elif os.path.exists(embedded_config): + EM_CONFIG = embedded_config +elif os.path.exists(emsdk_embedded_config): + EM_CONFIG = emsdk_embedded_config +elif os.path.exists(user_home_config): + EM_CONFIG = user_home_config +else: + if root_is_writable(): + generate_config(embedded_config, first_time=True) + else: + generate_config(user_home_config, first_time=True) + sys.exit(0) + +if '\n' in EM_CONFIG: + config_file = None + logger.debug('config is specified inline without a file') +else: + config_file = os.path.expanduser(EM_CONFIG) + logger.debug('emscripten config is located in ' + config_file) + if not os.path.exists(config_file): + exit_with_error('emscripten config file not found: ' + config_file) + +parse_config_file() diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index 5d1c62c88..071154b9e 100755 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -15,7 +15,7 @@ __rootpath__ = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) sys.path.insert(1, __rootpath__) from tools.toolchain_profiler import ToolchainProfiler -from tools import building +from tools import building, config if __name__ == '__main__': ToolchainProfiler.record_process_start() @@ -101,7 +101,7 @@ class Minifier(object): f.write('\n') f.write('// EXTRA_INFO:' + json.dumps(self.serialize())) - cmd = shared.NODE_JS + [JS_OPTIMIZER, temp_file, 'minifyGlobals', 'noPrintMetadata'] + cmd = config.NODE_JS + [JS_OPTIMIZER, temp_file, 'minifyGlobals', 'noPrintMetadata'] if minify_whitespace: cmd.append('minifyWhitespace') output = shared.run_process(cmd, stdout=subprocess.PIPE).stdout @@ -329,7 +329,7 @@ EMSCRIPTEN_FUNCS(); with ToolchainProfiler.profile_block('run_optimizer'): if len(filenames): - commands = [shared.NODE_JS + [JS_OPTIMIZER, f, 'noPrintMetadata'] + passes for f in filenames] + commands = [config.NODE_JS + [JS_OPTIMIZER, f, 'noPrintMetadata'] + passes for f in filenames] cores = min(cores, len(filenames)) if len(chunks) > 1 and cores >= 2: diff --git a/tools/shared.py b/tools/shared.py index db0f9bf66..649bb6d48 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -25,20 +25,19 @@ if sys.version_info < (3, 6): from .toolchain_profiler import ToolchainProfiler from .tempfiles import try_delete +from .utils import path_from_root, exit_with_error, safe_ensure_dirs, WINDOWS from . import cache, tempfiles, colored_logger from . import diagnostics +from . import config -__rootpath__ = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) -WINDOWS = sys.platform.startswith('win') -MACOS = sys.platform == 'darwin' -LINUX = sys.platform.startswith('linux') DEBUG = int(os.environ.get('EMCC_DEBUG', '0')) EXPECTED_NODE_VERSION = (4, 1, 1) EXPECTED_BINARYEN_VERSION = 98 EXPECTED_LLVM_VERSION = "12.0" SIMD_INTEL_FEATURE_TOWER = ['-msse', '-msse2', '-msse3', '-mssse3', '-msse4.1', '-msse4.2', '-mavx'] SIMD_NEON_FLAGS = ['-mfpu=neon'] +PYTHON = sys.executable # can add %(asctime)s to see timestamps logging.basicConfig(format='%(name)s:%(levelname)s: %(message)s', @@ -65,18 +64,6 @@ diagnostics.add_warning('unused-command-line-argument', shared=True) diagnostics.add_warning('pthreads-mem-growth') -def exit_with_error(msg, *args): - diagnostics.error(msg, *args) - - -def path_from_root(*pathelems): - return os.path.join(__rootpath__, *pathelems) - - -def root_is_writable(): - return os.access(__rootpath__, os.W_OK) - - # TODO(sbc): Investigate switching to shlex.quote def shlex_quote(arg): if ' ' in arg and (not (arg.startswith('"') and arg.endswith('"'))) and (not (arg.startswith("'") and arg.endswith("'"))): @@ -126,43 +113,11 @@ def run_js_tool(filename, jsargs=[], *args, **kw): This is used by emcc to run parts of the build process that are written implemented in javascript. """ - command = NODE_JS + [filename] + jsargs + command = config.NODE_JS + [filename] + jsargs print_compiler_stage(command) return check_call(command, *args, **kw).stdout -# Finds the given executable 'program' in PATH. Operates like the Unix tool 'which'. -def which(program): - def is_exe(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - - if os.path.isabs(program): - if os.path.isfile(program): - return program - - if WINDOWS: - for suffix in ['.exe', '.cmd', '.bat']: - if is_exe(program + suffix): - return program + suffix - - fpath, fname = os.path.split(program) - if fpath: - if is_exe(program): - return program - else: - for path in os.environ["PATH"].split(os.pathsep): - path = path.strip('"') - exe_file = os.path.join(path, program) - if is_exe(exe_file): - return exe_file - if WINDOWS: - for suffix in ('.exe', '.cmd', '.bat'): - if is_exe(exe_file + suffix): - return exe_file + suffix - - return None - - # Only used by tests and by ctor_evaller.py. Once fastcomp is removed # this can most likely be moved into the tests/jsrun.py. def timeout_run(proc, timeout=None, full_output=False, check=True): @@ -182,174 +137,16 @@ def timeout_run(proc, timeout=None, full_output=False, check=True): return '\n'.join(out) if full_output else out[0] -def generate_config(path, first_time=False): - # Note: repr is used to ensure the paths are escaped correctly on Windows. - # The full string is replaced so that the template stays valid Python. - config_file = open(path_from_root('tools', 'settings_template.py')).read().splitlines() - config_file = config_file[3:] # remove the initial comment - config_file = '\n'.join(config_file) - # autodetect some default paths - config_file = config_file.replace('\'{{{ EMSCRIPTEN_ROOT }}}\'', repr(EMSCRIPTEN_ROOT)) - llvm_root = os.path.dirname(which('llvm-dis') or '/usr/bin/llvm-dis') - config_file = config_file.replace('\'{{{ LLVM_ROOT }}}\'', repr(llvm_root)) - - node = which('nodejs') or which('node') or 'node' - config_file = config_file.replace('\'{{{ NODE }}}\'', repr(node)) - - abspath = os.path.abspath(os.path.expanduser(path)) - # write - with open(abspath, 'w') as f: - f.write(config_file) - - if first_time: - print(f''' -============================================================================== -Welcome to Emscripten! - -This is the first time any of the Emscripten tools has been run. - -A settings file has been copied to {path}, at absolute path: {abspath} - -It contains our best guesses for the important paths, which are: - - LLVM_ROOT = {llvm_root} - NODE_JS = {node} - EMSCRIPTEN_ROOT = {EMSCRIPTEN_ROOT} - -Please edit the file if any of those are incorrect. - -This command will now exit. When you are done editing those paths, re-run it. -============================================================================== -''', file=sys.stderr) - - -def parse_config_file(): - """Parse the emscripten config file using python's exec. - - Also also EM_ environment variables to override specific config keys. - """ - config = {} - config_text = open(CONFIG_FILE, 'r').read() if CONFIG_FILE else EM_CONFIG - try: - exec(config_text, config) - except Exception as e: - exit_with_error('Error in evaluating %s (at %s): %s, text: %s', EM_CONFIG, CONFIG_FILE, str(e), config_text) - - CONFIG_KEYS = ( - 'NODE_JS', - 'BINARYEN_ROOT', - 'SPIDERMONKEY_ENGINE', - 'V8_ENGINE', - 'LLVM_ROOT', - 'LLVM_ADD_VERSION', - 'CLANG_ADD_VERSION', - 'CLOSURE_COMPILER', - 'JAVA', - 'JS_ENGINE', - 'JS_ENGINES', - 'WASMER', - 'WASMTIME', - 'WASM_ENGINES', - 'FROZEN_CACHE', - 'CACHE', - 'PORTS', - 'COMPILER_WRAPPER', - ) - - # Only propagate certain settings from the config file. - for key in CONFIG_KEYS: - env_var = 'EM_' + key - env_value = os.environ.get(env_var) - if env_value is not None: - globals()[key] = env_value - elif key in config: - globals()[key] = config[key] - - # Certain keys are mandatory - for key in ('LLVM_ROOT', 'NODE_JS', 'BINARYEN_ROOT'): - if key not in config: - exit_with_error('%s is not defined in %s', key, config_file_location()) - if not globals()[key]: - exit_with_error('%s is set to empty value in %s', key, config_file_location()) - - if not NODE_JS: - exit_with_error('NODE_JS is not defined in %s', config_file_location()) - - -def listify(x): - if type(x) is not list: - return [x] - return x - - -def fix_js_engine(old, new): - global JS_ENGINES - if old is None: - return - JS_ENGINES = [new if x == old else x for x in JS_ENGINES] - return new - - def get_npm_cmd(name): if WINDOWS: cmd = [path_from_root('node_modules', '.bin', name + '.cmd')] else: - cmd = NODE_JS + [path_from_root('node_modules', '.bin', name)] + cmd = config.NODE_JS + [path_from_root('node_modules', '.bin', name)] if not os.path.exists(cmd[-1]): exit_with_error('%s was not found! Please run "npm install" in Emscripten root directory to set up npm dependencies' % name) return cmd -def normalize_config_settings(): - global CACHE, PORTS, JAVA - global NODE_JS, V8_ENGINE, JS_ENGINE, JS_ENGINES, SPIDERMONKEY_ENGINE, WASM_ENGINES - - # EM_CONFIG stuff - if not JS_ENGINES: - JS_ENGINES = [NODE_JS] - if not JS_ENGINE: - JS_ENGINE = JS_ENGINES[0] - - # Engine tweaks - if SPIDERMONKEY_ENGINE: - new_spidermonkey = SPIDERMONKEY_ENGINE - if '-w' not in str(new_spidermonkey): - new_spidermonkey += ['-w'] - SPIDERMONKEY_ENGINE = fix_js_engine(SPIDERMONKEY_ENGINE, new_spidermonkey) - NODE_JS = fix_js_engine(NODE_JS, listify(NODE_JS)) - V8_ENGINE = fix_js_engine(V8_ENGINE, listify(V8_ENGINE)) - JS_ENGINE = fix_js_engine(JS_ENGINE, listify(JS_ENGINE)) - JS_ENGINES = [listify(engine) for engine in JS_ENGINES] - WASM_ENGINES = [listify(engine) for engine in WASM_ENGINES] - if not CACHE: - if root_is_writable(): - CACHE = path_from_root('cache') - else: - # Use the legacy method of putting the cache in the user's home directory - # if the emscripten root is not writable. - # This is useful mostly for read-only installation and perhaps could - # be removed in the future since such installations should probably be - # setting a specific cache location. - logger.debug('Using home-directory for emscripten cache due to read-only root') - CACHE = os.path.expanduser(os.path.join('~', '.emscripten_cache')) - if not PORTS: - PORTS = os.path.join(CACHE, 'ports') - - if JAVA is None: - logger.debug('JAVA not defined in ' + config_file_location() + ', using "java"') - JAVA = 'java' - - -# Returns the location of the emscripten config file. -def config_file_location(): - # Handle the case where there is no config file at all (i.e. If EM_CONFIG is passed as python code - # direclty on the command line). - if not CONFIG_FILE: - return '' - - return CONFIG_FILE - - def get_clang_version(): if not hasattr(get_clang_version, 'found_version'): if not os.path.exists(CLANG_CC): @@ -394,7 +191,7 @@ def check_llvm(): def get_node_directory(): - return os.path.dirname(NODE_JS[0] if type(NODE_JS) is list else NODE_JS) + return os.path.dirname(config.NODE_JS[0] if type(config.NODE_JS) is list else config.NODE_JS) # When we run some tools from npm (closure, html-minifier-terser), those @@ -408,7 +205,7 @@ def env_with_node_in_path(): def check_node_version(): try: - actual = run_process(NODE_JS + ['--version'], stdout=PIPE).stdout.strip() + actual = run_process(config.NODE_JS + ['--version'], stdout=PIPE).stdout.strip() version = tuple(map(int, actual.replace('v', '').replace('-pre', '').split('.'))) except Exception as e: diagnostics.warning('version-check', 'cannot check node version: %s', e) @@ -431,12 +228,12 @@ def set_version_globals(): def generate_sanity(): - sanity_file_content = EMSCRIPTEN_VERSION + '|' + LLVM_ROOT + '|' + get_clang_version() - if CONFIG_FILE: - config = open(CONFIG_FILE).read() + sanity_file_content = EMSCRIPTEN_VERSION + '|' + config.LLVM_ROOT + '|' + get_clang_version() + if config.config_file: + config_data = open(config.config_file).read() else: - config = EM_CONFIG - checksum = binascii.crc32(config.encode()) + config_data = config.EM_CONFIG + checksum = binascii.crc32(config_data.encode()) sanity_file_content += '|%#x\n' % checksum return sanity_file_content @@ -446,14 +243,14 @@ def perform_sanity_checks(): with ToolchainProfiler.profile_block('sanity compiler_engine'): try: - run_process(NODE_JS + ['-e', 'console.log("hello")'], stdout=PIPE) + run_process(config.NODE_JS + ['-e', 'console.log("hello")'], stdout=PIPE) except Exception as e: - exit_with_error('The configured node executable (%s) does not seem to work, check the paths in %s (%s)', NODE_JS, config_file_location, str(e)) + exit_with_error('The configured node executable (%s) does not seem to work, check the paths in %s (%s)', config.NODE_JS, config.config_file_location(), str(e)) with ToolchainProfiler.profile_block('sanity LLVM'): for cmd in [CLANG_CC, LLVM_AR, LLVM_NM]: if not os.path.exists(cmd) and not os.path.exists(cmd + '.exe'): # .exe extension required for Windows - exit_with_error('Cannot find %s, check the paths in %s', cmd, EM_CONFIG) + exit_with_error('Cannot find %s, check the paths in %s', cmd, config.EM_CONFIG) def check_sanity(force=False): @@ -472,7 +269,7 @@ def check_sanity(force=False): os.environ['EMCC_SKIP_SANITY_CHECK'] = '1' with ToolchainProfiler.profile_block('sanity'): check_llvm_version() - if not CONFIG_FILE: + if not config.config_file: return # config stored directly in EM_CONFIG => skip sanity checks expected = generate_sanity() @@ -482,7 +279,7 @@ def check_sanity(force=False): if sanity_data != expected: logger.debug('old sanity: %s' % sanity_data) logger.debug('new sanity: %s' % expected) - if FROZEN_CACHE: + if config.FROZEN_CACHE: logger.info('(Emscripten: config changed, cache may need to be cleared, but FROZEN_CACHE is set)') else: logger.info('(Emscripten: config changed, clearing cache)') @@ -523,19 +320,19 @@ def check_sanity(force=False): # Some distributions ship with multiple llvm versions so they add # the version to the binaries, cope with that def build_llvm_tool_path(tool): - if LLVM_ADD_VERSION: - return os.path.join(LLVM_ROOT, tool + "-" + LLVM_ADD_VERSION) + if config.LLVM_ADD_VERSION: + return os.path.join(config.LLVM_ROOT, tool + "-" + config.LLVM_ADD_VERSION) else: - return os.path.join(LLVM_ROOT, tool) + return os.path.join(config.LLVM_ROOT, tool) # Some distributions ship with multiple clang versions so they add # the version to the binaries, cope with that def build_clang_tool_path(tool): - if CLANG_ADD_VERSION: - return os.path.join(LLVM_ROOT, tool + "-" + CLANG_ADD_VERSION) + if config.CLANG_ADD_VERSION: + return os.path.join(config.LLVM_ROOT, tool + "-" + config.CLANG_ADD_VERSION) else: - return os.path.join(LLVM_ROOT, tool) + return os.path.join(config.LLVM_ROOT, tool) def exe_suffix(cmd): @@ -559,16 +356,6 @@ def replace_or_append_suffix(filename, new_suffix): return replace_suffix(filename, new_suffix) if Settings.MINIMAL_RUNTIME else filename + new_suffix -def safe_ensure_dirs(dirname): - try: - os.makedirs(dirname) - except OSError: - # Python 2 compatibility: makedirs does not support exist_ok parameter - # Ignore error for already existing dirname as exist_ok does - if not os.path.isdir(dirname): - raise - - # Temp dir. Create a random one, unless EMCC_DEBUG is set, in which case use the canonical # temp directory (TEMP_DIR/emscripten_temp). def get_emscripten_temp_dir(): @@ -605,7 +392,7 @@ class Configuration(object): try: safe_ensure_dirs(self.EMSCRIPTEN_TEMP_DIR) except Exception as e: - exit_with_error(str(e) + 'Could not create canonical temp dir. Check definition of TEMP_DIR in ' + config_file_location()) + exit_with_error(str(e) + 'Could not create canonical temp dir. Check definition of TEMP_DIR in ' + config.config_file_location()) def get_temp_files(self): return tempfiles.TempFiles( @@ -622,7 +409,7 @@ def apply_configuration(): def get_llvm_target(): - return LLVM_TARGET + return 'wasm32-unknown-emscripten' def emsdk_ldflags(user_args): @@ -957,7 +744,7 @@ def asmjs_mangle(name): def reconfigure_cache(): global Cache - Cache = cache.Cache(CACHE) + Cache = cache.Cache(config.CACHE) class JS(object): @@ -1183,112 +970,10 @@ def read_and_preprocess(filename, expand_macros=False): # file. TODO(sbc): We should try to reduce that amount we do here and instead # have consumers explicitly call initialization functions. -# Emscripten configuration is done through the --em-config command line option -# or the EM_CONFIG environment variable. If the specified string value contains -# newline or semicolon-separated definitions, then these definitions will be -# used to configure Emscripten. Otherwise, the string is understood to be a -# path to a settings file that contains the required definitions. -# The search order from the config file is as follows: -# 1. Specified on the command line (--em-config) -# 2. Specified via EM_CONFIG environment variable -# 3. Local .emscripten file, if found -# 4. Local .emscripten file, as used by `emsdk --embedded` (two levels above, -# see below) -# 5. User home directory config (~/.emscripten), if found. - -embedded_config = path_from_root('.emscripten') -# For compatibility with `emsdk --embedded` mode also look two levels up. The -# layout of the emsdk puts emcc two levels below emsdk. For exmaple: -# - emsdk/upstream/emscripten/emcc -# - emsdk/emscipten/1.38.31/emcc -# However `emsdk --embedded` stores the config file in the emsdk root. -# Without this check, when emcc is run from within the emsdk in embedded mode -# and the user forgets to first run `emsdk_env.sh` (which sets EM_CONFIG) emcc -# will not see any config file at all and fall back to creating a new/emtpy -# one. -# We could remove this special case if emsdk were to write its embedded config -# file into the emscripten directory itself. -# See: https://github.com/emscripten-core/emsdk/pull/367 -emsdk_root = os.path.dirname(os.path.dirname(__rootpath__)) -emsdk_embedded_config = os.path.join(emsdk_root, '.emscripten') -user_home_config = os.path.expanduser('~/.emscripten') - -EMSCRIPTEN_ROOT = __rootpath__ - -if '--em-config' in sys.argv: - EM_CONFIG = sys.argv[sys.argv.index('--em-config') + 1] - # And now remove it from sys.argv - skip = False - newargs = [] - for arg in sys.argv: - if not skip and arg != '--em-config': - newargs += [arg] - elif arg == '--em-config': - skip = True - elif skip: - skip = False - sys.argv = newargs - if not os.path.isfile(EM_CONFIG): - if EM_CONFIG.startswith('-'): - exit_with_error('Passed --em-config without an argument. Usage: --em-config /path/to/.emscripten or --em-config LLVM_ROOT=/path;...') - if '=' not in EM_CONFIG: - exit_with_error('File ' + EM_CONFIG + ' passed to --em-config does not exist!') - else: - EM_CONFIG = EM_CONFIG.replace(';', '\n') + '\n' -elif 'EM_CONFIG' in os.environ: - EM_CONFIG = os.environ['EM_CONFIG'] -elif os.path.exists(embedded_config): - EM_CONFIG = embedded_config -elif os.path.exists(emsdk_embedded_config): - EM_CONFIG = emsdk_embedded_config -elif os.path.exists(user_home_config): - EM_CONFIG = user_home_config -else: - if root_is_writable(): - generate_config(embedded_config, first_time=True) - else: - generate_config(user_home_config, first_time=True) - sys.exit(0) - -PYTHON = sys.executable - -# The following globals can be overridden by the config file. -# See parse_config_file below. -NODE_JS = None -BINARYEN_ROOT = None -SPIDERMONKEY_ENGINE = None -V8_ENGINE = None -LLVM_ROOT = None -LLVM_ADD_VERSION = None -CLANG_ADD_VERSION = None -CLOSURE_COMPILER = None -JAVA = None -JS_ENGINE = None -JS_ENGINES = [] -WASMER = None -WASMTIME = None -WASM_ENGINES = [] -CACHE = None -PORTS = None -FROZEN_CACHE = False -COMPILER_WRAPPER = None - # Emscripten compiler spawns other processes, which can reimport shared.py, so # make sure that those child processes get the same configuration file by # setting it to the currently active environment. -os.environ['EM_CONFIG'] = EM_CONFIG - -if '\n' in EM_CONFIG: - CONFIG_FILE = None - logger.debug('EM_CONFIG is specified inline without a file') -else: - CONFIG_FILE = os.path.expanduser(EM_CONFIG) - logger.debug('EM_CONFIG is located in ' + CONFIG_FILE) - if not os.path.exists(CONFIG_FILE): - exit_with_error('emscripten config file not found: ' + CONFIG_FILE) - -parse_config_file() -normalize_config_settings() +os.environ['EM_CONFIG'] = config.EM_CONFIG # Verbosity level control for any intermediate subprocess spawns from the compiler. Useful for internal debugging. # 0: disabled. @@ -1300,13 +985,6 @@ TRACK_PROCESS_SPAWNS = EM_BUILD_VERBOSE >= 3 set_version_globals() -# Tools/paths -if LLVM_ADD_VERSION is None: - LLVM_ADD_VERSION = os.getenv('LLVM_ADD_VERSION') - -if CLANG_ADD_VERSION is None: - CLANG_ADD_VERSION = os.getenv('CLANG_ADD_VERSION') - CLANG_CC = os.path.expanduser(build_clang_tool_path(exe_suffix('clang'))) CLANG_CXX = os.path.expanduser(build_clang_tool_path(exe_suffix('clang++'))) LLVM_LINK = build_llvm_tool_path(exe_suffix('llvm-link')) @@ -1328,11 +1006,8 @@ FILE_PACKAGER = path_from_root('tools', 'file_packager.py') apply_configuration() -# Target choice. -LLVM_TARGET = 'wasm32-unknown-emscripten' - Settings = SettingsManager() verify_settings() -Cache = cache.Cache(CACHE) +Cache = cache.Cache(config.CACHE) PRINT_STAGES = int(os.getenv('EMCC_VERBOSE', '0')) diff --git a/tools/system_libs.py b/tools/system_libs.py index 45ce63694..72c1f9edb 100755 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -16,7 +16,7 @@ import tarfile import zipfile from glob import iglob -from . import shared, building, ports +from . import shared, building, ports, config from tools.shared import mangle_c_symbol_name, demangle_c_symbol_name stdout = None @@ -1716,7 +1716,7 @@ class Ports(object): @staticmethod def get_dir(): - dirname = shared.PORTS + dirname = config.PORTS shared.safe_ensure_dirs(dirname) return dirname diff --git a/tools/utils.py b/tools/utils.py new file mode 100644 index 000000000..6229f524b --- /dev/null +++ b/tools/utils.py @@ -0,0 +1,64 @@ +# Copyright 2020 The Emscripten Authors. All rights reserved. +# Emscripten is available under two separate licenses, the MIT license and the +# University of Illinois/NCSA Open Source License. Both these licenses can be +# found in the LICENSE file. + +import os +import sys + +from . import diagnostics + +__rootpath__ = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) +WINDOWS = sys.platform.startswith('win') +MACOS = sys.platform == 'darwin' +LINUX = sys.platform.startswith('linux') + + +def exit_with_error(msg, *args): + diagnostics.error(msg, *args) + + +def path_from_root(*pathelems): + return os.path.join(__rootpath__, *pathelems) + + +def safe_ensure_dirs(dirname): + try: + os.makedirs(dirname) + except OSError: + # Python 2 compatibility: makedirs does not support exist_ok parameter + # Ignore error for already existing dirname as exist_ok does + if not os.path.isdir(dirname): + raise + + +# Finds the given executable 'program' in PATH. Operates like the Unix tool 'which'. +def which(program): + def is_exe(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + if os.path.isabs(program): + if os.path.isfile(program): + return program + + if WINDOWS: + for suffix in ['.exe', '.cmd', '.bat']: + if is_exe(program + suffix): + return program + suffix + + fpath, fname = os.path.split(program) + if fpath: + if is_exe(program): + return program + else: + for path in os.environ["PATH"].split(os.pathsep): + path = path.strip('"') + exe_file = os.path.join(path, program) + if is_exe(exe_file): + return exe_file + if WINDOWS: + for suffix in ('.exe', '.cmd', '.bat'): + if is_exe(exe_file + suffix): + return exe_file + suffix + + return None diff --git a/tools/wasm2c.py b/tools/wasm2c.py index 727cd0230..b3750c071 100644 --- a/tools/wasm2c.py +++ b/tools/wasm2c.py @@ -6,7 +6,7 @@ import os import re -from tools.shared import Settings, path_from_root, unsuffixed, NODE_JS, check_call, exit_with_error +from tools.shared import Settings, path_from_root, unsuffixed, config, check_call, exit_with_error # map an emscripten-style signature letter to a wasm2c C type @@ -75,7 +75,7 @@ def get_func_types(code): def do_wasm2c(infile): assert Settings.STANDALONE_WASM - WASM2C = NODE_JS + [path_from_root('node_modules', 'wasm2c', 'wasm2c.js')] + WASM2C = config.NODE_JS + [path_from_root('node_modules', 'wasm2c', 'wasm2c.js')] WASM2C_DIR = path_from_root('node_modules', 'wasm2c') c_file = unsuffixed(infile) + '.wasm.c' h_file = unsuffixed(infile) + '.wasm.h'