Add support for potentially very large exported function response files.

Avoid file creation overhead by conditionally creating the exported response file when EXPORTED_FUNCTIONS is > 8k.

Avoid file creation overhead where possible. Use the file API only when EXPORTED_FUNCTIONS is > 8k.

emcc

@@ -858,8 +858,12 @@ try:
   # Apply -s settings in newargs here (after optimization levels, so they can override them)
   for change in settings_changes:
     key, value = change.split('=')
+    original_exported_response = False
+
     if value[0] == '@':
       if key not in DEFERRED_REPONSE_FILES:
+        if key == 'EXPORTED_FUNCTIONS':
+          original_exported_response = value
         value = open(value[1:]).read()
       else:
         value = '"' + value + '"'
@@ -867,7 +871,8 @@ try:
       value = value.replace('\\', '\\\\')
     exec('shared.Settings.' + key + ' = ' + value)
     if key == 'EXPORTED_FUNCTIONS':
-      shared.Settings.ORIGINAL_EXPORTED_FUNCTIONS = shared.Settings.EXPORTED_FUNCTIONS[:] # used for warnings in emscripten.py
+      # used for warnings in emscripten.py
+      shared.Settings.ORIGINAL_EXPORTED_FUNCTIONS = original_exported_response or shared.Settings.EXPORTED_FUNCTIONS[:]
 
   try:
     assert shared.Settings.ASM_JS > 0, 'ASM_JS must be enabled in fastcomp'

emscripten.py

@@ -238,7 +238,8 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None,
 
     # merge forwarded data
     settings['EXPORTED_FUNCTIONS'] = forwarded_json['EXPORTED_FUNCTIONS']
-    all_exported_functions = set(settings['EXPORTED_FUNCTIONS']) # both asm.js and otherwise
+    all_exported_functions = set(shared.expand_response(settings['EXPORTED_FUNCTIONS'])) # both asm.js and otherwise
+
     for additional_export in settings['DEFAULT_LIBRARY_FUNCS_TO_INCLUDE']: # additional functions to export from asm, if they are implemented
       all_exported_functions.add('_' + additional_export)
     if settings['EXPORT_FUNCTION_TABLES']:

tools/shared.py

@@ -1500,6 +1500,17 @@ class Building:
     # Allow usage of emscripten.py without warning
     os.environ['EMSCRIPTEN_SUPPRESS_USAGE_WARNING'] = '1'
 
+    # EXPORTED_FUNCTIONS can potentially be very large.
+    # 8k is a bit of an arbitrary limit, but a reasonable one
+    # for max command line size before we use a response file
+    if len(' '.join(Settings.EXPORTED_FUNCTIONS)) > 8192:
+      logging.debug('using response file for EXPORTED_FUNCTIONS')
+      exports_response = configuration.get_temp_files().get(suffix='.response').name
+      exports_response_fh = open(exports_response, 'w')
+      json.dump(Settings.EXPORTED_FUNCTIONS, exports_response_fh)
+      exports_response_fh.close()
+      Settings.EXPORTED_FUNCTIONS = '@' + exports_response
+
     # Run Emscripten
     settings = Settings.serialize()
     args = settings + extra_args
@@ -1531,10 +1542,26 @@ class Building:
   @staticmethod
   def get_safe_internalize():
     if not Building.can_build_standalone(): return [] # do not internalize anything
+
     exps = expand_response(Settings.EXPORTED_FUNCTIONS)
-    exports = ','.join(map(lambda exp: exp[1:], exps))
+    internalize_public_api = '-internalize-public-api-'
+    internalize_list = ','.join(map(lambda exp: exp[1:], exps))
+
+    # EXPORTED_FUNCTIONS can potentially be very large.
+    # 8k is a bit of an arbitrary limit, but a reasonable one
+    # for max command line size before we use a response file
+    if len(internalize_list) > 8192:
+      finalized_exports = '\n'.join(map(lambda exp: exp[1:], exps))
+      internalize_list_file = configuration.get_temp_files().get(suffix='.response').name
+      internalize_list_fh = open(internalize_list_file, 'w')
+      internalize_list_fh.write(finalized_exports)
+      internalize_list_fh.close()
+      internalize_public_api += 'file=' + internalize_list_file
+    else:
+      internalize_public_api += 'list=' + internalize_list
+
     # internalize carefully, llvm 3.2 will remove even main if not told not to
-    return ['-internalize', '-internalize-public-api-list=' + exports]
+    return ['-internalize', internalize_public_api]
 
   @staticmethod
   def pick_llvm_opts(optimization_level):