Extend the use of response files to cover the case when emcc invokes emscripten.py, so that Windows command line length limitations don't break the test_asm_pgo on Windows.

emcc

@@ -79,6 +79,7 @@ import os, sys, shutil, tempfile, subprocess, shlex, time, re
 from subprocess import PIPE, STDOUT
 from tools import shared
 from tools.shared import Compression, execute, suffix, unsuffixed, unsuffixed_basename
+from tools.response_file import read_and_delete_response_file
 
 # Mapping of emcc opt levels to llvm opt levels. We use llvm opt level 3 in emcc opt
 # levels 2 and 3 (emcc 3 is unsafe opts, so unsuitable for the only level to get
@@ -129,19 +130,10 @@ while response_file:
   for index in range(1, len(sys.argv)):
     if sys.argv[index][0] == '@':
       # found one, loop again next time
-      response_file = sys.argv[index][1:]
-      print >>sys.stderr, 'emcc: using response file: %s' % response_file
-      if not os.path.exists(response_file):
-        print >>sys.stderr, 'emcc: error: Response file not found: %s' % response_file
-        exit(1)
-
-      response_fd = open(response_file, 'r')
-      extra_args = shlex.split(response_fd.read())
-      response_fd.close()
-
+      response_file = True
+      extra_args = read_and_delete_response_file(sys.argv[index])
       # slice in extra_args in place of the response file arg
       sys.argv[index:index+1] = extra_args
-      #if DEBUG: print >>sys.stderr, "Expanded response file: " + " | ".join(sys.argv)
       break
 
 if sys.argv[1] == '--version':

emscripten.py

@@ -12,6 +12,7 @@ headers, for the libc implementation in JS).
 import os, sys, json, optparse, subprocess, re, time, multiprocessing, functools
 
 from tools import jsrun, cache as cache_module, tempfiles
+from tools.response_file import read_and_delete_response_file
 
 __rootpath__ = os.path.abspath(os.path.dirname(__file__))
 def path_from_root(*pathelems):
@@ -629,6 +630,18 @@ def main(args, compiler_engine, cache, jcache, relooper, temp_files, DEBUG, DEBU
            jcache=jcache, temp_files=temp_files, DEBUG=DEBUG, DEBUG_CACHE=DEBUG_CACHE)
 
 def _main(environ):
+  response_file = True
+  while response_file:
+    response_file = None
+    for index in range(1, len(sys.argv)):
+      if sys.argv[index][0] == '@':
+        # found one, loop again next time
+        response_file = True
+        response_file_args = read_and_delete_response_file(sys.argv[index])
+        # slice in extra_args in place of the response file arg
+        sys.argv[index:index+1] = response_file_args
+        break
+
   parser = optparse.OptionParser(
     usage='usage: %prog [-h] [-H HEADERS] [-o OUTFILE] [-c COMPILER_ENGINE] [-s FOO=BAR]* infile',
     description=('You should normally never use this! Use emcc instead. '

tests/runner.py

@@ -8079,11 +8079,11 @@ def process(filename):
 
       shutil.move(self.in_dir('src.cpp.o.js'), self.in_dir('pgo.js'))
       pgo_output = run_js(self.in_dir('pgo.js')).split('\n')[1]
-      open('pgo_data', 'w').write(pgo_output)
+      open('pgo_data.rsp', 'w').write(pgo_output)
 
       # with response file
 
-      self.emcc_args += ['@pgo_data']
+      self.emcc_args += ['@pgo_data.rsp']
       self.do_run(src, output)
       self.emcc_args.pop()
       shutil.move(self.in_dir('src.cpp.o.js'), self.in_dir('pgoed.js'))

tools/response_file.py

@@ -0,0 +1,31 @@
+import tempfile, os, sys, shlex
+from tempfiles import try_delete
+
+# Routes the given cmdline param list in args into a new .rsp file and returns the filename to it.
+# The returned filename has '@' prepended to it already for convenience.
+def create_response_file(args, directory):
+  (response_fd, response_filename) = tempfile.mkstemp(prefix='emscripten_', suffix='.rsp', dir=directory, text=True)
+  response_fd = os.fdopen(response_fd, "w")
+  #print >> sys.stderr, "Creating response file '%s'" % response_filename
+  args = map(lambda p: p.replace(' ', '').replace('\\', '\\\\').replace('"', '\\"'), args)
+  response_fd.write(' '.join(args))
+  response_fd.close
+  return '@' + response_filename
+
+# Reads and deletes a .rsp file, and returns the list of cmdline params found in the file.
+def read_and_delete_response_file(response_filename):
+  # Ensure safety so that this function can never accidentally delete any non-.rsp files if things go wrong!
+  if not (response_filename.startswith('@') and response_filename.endswith('.rsp')):
+    raise Exception("'%s' is not a valid response file name! Response file names must start with '@' and end in '.rsp'!" % response_filename)
+  response_filename = response_filename[1:]
+
+  #print >> sys.stderr, "Using response file '%s'" % response_filename
+  if not os.path.exists(response_filename):
+    raise Exception("Response file '%s' not found!" % response_filename)
+
+  response_fd = open(response_filename, 'r')
+  args = response_fd.read()
+  response_fd.close()
+  try_delete(response_filename)
+  args = shlex.split(args)
+  return args

tools/shared.py

@@ -2,6 +2,7 @@ import shutil, time, os, sys, json, tempfile, copy, shlex, atexit, subprocess, h
 from subprocess import Popen, PIPE, STDOUT
 from tempfile import mkstemp
 import jsrun, cache, tempfiles
+from response_file import create_response_file
 
 def listify(x):
   if type(x) is not list: return [x]
@@ -34,13 +35,20 @@ class WindowsPopen:
         self.stdout_ = PIPE
       if self.stderr_ == None:
         self.stderr_ = PIPE
-  
-    # Call the process with fixed streams.
+
+    # emscripten.py supports reading args from a response file instead of cmdline.
+    # Use .rsp to avoid cmdline length limitations on Windows.
+    if len(args) >= 2 and args[1].endswith("emscripten.py"):
+      response_filename = create_response_file(args[2:], TEMP_DIR)
+      args = args[0:2] + [response_filename]
+      
     try:
+      # Call the process with fixed streams.
       self.process = subprocess.Popen(args, bufsize, executable, self.stdin_, self.stdout_, self.stderr_, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags)
     except Exception, e:
       print >> sys.stderr, '\nsubprocess.Popen(args=%s) failed! Exception %s\n' % (' '.join(args), str(e))
       raise e
+      raise
 
   def communicate(self, input=None):
     output = self.process.communicate(input)