Add a simple one-entry-per-line file format for specifying symbol lists (#14170)

For options such as `EXPORTED_FUNCTIONS` that take lists we now support
a simple file format that is just one line per symbol.   It doesn't
require any escaping or punctuation.

This is much easier to use and maintain and is in line with other compiler
toolchain options used by gcc and ld.  For example ld's
`--retain-symbols-file=filename` or wasm-ld's
`--undefined-file=filename`.

This should be especially useful for options like `ASYNCIFY_ADD` or
`ASYNCIFY_REMOVE` which often involvea lot of quoting (because they
include things like spaces and commas).

ChangeLog.md

@@ -20,6 +20,10 @@ See docs/process.md for more on how version tagging works.
 
 2.0.21
 ------
+- Options such as EXPORTED_FUNCTIONS that can take a response file containing
+  list of symbols can now use a simple one-symbol-per-line format.  This new
+  format is much simpler and doesn't require commas between symbols, opening
+  or closing braces, or any kind of escaping for special characters.
 - The WebAssembly linker (`wasm-ld`) now performes string tail merging on any
   static string data in your program.   This has long been part of the native
   ELF linker and should not be observable in well-behaved programs.  This

docs/emcc.txt

@@ -123,16 +123,16 @@ Options that are modified or new in *emcc* are listed below:
         -s "EXPORTED_FUNCTIONS=['liblib.so']"
 
    You can also specify that the value of an option will be read from
-   a specified JSON-formatted file. For example, the following option
-   sets the "EXPORTED_FUNCTIONS" option with the contents of the file
-   at **path/to/file**.
+   a file. For example, the following will set "EXPORTED_FUNCTIONS"
+   based on the contents of the file at **path/to/file**.
 
       -s EXPORTED_FUNCTIONS=@/path/to/file
 
    Note:
 
-     * In this case the file might contain a JSON-formatted list of
-       functions: "["_func1", "func2"]".
+     * In this case the file should contain a list of symbols, one per
+       line.  For legacy use cases JSON-formatted files are also
+       supported: e.g. "["_func1", "func2"]".
 
      * The specified file path must be absolute, not relative.
 

emcc.py

@@ -360,6 +360,7 @@ def apply_settings(changes):
     if key in MEM_SIZE_SETTINGS:
       value = str(expand_byte_size_suffixes(value))
 
+    filename = None
     if value and value[0] == '@':
       filename = value[1:]
       if not os.path.exists(filename):
@@ -371,10 +372,14 @@ def apply_settings(changes):
     existing = getattr(settings, user_key, None)
     expect_list = type(existing) == list
 
-    try:
-      value = parse_value(value, expect_list)
-    except Exception as e:
-      exit_with_error('a problem occurred in evaluating the content after a "-s", specifically "%s=%s": %s', key, value, str(e))
+    if filename and expect_list and value.strip()[0] != '[':
+      # Prefer simpler one-line-per value parser
+      value = parse_symbol_list_file(value)
+    else:
+      try:
+        value = parse_value(value, expect_list)
+      except Exception as e:
+        exit_with_error('a problem occurred in evaluating the content after a "-s", specifically "%s=%s": %s', key, value, str(e))
 
     # Do some basic type checking by comparing to the existing settings.
     # Sadly we can't do this generically in the SettingsManager since there are settings
@@ -3573,6 +3578,15 @@ def is_valid_abspath(options, path_name):
   return False
 
 
+def parse_symbol_list_file(contents):
+  """Parse contents of one-symbol-per-line response file.  This format can by used
+  with, for example, -sEXPORTED_FUNCTIONS=@filename and avoids the need for any
+  kind of quoting or escaping.
+  """
+  values = contents.splitlines()
+  return [v.strip() for v in values]
+
+
 def parse_value(text, expect_list):
   # Note that using response files can introduce whitespace, if the file
   # has a newline at the end. For that reason, we rstrip() in relevant

site/source/docs/tools_reference/emcc.rst

@@ -112,7 +112,7 @@ Options that are modified or new in *emcc* are listed below:
       -s EXPORTED_FUNCTIONS="['liblib.so']"
       -s "EXPORTED_FUNCTIONS=['liblib.so']"
 
-  You can also specify that the value of an option will be read from a specified JSON-formatted file. For example, the following option sets the ``EXPORTED_FUNCTIONS`` option with the contents of the file at **path/to/file**.
+  You can also specify that the value of an option will be read from a file. For example, the following will set ``EXPORTED_FUNCTIONS`` based on the contents of the file at **path/to/file**.
 
   ::
 
@@ -120,7 +120,7 @@ Options that are modified or new in *emcc* are listed below:
 
   .. note::
 
-    - In this case the file might contain a JSON-formatted list of functions: ``["_func1", "func2"]``.
+    - In this case the file should contain a list of symbols, one per line.  For legacy use cases JSON-formatted files are also supported: e.g. ``["_func1", "func2"]``.
     - The specified file path must be absolute, not relative.
 
   .. note:: Options can be specified as a single argument without a space

tests/test_browser.py

@@ -3299,7 +3299,7 @@ window.close = function() {
   })
   def test_async_returnvalue(self, args):
     if '@' in str(args):
-      create_file('filey.txt', '["sync_tunnel", "sync_tunnel_bool"]')
+      create_file('filey.txt', 'sync_tunnel\nsync_tunnel_bool\n')
     self.btest('browser/async_returnvalue.cpp', '0', args=['-s', 'ASYNCIFY', '-s', 'ASYNCIFY_IGNORE_INDIRECT', '--js-library', test_file('browser/async_returnvalue.js')] + args + ['-s', 'ASSERTIONS'])
 
   def test_async_stack_overflow(self):

tests/test_other.py

@@ -6218,12 +6218,16 @@ high = 1234
     self.assertContained('hello, world!', self.run_js('a.out.js'))
 
   def test_dash_s_response_file_string(self):
-    create_file('response_file', '"MyModule"\n')
-    self.run_process([EMXX, test_file('hello_world.cpp'), '-s', 'EXPORT_NAME=@response_file'])
+    create_file('response_file.txt', 'MyModule\n')
+    create_file('response_file.json', '"MyModule"\n')
+    self.run_process([EMXX, test_file('hello_world.cpp'), '-s', 'EXPORT_NAME=@response_file.txt'])
+    self.run_process([EMXX, test_file('hello_world.cpp'), '-s', 'EXPORT_NAME=@response_file.json'])
 
   def test_dash_s_response_file_list(self):
-    create_file('response_file', '["_main", "_malloc"]\n')
-    self.run_process([EMXX, test_file('hello_world.cpp'), '-s', 'EXPORTED_FUNCTIONS=@response_file'])
+    create_file('response_file.txt', '_main\n_malloc\n')
+    create_file('response_file.json', '["_main", "_malloc"]\n')
+    self.run_process([EMXX, test_file('hello_world.cpp'), '-s', 'EXPORTED_FUNCTIONS=@response_file.txt'])
+    self.run_process([EMXX, test_file('hello_world.cpp'), '-s', 'EXPORTED_FUNCTIONS=@response_file.json'])
 
   def test_dash_s_response_file_misssing(self):
     err = self.expect_fail([EMXX, test_file('hello_world.cpp'), '-s', 'EXPORTED_FUNCTIONS=@foo'])
@@ -8095,12 +8099,19 @@ test_module().then((test_module_instance) => {
         void c() { printf("c\n"); }
         void d() { printf("d\n"); }
       ''')
-    create_file('response', r'''[
+    create_file('response.json', '''\
+[
 "_a",
 "_b",
 "_c",
 "_d"
 ]
+''')
+    create_file('response.txt', '''\
+_a
+_b
+_c
+_d
 ''')
 
     for export_arg, expected in [
@@ -8115,7 +8126,9 @@ test_module().then((test_module_instance) => {
       # extra space at end - should be ignored
       ("EXPORTED_FUNCTIONS=['_a', '_b', '_c', '_d' ]", ''),
       # extra newline in response file - should be ignored
-      ("EXPORTED_FUNCTIONS=@response", ''),
+      ("EXPORTED_FUNCTIONS=@response.json", ''),
+      # Simple one-per-line response file format
+      ("EXPORTED_FUNCTIONS=@response.txt", ''),
       # stray slash
       ("EXPORTED_FUNCTIONS=['_a', '_b', \\'_c', '_d']", '''undefined exported symbol: "\\\\'_c'"'''),
       # stray slash
@@ -8132,6 +8145,9 @@ test_module().then((test_module_instance) => {
       print(proc.stderr)
       if not expected:
         self.assertFalse(proc.stderr)
+        js = open('a.out.js').read()
+        for sym in ('_a', '_b', '_c', '_d'):
+          self.assertContained(f'var {sym} = ', js)
       else:
         self.assertNotEqual(proc.returncode, 0)
         self.assertContained(expected, proc.stderr)
@@ -8143,16 +8159,18 @@ test_module().then((test_module_instance) => {
     self.assertContained('Try to quote the entire argument', proc.stderr)
 
   def test_asyncify_response_file(self):
-    return self.skipTest(' TODO remove the support for multiple binaryen versions warning output ("function name" vs "pattern" etc).')
     create_file('a.txt', r'''[
   "DOS_ReadFile(unsigned short, unsigned char*, unsigned short*, bool)"
 ]
 ''')
-    proc = self.run_process([EMCC, test_file('hello_world.c'), '-s', 'ASYNCIFY', '-s', "ASYNCIFY_ONLY=@a.txt"], stdout=PIPE, stderr=PIPE)
-    # we should parse the response file properly, and then issue a proper warning for the missing function
-    self.assertContained(
-        'Asyncify onlylist contained a non-matching pattern: DOS_ReadFile(unsigned short, unsigned char*, unsigned short*, bool)',
-        proc.stderr)
+
+    create_file('b.txt', 'DOS_ReadFile(unsigned short, unsigned char*, unsigned short*, bool)')
+    for file in ('a.txt', 'b.txt'):
+      proc = self.run_process([EMCC, test_file('hello_world.c'), '-sASYNCIFY', f'-sASYNCIFY_ONLY=@{file}'], stdout=PIPE, stderr=PIPE)
+      # we should parse the response file properly, and then issue a proper warning for the missing function
+      self.assertContained(
+          'Asyncify onlylist contained a non-matching pattern: DOS_ReadFile(unsigned short, unsigned char*, unsigned short*, bool)',
+          proc.stderr)
 
   def test_asyncify_advise(self):
     src = test_file('other/asyncify_advise.c')