emscripten/emcc

#!/usr/bin/env python2
# -*- Mode: python -*-

'''
emcc - compiler helper script
=============================

emcc is a drop-in replacement for a compiler like gcc or clang.

Tell your build system to use this instead of the compiler, and similarly
use emar, emranlib etc. instead of the same command without 'em'.

Example uses:

 * For configure, instead of ./configure, cmake, etc., run emconfigure.py
   with that command as an argument, for example

    emconfigure.py ./configure [options]

   emconfigure.py is a tiny script that just sets some environment vars
   as a convenience. The command just shown is equivalent to

    EMMAKEN_JUST_CONFIGURE=1 RANLIB=PATH/emranlib AR=PATH/emar CXX=PATH/em++ CC=PATH/emcc ./configure [options]

   where PATH is the path to this file.

   EMMAKEN_JUST_CONFIGURE tells emcc that it is being run in ./configure,
   so it should relay everything to gcc/g++. You should not define that when
   running make, of course.

 * With CMake, the same command will work (with cmake instead of ./configure). You may also be
   able to do the following in your CMakeLists.txt:

    SET(CMAKE_C_COMPILER "PATH/emcc")
    SET(CMAKE_CXX_COMPILER "PATH/em++")
    SET(CMAKE_LINKER "PATH/emcc")
    SET(CMAKE_CXX_LINKER "PATH/emcc")
    SET(CMAKE_C_LINK_EXECUTABLE "PATH/emcc")
    SET(CMAKE_CXX_LINK_EXECUTABLE "PATH/emcc")
    SET(CMAKE_AR "PATH/emar")
    SET(CMAKE_RANLIB "PATH/emranlib")

 * For SCons the shared.py can be imported like so:
    __file__ = str(Dir('#/project_path_to_emscripten/dummy/dummy'))
    __rootpath__ = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    def path_from_root(*pathelems):
      return os.path.join(__rootpath__, *pathelems)
    sys.path += [path_from_root('')]
    from tools.shared import *

   For using the Emscripten compilers/linkers/etc. you can do:
    env = Environment()
    ...
    env.Append(CCFLAGS = COMPILER_OPTS)
    env.Replace(LINK = LLVM_LD)
    env.Replace(LD   = LLVM_LD)
   TODO: Document all relevant setup changes

After setting that up, run your build system normally.

Note the appearance of em++ instead of emcc
for the C++ compiler. This is needed for cases where we get
a C++ file with a C extension, in which case CMake can be told
to run g++ on it despite the .c extension, see

  https://github.com/kripken/emscripten/issues/6

(If a similar situation occurs with ./configure, you can do the same there too.)

emcc can be influenced by a few environment variables:

  EMMAKEN_NO_SDK - Will tell emcc *not* to use the emscripten headers. Instead
                   your system headers will be used.

  EMMAKEN_COMPILER - The compiler to be used, if you don't want the default clang.
'''

import os, sys, shutil, tempfile, subprocess, shlex, time, re, logging
from subprocess import PIPE, STDOUT
from tools import shared
from tools.shared import Compression, execute, suffix, unsuffixed, unsuffixed_basename
from tools.response_file import read_response_file

logging = logging.getLogger('emcc')

# Mapping of emcc opt levels to llvm opt levels. We use llvm opt level 3 in emcc opt
# levels 2 and 3 (emcc 3 is unsafe opts, so unsuitable for the only level to get
# llvm opt level 3, and speed-wise emcc level 2 is already the slowest/most optimizing
# level)
LLVM_OPT_LEVEL = {
  0: 0,
  1: 1,
  2: 3,
  3: 3,
}

DEBUG = os.environ.get('EMCC_DEBUG')
if DEBUG == "0":
  DEBUG = None

TEMP_DIR = os.environ.get('EMCC_TEMP_DIR')
LEAVE_INPUTS_RAW = os.environ.get('EMCC_LEAVE_INPUTS_RAW') # Do not compile .ll files into .bc, just compile them with emscripten directly
                                                           # Not recommended, this is mainly for the test runner, or if you have some other
                                                           # specific need.
                                                           # One major limitation with this mode is that libc and libc++ cannot be
                                                           # added in. Also, LLVM optimizations will not be done, nor dead code elimination
AUTODEBUG = os.environ.get('EMCC_AUTODEBUG') # If set to 1, we will run the autodebugger (the automatic debugging tool, see tools/autodebugger).
                                             # Note that this will disable inclusion of libraries. This is useful because including
                                             # dlmalloc makes it hard to compare native and js builds
EMCC_CFLAGS = os.environ.get('EMCC_CFLAGS') # Additional compiler flags that we treat as if they were passed to us on the commandline

logging.debug('invocation: ' + ' '.join(sys.argv) + (' + ' + EMCC_CFLAGS if EMCC_CFLAGS else ''))
if EMCC_CFLAGS: sys.argv.append(EMCC_CFLAGS)

if DEBUG and LEAVE_INPUTS_RAW: logging.error('leaving inputs raw')

stdout = PIPE if not DEBUG else None # suppress output of child processes
stderr = PIPE if not DEBUG else None # unless we are in DEBUG mode

shared.check_sanity(force=DEBUG)

# Handle some global flags

# read response files very early on
response_file = True
while response_file:
  response_file = None
  for index in range(1, len(sys.argv)):
    if sys.argv[index][0] == '@':
      # found one, loop again next time
      response_file = True
      extra_args = read_response_file(sys.argv[index])
      # slice in extra_args in place of the response file arg
      sys.argv[index:index+1] = extra_args
      break

if len(sys.argv) == 1 or sys.argv[1] == '--help':
  this = os.path.basename('em++' if os.environ.get('EMMAKEN_CXX') else 'emcc')

  print '''%s [options] file...

Most normal gcc/g++ options will work, for example:
  --help                   Display this information
  --version                Display compiler version information

Options that are modified or new in %s include:
  -O0                      No optimizations (default)
  -O1                      Simple optimizations, including asm.js, LLVM -O1
                           optimizations, and no runtime assertions
                           or C++ exception catching (to re-enable
                           C++ exception catching, use
                           -s DISABLE_EXCEPTION_CATCHING=0 ).
                           (For details on the affects of different
                           opt levels, see apply_opt_level() in
                           tools/shared.py and also src/settings.js.)
                           Note: Optimizations are only done when
                           compiling to JavaScript, not to intermediate
                           bitcode, *unless* you build with
                           EMCC_OPTIMIZE_NORMALLY=1 (not recommended
                           unless you know what you are doing!)
  -O2                      As -O1, plus the relooper (loop recreation),
                           plus LLVM -O2 optimizations
  -O3                      As -O2, plus dangerous optimizations that may
                           break the generated code! This adds

                              -s DOUBLE_MODE=0
                              -s PRECISE_I64_MATH=0
                              --closure 1
                              --llvm-lto 1

                           This is not recommended at all. A better idea
                           is to try each of these separately on top of
                           -O2 to see what works. See the wiki for more
                           information.

  -s OPTION=VALUE          JavaScript code generation option passed
                           into the emscripten compiler. For the
                           available options, see src/settings.js
                           Note that for options that are lists, you
                           need quotation marks in most shells, for
                           example

                            -s RUNTIME_LINKED_LIBS="['liblib.so']"

                           or

                            -s "RUNTIME_LINKED_LIBS=['liblib.so']"

                           (without the external "s in either of those,
                           you would get an error)

                           You can also specify a file from which the
                           value would be read, for example,

                            -s DEAD_FUNCTIONS=@/path/to/file

                           The contents of /path/to/file will be read,
                           JSON.parsed and set into DEAD_FUNCTIONS (so
                           the file could contain

                            ["_func1", "func2"]

                           ). Note that the path must be absolute, not
                           relative.

  -g                       Use debug info. Note that you need this during
                           the last compilation phase from bitcode to
                           JavaScript, or else we will remove it by
                           default in -O1 and above.
                           In -O0, line numbers wil be shown in the
                           generated code. In -O1 and above, the optimizer
                           removes those comments. This flag does however
                           have the effect of disabling anything that
                           causes name mangling or minification (closure
                           or the registerize pass).

  --typed-arrays <mode>    0: No typed arrays
                           1: Parallel typed arrays
                           2: Shared (C-like) typed arrays (default)

  --llvm-opts <level>      0: No LLVM optimizations (default in -O0)
                           1: -O1 LLVM optimizations (default in -O1)
                           2: -O2 LLVM optimizations
                           3: -O3 LLVM optimizations (default in -O2+)

  --llvm-lto <level>       0: No LLVM LTO (default in -O2 and below)
                           1: LLVM LTO (default in -O3)
                           Note: If LLVM optimizations are not run
                           (see --llvm-opts), setting this to 1 has no
                           effect.

  --closure <on>           0: No closure compiler (default in -O2 and below)
                           1: Run closure compiler. This greatly reduces
                           code size and may in some cases increase
                           runtime speed (although the opposite can also
                           occur). Note that it takes time to run, and
                           may require some changes to the code. This
                           is run by default in -O3.

                           Note: If closure compiler hits an out-of-memory,
                           try adjusting JAVA_HEAP_SIZE in the environment
                           (for example, to 4096m for 4GB).

  --js-transform <cmd>     <cmd> will be called on the generated code
                           before it is optimized. This lets you modify
                           the JavaScript, for example adding some code
                           or removing some code, in a way that those
                           modifications will be optimized together with
                           the generated code properly. <cmd> will be
                           called with the filename of the generated
                           code as a parameter; to modify the code, you
                           can read the original data and then append to
                           it or overwrite it with the modified data.
                           <cmd> is interpreted as a space-separated
                           list of arguments, for example, <cmd> of
                           "python processor.py" will cause a python
                           script to be run.

  --pre-js <file>          A file whose contents are added before the
                           generated code. This is done *before*
                           optimization, so it will be minified
                           properly if closure compiler is run.

  --post-js <file>         A file whose contents are added after the
                           generated code This is done *before*
                           optimization, so it will be minified
                           properly if closure compiler is run.

  --embed-file <file>      A file to embed inside the generated
                           JavaScript. The compiled code will be able
                           to access the file in the current directory
                           with the same name as given here. So if
                           you do --embed-file dir/file.dat, then
                           (1) dir/file.dat must exist relative to
                           where you run emcc, and (2) your compiled
                           code will be able to find the file by
                           reading that same path, dir/file.dat.
                           If a directory is passed here, its entire
                           contents will be embedded.

  --preload-file <name>    A file to preload before running the
                           compiled code asynchronously. Otherwise
                           similar to --embed-file, except that this
                           option is only relevant when generating
                           HTML (it uses asynchronous binary XHRs),
                           or JS that will be used in a web page.
                           If a directory is passed here, its entire
                           contents will be preloaded.
                           Preloaded files are stored in filename.data,
                           where filename.html is the main file you
                           are compiling to. To run your code, you
                           will need both the .html and the .data.

                           emcc runs tools/file_packager.py to do the
                           actual packaging of embedded and preloaded
                           files. You can run the file packager yourself
                           if you want, see docs inside that file. You
                           should then put the output of the file packager
                           in an emcc --pre-js, so that it executes before
                           your main compiled code (or run it before in
                           some other way).

  --compression <codec>    Compress both the compiled code and embedded/
                           preloaded files. <codec> should be a triple,

                              <native_encoder>,<js_decoder>,<js_name>

                           where native_encoder is a native executable
                           that compresses stdin to stdout (the simplest
                           possible interface), js_decoder is a
                           JavaScript file that implements a decoder,
                           and js_name is the name of the function to
                           call in the decoder file (which should
                           receive an array/typed array and return
                           an array/typed array.
                           Compression only works when generating HTML.
                           When compression is on, all filed specified
                           to be preloaded are compressed in one big
                           archive, which is given the same name as the
                           output HTML but with suffix .data.compress

  --minify <on>            0: Do not minify the generated JavaScript's
                              whitespace (default in -O0, -O1, or if
                              -g is used)
                           1: Minify the generated JavaScript's
                              whitespace (default in -O2+, assuming
                              -g is not used)

  --split <size>           Splits the resulting javascript file into pieces
                           to ease debugging. This option only works if
                           Javascript is generated (target -o <name>.js).
                           Files with function declarations must be loaded
                           before main file upon execution.

                           Without "-g" option:
                              Creates files with function declarations up
                              to the given size with the suffix
                              "_functions.partxxx.js" and a main file with
                              the suffix ".js".

                           With "-g" option:
                              Recreates the directory structure of the C
                              source files and stores function declarations
                              in their respective C files with the suffix
                              ".js". If such a file exceeds the given size,
                              files with the suffix ".partxxx.js" are created.
                              The main file resides in the base directory and
                              has the suffix ".js".

  --bind                   Compiles the source code using the "embind"
                           bindings approach, which connects C/C++ and JS.

  --ignore-dynamic-linking Normally emcc will treat dynamic linking like
                           static linking, by linking in the code from
                           the dynamic library. This fails if the same
                           dynamic library is linked more than once.
                           With this option, dynamic linking is ignored,
                           which allows the build system to proceed without
                           errors. However, you will need to manually
                           link to the shared libraries later on yourself.

  --shell-file <path>      The path name to a skeleton HTML file used
                           when generating HTML output. The shell file
                           used needs to have this token inside it:
                           {{{ SCRIPT_CODE }}}
                           Note that this argument is ignored if a
                           target other than HTML is specified using
                           the -o option.

  --js-library <lib>       A JavaScript library to use in addition to
                           those in Emscripten's src/library_*

  -v                       Turns on verbose output. This will pass
                           -v to Clang, and also enable EMCC_DEBUG
                           to details emcc's operations

  --jcache                 Use a JavaScript cache. This is disabled by
                           default. When enabled, emcc will store the
                           results of compilation in a cache and check
                           the cache when compiling later, something
                           like what ccache does. This allows incremental
                           builds - where you are compiling a large
                           program but only modified a small part of it -
                           to be much faster (at the cost of more disk
                           IO for cache accesses). Note that you need
                           to enable --jcache for both loading and saving
                           of data, so you must enable it on a full build
                           for a later incremental build (where you also
                           enable it) to be sped up.

                           Caching works separately on 4 parts of compilation:
                           'pre' which is types and global variables; that
                           information is then fed into 'funcs' which are
                           the functions (which we parallelize), and then
                           'post' which adds final information based on
                           the functions (e.g., do we need long64 support
                           code). Finally, 'jsfuncs' are JavaScript-level
                           optimizations. Each of the 4 parts can be cached
                           separately, but note that they can affect each
                           other: If you recompile a single C++ file that
                           changes a global variable - e.g., adds, removes
                           or modifies a global variable, say by adding
                           a printf or by adding a compile-time timestamp,
                           then 'pre' cannot be loaded from the cache. And
                           since 'pre's output is sent to 'funcs' and 'post',
                           they will get invalidated as well, and only
                           'jsfuncs' will be cached. So avoid modifying
                           globals to let caching work fully.

                           To work around the problem mentioned in the
                           previous paragraph, you can use

                            emscripten_jcache_printf

                           when adding debug printfs to your code. That
                           function is specially preprocessed so that it
                           does not create a constant string global for
                           its first argument. See emscripten.h for more
                           details. Note in particular that you need to
                           already have a call to that function in your
                           code *before* you add one and do an incremental
                           build, so that adding an external reference
                           (also a global property) does not invalidate
                           everything.

                           Note that you should use -g during the linking
                           stage (bitcode to JS), for jcache to work
                           (otherwise, JS minification can confuse it).

  --clear-cache            Manually clears the cache of compiled
                           emscripten system libraries (libc++,
                           libc++abi, libc). This is normally
                           handled automatically, but if you update
                           llvm in-place (instead of having a different
                           directory for a new version), the caching
                           mechanism can get confused. Clearing the
                           cache can fix weird problems related to
                           cache incompatibilities, like clang failing
                           to link with library files. This also clears
                           other cached data like the jcache and
                           the bootstrapped relooper. After the cache
                           is cleared, this process will exit.

  --save-bc PATH           When compiling to JavaScript or HTML, this
                           option will save a copy of the bitcode to
                           the specified path. The bitcode will include
                           all files being linked, including standard
                           libraries, and after any link-time optimizations
                           (if any).

  --memory-init-file <on>  If on, we generate a separate memory initialization
                           file. This is more efficient than storing the
                           memory initialization data embedded inside
                           JavaScript as text. (default is off)

The target file, if specified (-o <target>), defines what will
be generated:

  <name>.js                JavaScript
  <name>.html              HTML with embedded JavaScript
  <name>.bc                LLVM bitcode (default)
  <name>.o                 LLVM bitcode (same as .bc)

(Note that if --memory-init-file is used, then in addition to a
.js or .html file that is generated, a .mem file will also appear.)

The -c option (which tells gcc not to run the linker) will
cause LLVM bitcode to be generated, as %s only generates
JavaScript in the final linking stage of building.

The input file(s) can be either source code files that
Clang can handle (C or C++), LLVM bitcode in binary form,
or LLVM assembly files in human-readable form.

emcc is affected by several environment variables. For details, view
the source of emcc (search for 'os.environ').

emcc: supported targets: llvm bitcode, javascript, NOT elf
(autoconf likes to see elf above to enable shared object support)
''' % (this, this, this)
  exit(0)

elif sys.argv[1] == '--version':
  revision = '(unknown revision)'
  here = os.getcwd()
  os.chdir(shared.path_from_root())
  try:
    revision = execute(['git', 'show'], stdout=PIPE, stderr=PIPE)[0].split('\n')[0]
  except:
    pass
  finally:
    os.chdir(here)
  print '''emcc (Emscripten GCC-like replacement) %s (%s)
Copyright (C) 2013 the Emscripten authors (see AUTHORS.txt)
This is free and open source software under the MIT license.
There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  ''' % (shared.EMSCRIPTEN_VERSION, revision)
  exit(0)  

elif len(sys.argv) == 2 and sys.argv[1] == '-v': # -v with no inputs
  print 'emcc (Emscripten GCC-like replacement + linker emulating GNU ld ) 2.0'
  exit(subprocess.call([shared.CLANG, '-v']))

def is_minus_s_for_emcc(newargs,i):
  assert newargs[i] == '-s'
  if i+1 < len(newargs) and '=' in newargs[i+1]: # -s OPT=VALUE is for us, -s by itself is a linker option
    return True
  else:
    logging.warning('treating -s as linker option and not as -s OPT=VALUE for js compilation')
    return False

# If this is a configure-type thing, do not compile to JavaScript, instead use clang
# to compile to a native binary (using our headers, so things make sense later)
CONFIGURE_CONFIG = (os.environ.get('EMMAKEN_JUST_CONFIGURE') or 'conftest.c' in sys.argv) and not os.environ.get('EMMAKEN_JUST_CONFIGURE_RECURSE')
CMAKE_CONFIG = 'CMakeFiles/cmTryCompileExec.dir' in ' '.join(sys.argv)# or 'CMakeCCompilerId' in ' '.join(sys.argv)
if CONFIGURE_CONFIG or CMAKE_CONFIG:
  debug_configure = 0 # XXX use this to debug configure stuff. ./configure's generally hide our normal output including stderr so we write to a file
  use_js = os.environ.get('EMCONFIGURE_JS') # whether we fake configure tests using clang - the local, native compiler - or not. if not we generate JS and use node with a shebang
                   # neither approach is perfect, you can try both, but may need to edit configure scripts in some cases
                   # XXX False is not fully tested yet

  if debug_configure:
    tempout = '/tmp/emscripten_temp/out'
    if not os.path.exists(tempout):
      open(tempout, 'w').write('//\n')

  src = None
  for i in range(len(sys.argv)):
    if sys.argv[i].endswith('.c'):
      try:
        src = open(sys.argv[i]).read()
        if debug_configure: open(tempout, 'a').write('============= ' + sys.argv[i] + '\n' + src + '\n=============\n\n')
      except:
        pass
    if sys.argv[i].endswith('.s'):
      if debug_configure: open(tempout, 'a').write('(compiling .s assembly, must use clang\n')
      use_js = 0

  if src:
    if 'fopen' in src and '"w"' in src:
      use_js = 0 # we cannot write to files from js!
      if debug_configure: open(tempout, 'a').write('Forcing clang since uses fopen to write\n')

  compiler = os.environ.get('CONFIGURE_CC') or (shared.CLANG if not use_js else shared.EMCC) # if CONFIGURE_CC is defined, use that. let's you use local gcc etc. if you need that
  if not ('CXXCompiler' in ' '.join(sys.argv) or os.environ.get('EMMAKEN_CXX')):
    compiler = shared.to_cc(compiler)

  def filter_emscripten_options(argv):
    idx = 0
    skip_next = False
    for el in argv:
      if skip_next:
        skip_next = False
        idx += 1
        continue
      if el == '-s' and is_minus_s_for_emcc(argv, idx):
        skip_next = True
      else:
        yield el
      idx += 1

  cmd = [compiler] + list(filter_emscripten_options(sys.argv[1:]))
  if not use_js: cmd += shared.EMSDK_OPTS + ['-DEMSCRIPTEN']
  if use_js: cmd += ['-s', 'ERROR_ON_UNDEFINED_SYMBOLS=1'] # configure tests should fail when an undefined symbol exists

  logging.debug('just configuring: ' + ' '.join(cmd))
  if debug_configure: open(tempout, 'a').write('emcc, just configuring: ' + ' '.join(cmd) + '\n\n')

  if not use_js:
    exit(subprocess.call(cmd))
  else:
    only_object = '-c' in cmd
    target = None
    for i in range(len(cmd)-1):
      if cmd[i] == '-o':
        if not only_object:
          cmd[i+1] += '.js'
        target = cmd[i+1]
        break
    print 't1', target
    if not target:
      target = 'a.out.js'
    print 't2', target, only_object
    os.environ['EMMAKEN_JUST_CONFIGURE_RECURSE'] = '1'
    ret = subprocess.call(cmd)
    os.environ['EMMAKEN_JUST_CONFIGURE_RECURSE'] = ''
    if not os.path.exists(target): exit(1)
    if target.endswith('.js'):
      shutil.copyfile(target, target[:-3])
      target = target[:-3]
    src = open(target).read()
    full_node = ' '.join(shared.listify(shared.NODE_JS))
    if os.path.sep not in full_node:
      full_node = '/usr/bin/' + full_node # TODO: use whereis etc. And how about non-*NIX?
    open(target, 'w').write('#!' + full_node + '\n' + src) # add shebang
    import stat
    os.chmod(target, stat.S_IMODE(os.stat(target).st_mode) | stat.S_IXUSR) # make executable
    exit(ret)

if os.environ.get('EMMAKEN_COMPILER'):
  CXX = os.environ['EMMAKEN_COMPILER']
else:
  CXX = shared.CLANG

CC = shared.to_cc(CXX)

# If we got here from a redirection through emmakenxx.py, then force a C++ compiler here
if os.environ.get('EMMAKEN_CXX'):
  CC = CXX

CC_ADDITIONAL_ARGS = shared.COMPILER_OPTS

EMMAKEN_CFLAGS = os.environ.get('EMMAKEN_CFLAGS')
if EMMAKEN_CFLAGS: CC_ADDITIONAL_ARGS += shlex.split(EMMAKEN_CFLAGS)

# ---------------- Utilities ---------------

SOURCE_SUFFIXES = ('.c', '.cpp', '.cxx', '.cc')
BITCODE_SUFFIXES = ('.bc', '.o', '.obj')
DYNAMICLIB_SUFFIXES = ('.dylib', '.so', '.dll')
STATICLIB_SUFFIXES = ('.a',)
ASSEMBLY_SUFFIXES = ('.ll',)
LIB_PREFIXES = ('', 'lib')

JS_CONTAINING_SUFFIXES = ('js', 'html')

seen_names = {}
def uniquename(name):
  if name not in seen_names:
    seen_names[name] = str(len(seen_names))
  return unsuffixed(name) + '_' + seen_names[name] + (('.' + suffix(name)) if suffix(name) else '')

# ---------------- End configs -------------

if len(sys.argv) == 1 or sys.argv[1] in ['x', 't']:
  # noop ar
  logging.debug('just ar')
  sys.exit(0)

use_cxx = True
header = False # pre-compiled headers. We fake that by just copying the file

for i in range(1, len(sys.argv)):
  arg = sys.argv[i]
  if not arg.startswith('-'):
    if arg.endswith('.c'):
      use_cxx = False
    if arg.endswith('.h') and sys.argv[i-1] != '-include':
      header = True

if '-M' in sys.argv or '-MM' in sys.argv:
  # Just output dependencies, do not compile. Warning: clang and gcc behave differently with -MF! (clang seems to not recognize it)
  cmd = [CC] + shared.COMPILER_OPTS + sys.argv[1:]
  logging.debug('just dependencies: ' + ' '.join(cmd))
  exit(subprocess.call(cmd))

# Check if a target is specified
target = None
for i in range(len(sys.argv)-1):
  if sys.argv[i].startswith('-o='):
    raise Exception('Invalid syntax: do not use -o=X, use -o X')

  if sys.argv[i] == '-o':
    target = sys.argv[i+1]
    sys.argv = sys.argv[:i] + sys.argv[i+2:]
    break

specified_target = target
target = specified_target if specified_target is not None else 'a.out.js' # specified_target is the user-specified one, target is what we will generate
target_basename = unsuffixed_basename(target)

if '.' in target:
  final_suffix = target.split('.')[-1]
else:
  final_suffix = ''

if header: # header or such
  if len(sys.argv) >= 3: # if there is a source and a target, then copy, otherwise do nothing
    sys.argv = filter(lambda arg: not arg.startswith('-I'), sys.argv)
    logging.debug('Just copy:' + sys.argv[-1] + target)
    shutil.copy(sys.argv[-1], target)
  else:
    logging.debug('No-op.')
  exit(0)

if TEMP_DIR:
  temp_dir = TEMP_DIR
  if os.path.exists(temp_dir):
    shutil.rmtree(temp_dir) # clear it
  os.makedirs(temp_dir)
else:
  temp_root = shared.TEMP_DIR
  if not os.path.exists(temp_root):
    os.makedirs(temp_root)
  temp_dir = tempfile.mkdtemp(dir=temp_root)

def in_temp(name):
  return os.path.join(temp_dir, os.path.basename(name))

try:
  call = CXX if use_cxx else CC

  ## Parse args

  newargs = sys.argv[1:]

  opt_level = 0
  llvm_opts = None
  llvm_lto = None
  closure = None
  js_transform = None
  pre_js = ''
  post_js = ''
  minify_whitespace = None
  split_js_file = None
  preload_files = []
  embed_files = []
  compression = None
  ignore_dynamic_linking = False
  shell_path = shared.path_from_root('src', 'shell.html')
  js_libraries = []
  keep_llvm_debug = False
  keep_js_debug = False
  bind = False
  jcache = False
  save_bc = False
  memory_init_file = False
  use_preload_cache = False

  if use_cxx:
    default_cxx_std = '-std=c++03' # Enforce a consistent C++ standard when compiling .cpp files, if user does not specify one on the cmdline.
  else:
    default_cxx_std = '' # Compiling C code with .c files, don't enforce a default C++ std.

  def check_bad_eq(arg):
    assert '=' not in arg, 'Invalid parameter (do not use "=" with "--" options)'

  absolute_warning_shown = False

  settings_changes = []

  for i in range(len(newargs)):
    newargs[i] = newargs[i].strip() # On Windows Vista (and possibly others), excessive spaces in the command line leak into the items in this array, so trim e.g. 'foo.cpp ' -> 'foo.cpp'
    if newargs[i].startswith('-O'):
      # Let -O default to -O2, which is what gcc does.
      requested_level = newargs[i][2:] or '2'
      if requested_level == 's':
        requested_level = 2
        settings_changes.append('INLINING_LIMIT=50')
      try:
        opt_level = int(requested_level)
        assert 0 <= opt_level <= 3
      except:
        raise Exception('Invalid optimization level: ' + newargs[i])
      newargs[i] = ''
    elif newargs[i].startswith('--llvm-opts'):
      check_bad_eq(newargs[i])
      llvm_opts = eval(newargs[i+1])
      newargs[i] = ''
      newargs[i+1] = ''
    elif newargs[i].startswith('--llvm-lto'):
      check_bad_eq(newargs[i])
      llvm_lto = eval(newargs[i+1])
      newargs[i] = ''
      newargs[i+1] = ''
    elif newargs[i].startswith('--closure'):
      check_bad_eq(newargs[i])
      closure = int(newargs[i+1])
      newargs[i] = ''
      newargs[i+1] = ''
    elif newargs[i].startswith('--js-transform'):
      check_bad_eq(newargs[i])
      js_transform = newargs[i+1]
      newargs[i] = ''
      newargs[i+1] = ''
    elif newargs[i].startswith('--pre-js'):
      check_bad_eq(newargs[i])
      pre_js += open(newargs[i+1]).read() + '\n'
      newargs[i] = ''
      newargs[i+1] = ''
    elif newargs[i].startswith('--post-js'):
      check_bad_eq(newargs[i])
      post_js += open(newargs[i+1]).read() + '\n'
      newargs[i] = ''
      newargs[i+1] = ''
    elif newargs[i].startswith('--minify'):
      check_bad_eq(newargs[i])
      minify_whitespace = int(newargs[i+1])
      newargs[i] = ''
      newargs[i+1] = ''
    elif newargs[i].startswith('--split'):
      check_bad_eq(newargs[i])
      split_js_file = int(newargs[i+1])
      newargs[i] = ''
      newargs[i+1] = ''
    elif newargs[i] == '-g':
      keep_llvm_debug = True
      keep_js_debug = True
    elif newargs[i] == '--bind':
      bind = True
      newargs[i] = ''
      if default_cxx_std:
        default_cxx_std = '-std=c++11' # Force C++11 for embind code, but only if user has not explicitly overridden a standard.
    elif newargs[i].startswith('-std='):
      default_cxx_std = '' # User specified a standard to use, clear Emscripten from specifying it.
    elif newargs[i].startswith('--embed-file'):
      check_bad_eq(newargs[i])
      embed_files.append(newargs[i+1])
      newargs[i] = ''
      newargs[i+1] = ''
    elif newargs[i].startswith('--preload-file'):
      check_bad_eq(newargs[i])
      preload_files.append(newargs[i+1])
      newargs[i] = ''
      newargs[i+1] = ''
    elif newargs[i].startswith('--compression'):
      check_bad_eq(newargs[i])
      parts = newargs[i+1].split(',')
      assert len(parts) == 3, '--compression requires specifying    native_encoder,js_decoder,js_name  - see emcc --help. got: %s' % newargs[i+1]
      Compression.encoder = parts[0]
      Compression.decoder = parts[1]
      Compression.js_name = parts[2]
      assert os.path.exists(Compression.encoder), 'native encoder %s does not exist' % Compression.encoder
      assert os.path.exists(Compression.decoder), 'js decoder %s does not exist' % Compression.decoder
      Compression.on = True
      newargs[i] = ''
      newargs[i+1] = ''
    elif newargs[i].startswith('--use-preload-cache'):
      use_preload_cache = True;
      newargs[i] = ''
    elif newargs[i] == '--ignore-dynamic-linking':
      ignore_dynamic_linking = True
      newargs[i] = ''
    elif newargs[i] == '-v':
      shared.COMPILER_OPTS += ['-v']
      DEBUG = 1
      os.environ['EMCC_DEBUG'] = '1' # send to child processes too
      newargs[i] = ''
    elif newargs[i].startswith('--shell-file'):
      check_bad_eq(newargs[i])
      shell_path = newargs[i+1]
      newargs[i] = ''
      newargs[i+1] = ''
    elif newargs[i].startswith('--js-library'):
      check_bad_eq(newargs[i])
      js_libraries.append(newargs[i+1])
      newargs[i] = ''
      newargs[i+1] = ''
    elif newargs[i] == '--remove-duplicates':
      logging.warning ('--remove-duplicates is deprecated as it is no longer needed. If you cannot link without it, file a bug with a testcase')
      newargs[i] = ''
    elif newargs[i] == '--jcache':
      jcache = True
      newargs[i] = ''
    elif newargs[i] == '--clear-cache':
      newargs[i] = ''
      logging.info('clearing cache')
      shared.Cache.erase()
      sys.exit(0)
    elif newargs[i] == '--save-bc':
      check_bad_eq(newargs[i])
      save_bc = newargs[i+1]
      newargs[i] = ''
      newargs[i+1] = ''
    elif newargs[i] == '--memory-init-file':
      check_bad_eq(newargs[i])
      memory_init_file = int(newargs[i+1])
      newargs[i] = ''
      newargs[i+1] = ''
    elif newargs[i].startswith(('-I/', '-L/')):
      if not absolute_warning_shown:
        logging.warning ('-I or -L of an absolute path encountered. If this is to a local system header/library, it may cause problems (local system files make sense for compiling natively on your system, but not necessarily to JavaScript)') # Of course an absolute path to a non-system-specific library or header is fine, and you can ignore this warning. The danger are system headers that are e.g. x86 specific and nonportable. The emscripten bundled headers are modified to be portable, local system ones are generally not
        absolute_warning_shown = True
  newargs = [ arg for arg in newargs if arg is not '' ]

  # If user did not specify a default -std for C++ code, specify the emscripten default.
  if default_cxx_std:
    newargs = newargs + [default_cxx_std]

  if llvm_opts is None: llvm_opts = LLVM_OPT_LEVEL[opt_level]
  if llvm_lto is None: llvm_lto = opt_level >= 3
  if opt_level <= 0: keep_llvm_debug = keep_js_debug = True # always keep debug in -O0
  if opt_level > 0: keep_llvm_debug = False # JS optimizer wipes out llvm debug info from being visible
  if closure is None and opt_level == 3: closure = True

  if DEBUG: start_time = time.time() # done after parsing arguments, which might affect debug state

  if closure:
    assert os.path.exists(shared.CLOSURE_COMPILER), logging.error('fatal: Closure compiler (%s) does not exist' % shared.CLOSURE_COMPILER)

  for i in range(len(newargs)):
    if newargs[i] == '-s':
      if is_minus_s_for_emcc(newargs, i):
        settings_changes.append(newargs[i+1])
        newargs[i] = newargs[i+1] = ''
    elif newargs[i].startswith('--typed-arrays'):
      assert '=' not in newargs[i], 'Invalid typed arrays parameter (do not use "=")'
      settings_changes.append('USE_TYPED_ARRAYS=' + newargs[i+1])
      newargs[i] = ''
      newargs[i+1] = ''
  newargs = [ arg for arg in newargs if arg is not '' ]

  if split_js_file:
    settings_changes.append("PRINT_SPLIT_FILE_MARKER=1")

  # Find input files

  input_files = []
  has_source_inputs = False
  lib_dirs = [shared.path_from_root('system', 'local', 'lib'),
              shared.path_from_root('system', 'lib')]
  libs = []
  for i in range(len(newargs)): # find input files XXX this a simple heuristic. we should really analyze based on a full understanding of gcc params,
                                # right now we just assume that what is left contains no more |-x OPT| things
    arg = newargs[i]

    if i > 0:
      prev = newargs[i-1]
      if prev in ['-MT', '-install_name', '-I', '-L']: continue # ignore this gcc-style argument

    if not arg.startswith('-') and (arg.endswith(SOURCE_SUFFIXES + BITCODE_SUFFIXES + DYNAMICLIB_SUFFIXES + ASSEMBLY_SUFFIXES) or shared.Building.is_ar(arg)): # we already removed -o <target>, so all these should be inputs
      newargs[i] = ''
      if os.path.exists(arg):
        if arg.endswith(SOURCE_SUFFIXES):
          input_files.append(arg)
          has_source_inputs = True
        else:
          # this should be bitcode, make sure it is valid
          if arg.endswith(ASSEMBLY_SUFFIXES) or shared.Building.is_bitcode(arg):
            input_files.append(arg)
          elif arg.endswith(STATICLIB_SUFFIXES + DYNAMICLIB_SUFFIXES):
            # if it's not, and it's a library, just add it to libs to find later
            l = unsuffixed_basename(arg)
            for prefix in LIB_PREFIXES:
              if not prefix: continue
              if l.startswith(prefix):
                l = l[len(prefix):]
                break;
            libs.append(l)
            newargs[i] = ''
          else:
            logging.warning(arg + ' is not valid LLVM bitcode')
      else:
        logging.error(arg + ': No such file or directory')
        exit(1)
    elif arg.startswith('-L'):
      lib_dirs.append(arg[2:])
      newargs[i] = ''
    elif arg.startswith('-l'):
      libs.append(arg[2:])
      newargs[i] = ''

  original_input_files = input_files[:]

  newargs = [ arg for arg in newargs if arg is not '' ]

  # -c means do not link in gcc, and for us, the parallel is to not go all the way to JS, but stop at bitcode
  has_dash_c = '-c' in newargs
  if has_dash_c:
    assert has_source_inputs, 'Must have source code inputs to use -c'
    target = target_basename + '.o'
    final_suffix = 'o'

  # do not link in libs when just generating object code (not an 'executable', i.e. JS, or a library)
  if ('.' + final_suffix) in BITCODE_SUFFIXES and len(libs) > 0:
    logging.warning('not linking against libraries since only compiling to bitcode')
    libs = []

  # Find library files
  for lib in libs:
    logging.debug('looking for library "%s"' % lib)
    found = False
    for prefix in LIB_PREFIXES:
      for suff in STATICLIB_SUFFIXES + DYNAMICLIB_SUFFIXES:
        name = prefix + lib + suff
        for lib_dir in lib_dirs:
          path = os.path.join(lib_dir, name)
          if os.path.exists(path):
            logging.debug('found library "%s" at %s' % (lib, path))
            input_files.append(path)
            found = True
            break
        if found: break
      if found: break

  if ignore_dynamic_linking:
    input_files = filter(lambda input_file: not input_file.endswith(DYNAMICLIB_SUFFIXES), input_files)

  if len(input_files) == 0:
    logging.error('no input files\nnote that input files without a known suffix are ignored, make sure your input files end with one of: ' + str(SOURCE_SUFFIXES + BITCODE_SUFFIXES + DYNAMICLIB_SUFFIXES + STATICLIB_SUFFIXES + ASSEMBLY_SUFFIXES))
    exit(0)

  newargs += CC_ADDITIONAL_ARGS

  assert not (Compression.on and final_suffix != 'html'), 'Compression only works when generating HTML'

  # If we are using embind and generating JS, now is the time to link in bind.cpp
  if bind and final_suffix in JS_CONTAINING_SUFFIXES:
    input_files.append(shared.path_from_root('system', 'lib', 'embind', 'bind.cpp'))

  # Apply optimization level settings
  shared.Settings.apply_opt_level(opt_level, noisy=True)

  # Apply -s settings in newargs here (after optimization levels, so they can override them)
  for change in settings_changes:
    key, value = change.split('=')
    if value[0] == '@':
      value = '"' + value + '"'
    exec('shared.Settings.' + key + ' = ' + value)

  # Apply effects from settings
  if shared.Settings.ASM_JS:
    assert opt_level >= 1, 'asm.js requires -O1 or above'

    if bind:
      shared.Settings.ASM_JS = 0
      logging.warning('disabling asm.js because it is not compatible with embind yet')
    if closure:
      logging.warning('disabling closure because it is not compatible with asm.js code generation')
      closure = False
    if shared.Settings.CORRECT_SIGNS != 1:
      logging.warning('setting CORRECT_SIGNS to 1 for asm.js code generation')
      shared.Settings.CORRECT_SIGNS = 1
    if shared.Settings.CORRECT_OVERFLOWS != 1:
      logging.warning('setting CORRECT_OVERFLOWS to 1 for asm.js code generation')
      shared.Settings.CORRECT_OVERFLOWS = 1
    assert not shared.Settings.PGO, 'cannot run PGO in ASM_JS mode'

  if shared.Settings.CORRECT_SIGNS >= 2 or shared.Settings.CORRECT_OVERFLOWS >= 2 or shared.Settings.CORRECT_ROUNDINGS >= 2:
    keep_llvm_debug = True # must keep debug info to do line-by-line operations

  if (keep_llvm_debug or keep_js_debug) and closure:
    logging.warning('disabling closure because debug info was requested')
    closure = False

  if jcache and not keep_js_debug: print >> sys.stderr, 'emcc: warning: it is recommended to run jcache with -g when compiling bitcode to JS'

  if minify_whitespace is None:
    minify_whitespace = opt_level >= 2 and not keep_js_debug

  assert shared.LLVM_TARGET in shared.COMPILER_OPTS
  if shared.LLVM_TARGET == 'i386-pc-linux-gnu':
    shared.Settings.TARGET_X86 = 1
    shared.Settings.TARGET_LE32 = 0
    assert 'le32-unknown-nacl' not in shared.COMPILER_OPTS
  elif shared.LLVM_TARGET == 'le32-unknown-nacl':
    shared.Settings.TARGET_LE32 = 1
    shared.Settings.TARGET_X86 = 0
    assert 'i386-pc-linux-gnu' not in shared.COMPILER_OPTS
  else:
    raise Exception('unknown llvm target: ' + str(shared.LLVM_TARGET))

  ## Compile source code to bitcode

  logging.debug('compiling to bitcode')

  temp_files = []

  # First, generate LLVM bitcode. For each input file, we get base.o with bitcode
  for input_file in input_files:
    if input_file.endswith(SOURCE_SUFFIXES):
      logging.debug('compiling source file: ' + input_file)
      input_file = shared.Building.preprocess(input_file, in_temp(uniquename(input_file)))
      output_file = in_temp(unsuffixed(uniquename(input_file)) + '.o')
      temp_files.append(output_file)
      args = newargs + ['-emit-llvm', '-c', input_file, '-o', output_file]
      logging.debug("running:" + call + ' '.join(args))
      execute([call] + args) # let compiler frontend print directly, so colors are saved (PIPE kills that)
      if not os.path.exists(output_file):
        logging.error('compiler frontend failed to generate LLVM bitcode, halting')
        sys.exit(1)
    else: # bitcode
      if input_file.endswith(BITCODE_SUFFIXES):
        logging.debug('copying bitcode file: ' + input_file)
        temp_file = in_temp(unsuffixed(uniquename(input_file)) + '.o')
        shutil.copyfile(input_file, temp_file)
        temp_files.append(temp_file)
      elif input_file.endswith(DYNAMICLIB_SUFFIXES) or shared.Building.is_ar(input_file):
        logging.debug('copying library file: ' + input_file)
        temp_file = in_temp(uniquename(input_file))
        shutil.copyfile(input_file, temp_file)
        temp_files.append(temp_file)
      else: #.ll
        if not LEAVE_INPUTS_RAW:
          # Note that by assembling the .ll file, then disassembling it later, we will
          # remove annotations which is a good thing for compilation time
          logging.debug('assembling assembly file: ' + input_file)
          temp_file = in_temp(unsuffixed(uniquename(input_file)) + '.o')
          shared.Building.llvm_as(input_file, temp_file)
          temp_files.append(temp_file)

  if not LEAVE_INPUTS_RAW: assert len(temp_files) == len(input_files)

  # If we were just asked to generate bitcode, stop there
  if final_suffix not in JS_CONTAINING_SUFFIXES:
    if llvm_opts > 0:
      if not os.environ.get('EMCC_OPTIMIZE_NORMALLY'):
        logging.warning('-Ox flags ignored, since not generating JavaScript')
      else:
        for input_file in input_files:
          if input_file.endswith(SOURCE_SUFFIXES):
            logging.debug('optimizing %s with -O%d since EMCC_OPTIMIZE_NORMALLY defined' % (input_file, llvm_opts))
            shared.Building.llvm_opt(in_temp(unsuffixed(uniquename(input_file)) + '.o'), llvm_opts)
          else:
            logging.debug('not optimizing %s despite EMCC_OPTIMIZE_NORMALLY since not source code' % (input_file))
    if not specified_target:
      for input_file in input_files:
        shutil.move(in_temp(unsuffixed(uniquename(input_file)) + '.o'), unsuffixed_basename(input_file) + '.' + final_suffix)
    else:
      if len(input_files) == 1:
        shutil.move(in_temp(unsuffixed(uniquename(input_files[0])) + '.o'), specified_target)
      else:
        assert len(original_input_files) == 1 or not has_dash_c, 'fatal error: cannot specify -o with -c with multiple files' + str(sys.argv) + ':' + str(original_input_files)
        # We have a specified target (-o <target>), which is not JavaScript or HTML, and
        # we have multiple files: Link them
        logging.debug('link: ' + str(temp_files) + specified_target)
        shared.Building.link(temp_files, specified_target)
    exit(0)

  ## Continue on to create JavaScript

  logging.debug('will generate JavaScript')

  extra_files_to_link = []

  if not LEAVE_INPUTS_RAW and not AUTODEBUG and \
     not shared.Settings.BUILD_AS_SHARED_LIB == 2: # shared lib 2 use the library in the parent
    # Check if we need to include some libraries that we compile. (We implement libc ourselves in js, but
    # compile a malloc implementation and stdlibc++.)
    # Note that we assume a single symbol is enough to know if we have/do not have dlmalloc etc. If you
    # include just a few symbols but want the rest, this will not work.

    def read_symbols(path, exclude=None):
      symbols = map(lambda line: line.strip().split(' ')[1], open(path).readlines())
      if exclude:
        symbols = filter(lambda symbol: symbol not in exclude, symbols)
      return set(symbols)

    # XXX We also need to add libc symbols that use malloc, for example strdup. It's very rare to use just them and not
    #     a normal malloc symbol (like free, after calling strdup), so we haven't hit this yet, but it is possible.
    libc_symbols = read_symbols(shared.path_from_root('system', 'lib', 'libc.symbols'))
    libcextra_symbols = read_symbols(shared.path_from_root('system', 'lib', 'libcextra.symbols'))
    libcxx_symbols = read_symbols(shared.path_from_root('system', 'lib', 'libcxx', 'symbols'), exclude=libc_symbols)
    libcxxabi_symbols = read_symbols(shared.path_from_root('system', 'lib', 'libcxxabi', 'symbols'), exclude=libc_symbols)

    def build_libc(lib_filename, files):
      o_s = []
      prev_cxx = os.environ.get('EMMAKEN_CXX')
      if prev_cxx: os.environ['EMMAKEN_CXX'] = ''
      musl_internal_includes = shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'internal')
      for src in files:
        o = in_temp(os.path.basename(src) + '.o')
        execute([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-o', o, '-I', musl_internal_includes], stdout=stdout, stderr=stderr)
        o_s.append(o)
      if prev_cxx: os.environ['EMMAKEN_CXX'] = prev_cxx
      shared.Building.link(o_s, in_temp(lib_filename))
      return in_temp(lib_filename)

    def build_libcxx(src_dirname, lib_filename, files):
      o_s = []
      for src in files:
        o = in_temp(src + '.o')
        srcfile = shared.path_from_root(src_dirname, src)
        execute([shared.PYTHON, shared.EMXX, srcfile, '-o', o, '-std=c++11'], stdout=stdout, stderr=stderr)
        o_s.append(o)
      shared.Building.link(o_s, in_temp(lib_filename))
      return in_temp(lib_filename)

    # libc
    def create_libc():
      logging.debug(' building libc for cache')
      libc_files = [
        'dlmalloc.c',
        os.path.join('libcxx', 'new.cpp'),
        os.path.join('libc', 'stdlib', 'getopt_long.c'),
        os.path.join('libc', 'gen', 'err.c'),
        os.path.join('libc', 'gen', 'errx.c'),
        os.path.join('libc', 'gen', 'warn.c'),
        os.path.join('libc', 'gen', 'warnx.c'),
        os.path.join('libc', 'gen', 'verr.c'),
        os.path.join('libc', 'gen', 'verrx.c'),
        os.path.join('libc', 'gen', 'vwarn.c'),
        os.path.join('libc', 'gen', 'vwarnx.c'),
        os.path.join('libc', 'stdlib', 'strtod.c'),
      ];
      return build_libc('libc.bc', libc_files)

    def fix_libc(need):
      # libc needs some sign correction. # If we are in mode 0, switch to 2. We will add our lines
      try:
        if shared.Settings.CORRECT_SIGNS == 0: raise Exception('we need to change to 2')
      except: # we fail if equal to 0 - so we need to switch to 2 - or if CORRECT_SIGNS is not even in Settings
        shared.Settings.CORRECT_SIGNS = 2
      if shared.Settings.CORRECT_SIGNS == 2:
        shared.Settings.CORRECT_SIGNS_LINES = [shared.path_from_root('src', 'dlmalloc.c') + ':' + str(i+4) for i in [4816, 4191, 4246, 4199, 4205, 4235, 4227]]
      # If we are in mode 1, we are correcting everything anyhow. If we are in mode 3, we will be corrected
      # so all is well anyhow too.

    # libcextra
    def create_libcextra():
      logging.debug('building libcextra for cache')
      musl_files = [
         ['ctype', [
          'iswalnum.c',
          'iswalpha.c',
          'iswblank.c',
          'iswcntrl.c',
          'iswctype.c',
          'iswdigit.c',
          'iswgraph.c',
          'iswlower.c',
          'iswprint.c',
          'iswpunct.c',
          'iswspace.c',
          'iswupper.c',
          'iswxdigit.c',
          'towctrans.c',
          'wcswidth.c',
          'wctrans.c',
          'wcwidth.c',
         ]],
         ['multibyte', [
          'btowc.c',
          'mblen.c',
          'mbrlen.c',
          'mbrtowc.c',
          'mbsinit.c',
          'mbsnrtowcs.c',
          'mbsrtowcs.c',
          'mbstowcs.c',
          'mbtowc.c',
          'wcrtomb.c',
          'wcsnrtombs.c',
          'wcsrtombs.c',
          'wcstombs.c',
          'wctob.c',
          'wctomb.c',
         ]],
         ['string', [
           'wcpcpy.c',
           'wcpncpy.c',
           'wcscasecmp.c',
           # 'wcscasecmp_l.c', # XXX: alltypes.h issue
           'wcscat.c',
           'wcschr.c',
           'wcscmp.c',
           'wcscpy.c',
           'wcscspn.c',
           'wcsdup.c',
           'wcslen.c',
           'wcsncasecmp.c',
           # 'wcsncasecmp_l.c', # XXX: alltypes.h issue
           'wcsncat.c',
           'wcsncmp.c',
           'wcsncpy.c',
           'wcsnlen.c',
           'wcspbrk.c',
           'wcsrchr.c',
           'wcsspn.c',
           'wcsstr.c',
           'wcstok.c',
           'wcswcs.c',
           'wmemchr.c',
           'wmemcmp.c',
           'wmemcpy.c',
           'wmemmove.c',
           'wmemset.c',
         ]]
      ]
      libcextra_files = []
      for directory, sources in musl_files:
        libcextra_files += [os.path.join('libc', 'musl', 'src', directory, source) for source in sources]
      return build_libc('libcextra.bc', libcextra_files)

    def fix_libcextra(need):
      pass

    # libcxx
    def create_libcxx():
      logging.debug('building libcxx for cache')
      libcxx_files = [
        'algorithm.cpp',
        'condition_variable.cpp',
        'future.cpp',
        'iostream.cpp',
        'memory.cpp',
        'random.cpp',
        'stdexcept.cpp',
        'system_error.cpp',
        'utility.cpp',
        'bind.cpp',
        'debug.cpp',
        'hash.cpp',
        'mutex.cpp',
        'string.cpp',
        'thread.cpp',
        'valarray.cpp',
        'chrono.cpp',
        'exception.cpp',
        'ios.cpp',
        'locale.cpp',
        'regex.cpp',
        'strstream.cpp'
      ]
      return build_libcxx(os.path.join('system', 'lib', 'libcxx'), 'libcxx.bc', libcxx_files)

    def fix_libcxx(need):
      assert shared.Settings.QUANTUM_SIZE == 4, 'We do not support libc++ with QUANTUM_SIZE == 1'
      # libcxx might need corrections, so turn them all on. TODO: check which are actually needed
      shared.Settings.CORRECT_SIGNS = shared.Settings.CORRECT_OVERFLOWS = shared.Settings.CORRECT_ROUNDINGS = 1
      #logging.info('using libcxx turns on CORRECT_* options')

    # libcxxabi - just for dynamic_cast for now
    def create_libcxxabi():
      logging.debug('building libcxxabi for cache')
      libcxxabi_files = [
        'typeinfo.cpp',
        'private_typeinfo.cpp'
      ]
      return build_libcxx(os.path.join('system', 'lib', 'libcxxabi', 'src'), 'libcxxabi.bc', libcxxabi_files)

    def fix_libcxxabi(need):
      assert shared.Settings.QUANTUM_SIZE == 4, 'We do not support libc++abi with QUANTUM_SIZE == 1'
      #logging.info('using libcxxabi, this may need CORRECT_* options')
      #shared.Settings.CORRECT_SIGNS = shared.Settings.CORRECT_OVERFLOWS = shared.Settings.CORRECT_ROUNDINGS = 1

    # If we have libcxx, we must force inclusion of libc, since libcxx uses new internally. Note: this is kind of hacky
    # Settings this in the environment will avoid checking dependencies and make building big projects a little faster
    force = os.environ.get('EMCC_FORCE_STDLIBS')
    has = need = None
    for name, create, fix, library_symbols in [('libcxx',    create_libcxx,    fix_libcxx,    libcxx_symbols),
                                               ('libcextra', create_libcextra, fix_libcextra, libcextra_symbols),
                                               ('libcxxabi', create_libcxxabi, fix_libcxxabi, libcxxabi_symbols),
                                               ('libc',      create_libc,      fix_libc,      libc_symbols)]:
      if not force:
        need = set()
        has = set()
        for temp_file in temp_files:
          symbols = shared.Building.llvm_nm(temp_file)
          for library_symbol in library_symbols:
            if library_symbol in symbols.undefs:
              need.add(library_symbol)
            if library_symbol in symbols.defs:
              has.add(library_symbol)
        for haz in has: # remove symbols that are supplied by another of the inputs
          if haz in need:
            need.remove(haz)
        logging.debug('considering including %s: we need %s and have %s' % (name, str(need), str(has)))
      if force or len(need) > 0:
        # We need to build and link the library in
        logging.debug('including %s' % name)
        libfile = shared.Cache.get(name, create)
        extra_files_to_link.append(libfile)
        force = True
        if fix and need:
          fix(need)

  # First, combine the bitcode files if there are several. We must also link if we have a singleton .a
  if len(input_files) + len(extra_files_to_link) > 1 or \
     (not LEAVE_INPUTS_RAW and not (suffix(temp_files[0]) in BITCODE_SUFFIXES or suffix(temp_files[0]) in DYNAMICLIB_SUFFIXES) and shared.Building.is_ar(temp_files[0])):
    linker_inputs = temp_files + extra_files_to_link
    logging.debug('linking: ', linker_inputs)
    t0 = time.time()
    shared.Building.link(linker_inputs, in_temp(target_basename + '.bc'))
    t1 = time.time()
    logging.debug('    linking took %.2f seconds' % (t1 - t0))
    final = in_temp(target_basename + '.bc')
  else:
    if not LEAVE_INPUTS_RAW:
      shutil.move(temp_files[0], in_temp(target_basename + '.bc'))
      final = in_temp(target_basename + '.bc')
    else:
      final = in_temp(input_files[0])
      shutil.copyfile(input_files[0], final)

  if DEBUG:
    logging.debug('saving intermediate processing steps to %s' % shared.EMSCRIPTEN_TEMP_DIR)

    intermediate_counter = 0
    intermediate_time = None
    def save_intermediate(name=None, suffix='js'):
      global intermediate_counter, intermediate_time
      shutil.copyfile(final, os.path.join(shared.EMSCRIPTEN_TEMP_DIR, 'emcc-%d%s.%s' % (intermediate_counter, '' if name is None else '-' + name, suffix)))
      intermediate_counter += 1
      now = time.time()
      if intermediate_time:
        logging.debug('    step took %.2f seconds' % (now - intermediate_time))
      intermediate_time = now

    if not LEAVE_INPUTS_RAW: save_intermediate('basebc', 'bc')

  # Optimize, if asked to
  if not LEAVE_INPUTS_RAW:
    link_opts = [] if keep_llvm_debug else ['-strip-debug'] # remove LLVM debug info in -O1+, since the optimizer removes it anyhow
    if llvm_opts > 0:
      if not os.environ.get('EMCC_OPTIMIZE_NORMALLY'):
        shared.Building.llvm_opt(in_temp(target_basename + '.bc'), llvm_opts)
        if DEBUG: save_intermediate('opt', 'bc')
        # Do LTO in a separate pass to work around LLVM bug XXX (see failure e.g. in cubescript)
      else:
        logging.debug('not running opt because EMCC_OPTIMIZE_NORMALLY was specified, opt should have been run before')
    if shared.Building.can_build_standalone():
      # If we can LTO, do it before dce, since it opens up dce opportunities
      if llvm_lto and shared.Building.can_use_unsafe_opts():
        if not shared.Building.can_inline(): link_opts.append('-disable-inlining')
        # do not internalize in std-link-opts - it ignores internalize-public-api-list - and add a manual internalize
        link_opts += ['-disable-internalize'] + shared.Building.get_safe_internalize() + ['-std-link-opts']
      else:
        # At minimum remove dead functions etc., this potentially saves a lot in the size of the generated code (and the time to compile it)
        link_opts += shared.Building.get_safe_internalize() + ['-globaldce']
      shared.Building.llvm_opt(in_temp(target_basename + '.bc'), link_opts)
      if DEBUG: save_intermediate('linktime', 'bc')

  if save_bc:
    shutil.copyfile(final, save_bc)

  # Prepare .ll for Emscripten
  if not LEAVE_INPUTS_RAW:
    final = shared.Building.llvm_dis(final, final + '.ll')
  else:
    assert len(input_files) == 1
  if DEBUG: save_intermediate('ll', 'll')

  if AUTODEBUG:
    logging.debug('autodebug')
    execute([shared.PYTHON, shared.AUTODEBUGGER, final, final + '.ad.ll'])
    final += '.ad.ll'
    if DEBUG: save_intermediate('autodebug', 'll')

  # Emscripten
  logging.debug('LLVM => JS')
  extra_args = [] if not js_libraries else ['--libraries', ','.join(map(os.path.abspath, js_libraries))]
  if jcache: extra_args.append('--jcache')
  final = shared.Building.emscripten(final, append_ext=False, extra_args=extra_args)
  if DEBUG: save_intermediate('original')

  # Embed and preload files
  if len(preload_files) + len(embed_files) > 0:
    logging.debug('setting up files')
    file_args = []
    if len(preload_files) > 0:
      file_args.append('--preload')
      file_args += preload_files
    if len(embed_files) > 0:
      file_args.append('--embed')
      file_args += embed_files
    if Compression.on:
      file_args += ['--compress', Compression.encoder, Compression.decoder, Compression.js_name]
    if use_preload_cache:
      file_args.append('--use-preload-cache')
    code = execute([shared.PYTHON, shared.FILE_PACKAGER, unsuffixed(target) + '.data'] + file_args, stdout=PIPE)[0]
    src = open(final).read().replace('// {{PRE_RUN_ADDITIONS}}', '// {{PRE_RUN_ADDITIONS}}\n' + code)
    final += '.files.js'
    open(final, 'w').write(src)
    if DEBUG: save_intermediate('files')

  # Apply pre and postjs files
  if pre_js or post_js:
    logging.debug('applying pre/postjses')
    src = open(final).read()
    final += '.pp.js'
    open(final, 'w').write(pre_js + src + post_js)
    if DEBUG: save_intermediate('pre-post')

  # Add bindings glue if used
  if bind:
    logging.debug('adding embind glue')
    src = open(final).read().replace('// {{PRE_RUN_ADDITIONS}}', '// {{PRE_RUN_ADDITIONS}}\n' +
            open(shared.path_from_root('src', 'embind', 'embind.js')).read() +
            open(shared.path_from_root('src', 'embind', 'emval.js')).read()
          )
    final += '.bd.js'
    open(final, 'w').write(src)
    if DEBUG: save_intermediate('bind')

  # Apply a source code transformation, if requested
  if js_transform:
    shutil.copyfile(final, final + '.tr.js')
    final += '.tr.js'
    posix = True if not shared.WINDOWS else False
    logging.debug('applying transform: %s' % js_transform)
    execute(shlex.split(js_transform, posix=posix) + [os.path.abspath(final)])
    if DEBUG: save_intermediate('transformed')

  # It is useful to run several js optimizer passes together, to save on unneeded unparsing/reparsing
  js_optimizer_queue = []
  def flush_js_optimizer_queue():
    global final, js_optimizer_queue
    if len(js_optimizer_queue) > 0 and not(len(js_optimizer_queue) == 1 and js_optimizer_queue[0] == 'last'):
      if DEBUG != '2':
        if shared.Settings.ASM_JS:
          js_optimizer_queue = ['asm'] + js_optimizer_queue
        logging.debug('applying js optimization passes:', js_optimizer_queue)
        final = shared.Building.js_optimizer(final, js_optimizer_queue, jcache)
        if DEBUG: save_intermediate('js_opts')
      else:
        for name in js_optimizer_queue:
          passes = [name]
          if shared.Settings.ASM_JS:
            passes = ['asm'] + passes
          logging.info('applying js optimization pass:', passes)
          final = shared.Building.js_optimizer(final, passes, jcache)
          save_intermediate(name)
      js_optimizer_queue = []

  if opt_level >= 1:
    logging.debug('running pre-closure post-opts')

    if DEBUG == '2':
      # Clean up the syntax a bit
      final = shared.Building.js_optimizer(final, [], jcache)
      if DEBUG: save_intermediate('pretty')

    def get_eliminate():
      if shared.Settings.ALLOW_MEMORY_GROWTH:
        return 'eliminateMemSafe'
      else:
        return 'eliminate'

    js_optimizer_queue += [get_eliminate(), 'simplifyExpressionsPre']

    if shared.Settings.RELOOP and not shared.Settings.ASM_JS:
      js_optimizer_queue += ['optimizeShiftsAggressive', get_eliminate()] # aggressive shifts optimization requires loops, it breaks on switches

  if closure:
    flush_js_optimizer_queue()

    logging.debug('running closure')
    final = shared.Building.closure_compiler(final)
    if DEBUG: save_intermediate('closure')

  if opt_level >= 1:
    logging.debug('running post-closure post-opts')
    js_optimizer_queue += ['simplifyExpressionsPost']

  if not closure and shared.Settings.RELOOP and not keep_js_debug:
    # do this if closure is not enabled (it gives similar speedups), and we do not need to keep debug info around
    js_optimizer_queue += ['registerize']

  if minify_whitespace:
    js_optimizer_queue += ['compress']

  js_optimizer_queue += ['last']

  flush_js_optimizer_queue()

  # Remove some trivial whitespace # TODO: do not run when compress has already been done on all parts of the code
  src = open(final).read()
  src = re.sub(r'\n+[ \n]*\n+', '\n', src)
  open(final, 'w').write(src)

  if memory_init_file:
    if shared.Settings.USE_TYPED_ARRAYS != 2:
      if type(memory_init_file) == int: logging.warning('memory init file requires typed arrays mode 2')
    else:
      memfile = target + '.mem'
      shared.try_delete(memfile)
      def repl(m):
        # handle chunking of the memory initializer
        s = re.sub('[\[\]\n\(\)\. ]', '', m.groups(0)[0])
        s = s.replace('concat', ',')
        if s[-1] == ',': s = s[:-1]
        open(memfile, 'wb').write(''.join(map(lambda x: chr(int(x or '0')), s.split(','))))
        if DEBUG:
          # Copy into temp dir as well, so can be run there too
          temp_memfile = os.path.join(shared.EMSCRIPTEN_TEMP_DIR, os.path.basename(memfile))
          if os.path.abspath(memfile) != os.path.abspath(memfile):
            shutil.copyfile(memfile, temp_memfile)
        return 'loadMemoryInitializer("%s");' % os.path.basename(memfile)
      src = re.sub('/\* memory initializer \*/ allocate\(([\d,\.concat\(\)\[\]\\n ]+)"i8", ALLOC_NONE, TOTAL_STACK\)', repl, src, count=1)
      open(final + '.mem.js', 'w').write(src)
      final += '.mem.js'
      if DEBUG:
        if os.path.exists(memfile):
          save_intermediate('meminit')
          logging.debug('wrote memory initialization to %s' % memfile)
        else:
          logging.debug('did not see memory initialization')

  # If we were asked to also generate HTML, do that
  if final_suffix == 'html':
    logging.debug('generating HTML')
    shell = open(shell_path).read()
    html = open(target, 'w')
    if not Compression.on:
      html.write(shell.replace('{{{ SCRIPT_CODE }}}', open(final).read()))
    else:
      # Compress the main code
      js_target = unsuffixed(target) + '.js'
      shutil.move(final, js_target)
      Compression.compress(js_target)

      # Run the decompressor in a worker, and add code to
      #   1. download the compressed file
      #   2. decompress to a typed array
      #   3. convert to a string of source code
      #   4. insert a script element with that source code (more effective than eval)
      decoding = '''
        var decompressWorker = new Worker('decompress.js');
        var decompressCallbacks = [];
        var decompressions = 0;
        Module["decompress"] = function(data, callback) {
          var id = decompressCallbacks.length;
          decompressCallbacks.push(callback);
          decompressWorker.postMessage({ data: data, id: id });
          if (Module['setStatus']) {
            decompressions++;
            Module['setStatus']('Decompressing...');
          }
        };
        decompressWorker.onmessage = function(event) {
          decompressCallbacks[event.data.id](event.data.data);
          decompressCallbacks[event.data.id] = null;
          if (Module['setStatus']) {
            decompressions--;
            if (decompressions == 0) {
              Module['setStatus']('');
            }
          }
        };
        var compiledCodeXHR = new XMLHttpRequest();
        compiledCodeXHR.open('GET', '%s', true);
        compiledCodeXHR.responseType = 'arraybuffer';
        compiledCodeXHR.onload = function() {
          var arrayBuffer = compiledCodeXHR.response;
          if (!arrayBuffer) throw('Loading compressed code failed.');
          var byteArray = new Uint8Array(arrayBuffer);
          Module.decompress(byteArray, function(decompressed) {
            var source = Array.prototype.slice.apply(decompressed).map(function(x) { return String.fromCharCode(x) }).join(''); // createObjectURL instead?
            var scriptTag = document.createElement('script');
            scriptTag.setAttribute('type', 'text/javascript');
            scriptTag.innerHTML = source;
            document.body.appendChild(scriptTag);
          });
        };
        compiledCodeXHR.send(null);
''' % Compression.compressed_name(js_target)
      html.write(shell.replace('{{{ SCRIPT_CODE }}}', decoding))

      # Add decompressor with web worker glue code
      decompressor = open('decompress.js', 'w')
      decompressor.write(open(Compression.decoder).read())
      decompressor.write('''
        onmessage = function(event) {
          postMessage({ data: %s(event.data.data), id: event.data.id });
        };
''' % Compression.js_name)
      decompressor.close()

    html.close()
  else:
    if split_js_file:
        from tools.split import split_javascript_file
        split_javascript_file(final, unsuffixed(target), split_js_file)
    else:
        # copy final JS to output
        shutil.move(final, target)

  if DEBUG: logging.debug('total time: %.2f seconds' % (time.time() - start_time))

finally:
  if not TEMP_DIR:
    try:
      shutil.rmtree(temp_dir)
    except:
      pass
  else:
    logging.info('emcc saved files are in:' + temp_dir)