emscripten/tools/file_packager.py

'''
A tool that generates FS API calls to generate a filesystem, and packages the files
to work with that.

This is called by emcc. You can also call it yourself.

Usage:

  file_packager.py TARGET [--preload A [B..]] [--embed C [D..]] [--compress COMPRESSION_DATA] [--pre-run]

'''

import os, sys

from shared import Compression, execute

def suffix(name):
  return name.split('.')[-1]

IMAGE_SUFFIXES = ('.jpg', '.png', '.bmp')
AUDIO_SUFFIXES = ('.ogg', '.wav', '.mp3')
AUDIO_MIMETYPES = { 'ogg': 'audio/ogg', 'wav': 'audio/wav', 'mp3': 'audio/mpeg' }

data_files = []
in_preload = False
in_embed = False
has_preloaded = False
in_compress = 0
pre_run = False
for arg in sys.argv[1:]:
  if arg == '--preload':
    in_preload = True
    in_embed = False
    has_preloaded = True
    in_compress = 0
  elif arg == '--embed':
    in_embed = True
    in_preload = False
    in_compress = 0
  elif arg == '--compress':
    Compression.on = True
    in_compress = 1
    in_preload = False
    in_embed = False
  elif arg == '--pre-run':
    pre_run = True
    in_preload = False
    in_embed = False
    in_compress = 0
  elif in_preload:
    data_files.append({ 'name': arg, 'mode': 'preload' })
  elif in_embed:
    data_files.append({ 'name': arg, 'mode': 'embed' })
  elif in_compress:
    if in_compress == 1:
      Compression.encoder = arg
      in_compress = 2
    elif in_compress == 2:
      Compression.decoder = arg
      in_compress = 3
    elif in_compress == 3:
      Compression.js_name = arg
      in_compress = 0

code = '''
function assert(check, msg) {
  if (!check) throw msg;
}
'''

if has_preloaded:
  code += '''
  var BlobBuilder = typeof MozBlobBuilder != "undefined" ? MozBlobBuilder : (typeof WebKitBlobBuilder != "undefined" ? WebKitBlobBuilder : console.log("warning: cannot build blobs"));
  var URLObject = typeof window != "undefined" ? (window.URL ? window.URL : window.webkitURL) : console.log("warning: cannot create object URLs");
  var hasBlobConstructor;
  try {
    new Blob();
    hasBlobConstructor = true;
  } catch(e) {
    hasBlobConstructor = false;
    console.log("warning: no blob constructor, cannot create blobs with mimetypes");
  }
'''

  code += 'Module["preloadedImages"] = {}; // maps url to image data\n'
  code += 'Module["preloadedAudios"] = {}; // maps url to audio data\n'

# Expand directories into individual files
def add(mode, dirname, names):
  for name in names:
    fullname = os.path.join(dirname, name)
    if not os.path.isdir(fullname):
      data_files.append({ 'name': fullname, 'mode': mode })

for file_ in data_files:
  if os.path.isdir(file_['name']):
    os.path.walk(file_['name'], add, file_['mode'])
data_files = filter(lambda file_: not os.path.isdir(file_['name']), data_files)

for file_ in data_files:
  file_['name'] = file_['name'].replace(os.path.sep, '/')
  file_['net_name'] = file_['name']

# remove duplicates (can occur naively, for example preload dir/, preload dir/subdir/)
seen = {}
def was_seen(name):
  if seen.get(name): return True
  seen[name] = 1
  return False
data_files = filter(lambda file_: not was_seen(file_['name']), data_files)

data_target = sys.argv[1]

# Set up folders
partial_dirs = []
for file_ in data_files:
  dirname = os.path.dirname(file_['name'])
  dirname = dirname.lstrip('/') # absolute paths start with '/', remove that
  if dirname != '':
    parts = dirname.split('/')
    for i in range(len(parts)):
      partial = '/'.join(parts[:i+1])
      if partial not in partial_dirs:
        code += '''Module['FS_createFolder']('/%s', '%s', true, true);\n''' % ('/'.join(parts[:i]), parts[i])
        partial_dirs.append(partial)

if has_preloaded:
  # Bundle all datafiles into one archive. Avoids doing lots of simultaneous XHRs which has overhead.
  data = open(data_target, 'wb')
  start = 0
  for file_ in data_files:
    file_['data_start'] = start
    curr = open(file_['name'], 'rb').read()
    file_['data_end'] = start + len(curr)
    start += len(curr)
    data.write(curr)
  data.close()
  if Compression.on:
    Compression.compress(data_target)

  # Data requests - for getting a block of data out of the big archive - have a similar API to XHRs
  code += '''
    function DataRequest() {}
    DataRequest.prototype = {
      requests: {},
      open: function(mode, name) {
        this.requests[name] = this;
      },
      send: function() {}
    };
  '''

counter = 0
for file_ in data_files:
  filename = file_['name']
  if file_['mode'] == 'embed':
    # Embed
    code += '''Module['FS_createDataFile']('/', '%s', %s, true, true);\n''' % (os.path.basename(filename), str(map(ord, open(filename, 'rb').read())))
  elif file_['mode'] == 'preload':
    # Preload
    varname = 'filePreload%d' % counter
    counter += 1
    image = filename.endswith(IMAGE_SUFFIXES)
    audio = filename.endswith(AUDIO_SUFFIXES)

    if image:
      finish =  '''
        var bb = new BlobBuilder();
        bb.append(byteArray.buffer);
        var b = bb.getBlob();
        var url = URLObject.createObjectURL(b);
        var img = new Image();
        img.onload = function() {
          assert(img.complete, 'Image %(filename)s could not be decoded');
          var canvas = document.createElement('canvas');
          canvas.width = img.width;
          canvas.height = img.height;
          var ctx = canvas.getContext('2d');
          ctx.drawImage(img, 0, 0);
          Module["preloadedImages"]['%(filename)s'] = canvas;
          URLObject.revokeObjectURL(url);
          Module['removeRunDependency']();
        };
        img.onerror = function(event) {
          console.log('Image %(filename)s could not be decoded');
        };
        img.src = url;
''' % { 'filename': filename }
    elif audio:
      # Need actual blob constructor here, to set the mimetype or else audios fail to decode
      finish =  '''
        if (hasBlobConstructor) {
          var b = new Blob([byteArray.buffer], { type: '%(mimetype)s' });
          var url = URLObject.createObjectURL(b); // XXX we never revoke this!
          var audio = new Audio();
          audio.removedDependency = false;
          audio['oncanplaythrough'] = function() { // XXX string for closure
            audio['oncanplaythrough'] = null;
            Module["preloadedAudios"]['%(filename)s'] = audio;
            if (!audio.removedDependency) {
              Module['removeRunDependency']();
              audio.removedDependency = true;
            }
          };
          audio.onerror = function(event) {
            if (!audio.removedDependency) {
              console.log('Audio %(filename)s could not be decoded or timed out trying to decode');
              Module['removeRunDependency']();
              audio.removedDependency = true;
            }
          };
          setTimeout(audio.onerror, 2000); // workaround for chromium bug 124926 (still no audio with this, but at least we don't hang)
          audio.src = url;
        } else {
          Module["preloadedAudios"]['%(filename)s'] = new Audio(); // empty shim
          Module['removeRunDependency']();
        }
''' % { 'filename': filename, 'mimetype': AUDIO_MIMETYPES[suffix(filename)] }
    else:
      finish = "Module['removeRunDependency']();\n"

    code += '''
    var %(varname)s = new %(request)s();
    %(varname)s.open('GET', '%(netname)s', true);
    %(varname)s.responseType = 'arraybuffer';
    %(varname)s.onload = function() {
      var arrayBuffer = %(varname)s.response;
      assert(arrayBuffer, 'Loading file %(filename)s failed.');
      var byteArray = arrayBuffer.byteLength ? new Uint8Array(arrayBuffer) : arrayBuffer;
      Module['FS_createDataFile']('/%(dirname)s', '%(basename)s', byteArray, true, true);
      %(finish)s
    };
    Module['addRunDependency']();
    %(varname)s.send(null);
''' % {
        'request': 'DataRequest', # In the past we also supported XHRs here
        'varname': varname,
        'filename': filename,
        'netname': file_['net_name'],
        'dirname': os.path.dirname(filename),
        'basename': os.path.basename(filename),
        'finish': finish
  }
  else:
    assert 0

if has_preloaded:
  # Get the big archive and split it up
  use_data = ''
  for file_ in data_files:
    if file_['mode'] == 'preload':
      use_data += '''
        curr = DataRequest.prototype.requests['%s'];
        curr.response = byteArray.subarray(%d,%d);
        curr.onload();
      ''' % (file_['name'], file_['data_start'], file_['data_end'])
  use_data += "          Module['removeRunDependency']();\n"

  if Compression.on:
    use_data = '''
      Module["decompress"](byteArray, function(decompressed) {
        byteArray = new Uint8Array(decompressed);
        %s
      });
''' % use_data

  code += '''
    var dataFile = new XMLHttpRequest();
    dataFile.open('GET', '%s', true);
    dataFile.responseType = 'arraybuffer';
    dataFile.onload = function() {
      var arrayBuffer = dataFile.response;
      assert(arrayBuffer, 'Loading data file failed.');
      var byteArray = new Uint8Array(arrayBuffer);
      var curr;
      %s
    };
    Module['addRunDependency']();
    dataFile.send(null);
    if (Module['setStatus']) Module['setStatus']('Downloading...');
  ''' % (Compression.compressed_name(data_target) if Compression.on else data_target, use_data)

if pre_run:
  print 'Module["preRun"] = function() {'

print code

if pre_run:
  print '};'