pjs/xpcom/idl-parser/xpidl.py

#!/usr/bin/env python
# xpidl.py - A parser for cross-platform IDL (XPIDL) files.
#
# ***** BEGIN LICENSE BLOCK *****
# Version: MPL 1.1/GPL 2.0/LGPL 2.1
#
# The contents of this file are subject to the Mozilla Public License Version
# 1.1 (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
# http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS" basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.
#
# The Original Code is mozilla.org code.
#
# The Initial Developer of the Original Code is
#   Mozilla Foundation.
# Portions created by the Initial Developer are Copyright (C) 2008
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Benjamin Smedberg <benjamin@smedbergs.us>
#
# Alternatively, the contents of this file may be used under the terms of
# either of the GNU General Public License Version 2 or later (the "GPL"),
# or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
# in which case the provisions of the GPL or the LGPL are applicable instead
# of those above. If you wish to allow use of your version of this file only
# under the terms of either the GPL or the LGPL, and not to allow others to
# use your version of this file under the terms of the MPL, indicate your
# decision by deleting the provisions above and replace them with the notice
# and other provisions required by the GPL or the LGPL. If you do not delete
# the provisions above, a recipient may use your version of this file under
# the terms of any one of the MPL, the GPL or the LGPL.
#
# ***** END LICENSE BLOCK *****

"""A parser for cross-platform IDL (XPIDL) files."""

import sys, os.path, re

sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
                             'other-licenses', 'ply'))
from ply import lex, yacc

"""A type conforms to the following pattern:

    def isScriptable(self):
        'returns True or False'

    def nativeType(self, calltype):
        'returns a string representation of the native type
        calltype must be 'in', 'out', or 'inout'

Interface members const/method/attribute conform to the following pattern:

    name = 'string'

    def toIDL(self):
        'returns the member signature as IDL'
"""

def attlistToIDL(attlist):
    if len(attlist) == 0:
        return ''

    attlist = list(attlist)
    attlist.sort(cmp=lambda a,b: cmp(a[0], b[0]))

    return '[%s] ' % ','.join(["%s%s" % (name, value is not None and '(%s)' % value or '')
                              for name, value, aloc in attlist])

_paramsHardcode = {
    2: ('array', 'shared', 'iid_is', 'size_is', 'retval'),
    3: ('array', 'size_is', 'const'),
}

def paramAttlistToIDL(attlist):
    if len(attlist) == 0:
        return ''

    # Hack alert: g_hash_table_foreach is pretty much unimitatable... hardcode
    # quirk
    attlist = list(attlist)
    sorted = []
    if len(attlist) in _paramsHardcode:
        for p in _paramsHardcode[len(attlist)]:
            i = 0
            while i < len(attlist):
                if attlist[i][0] == p:
                    sorted.append(attlist[i])
                    del attlist[i]
                    continue

                i += 1

    sorted.extend(attlist)

    return '[%s] ' % ', '.join(["%s%s" % (name, value is not None and ' (%s)' % value or '')
                                for name, value, aloc in sorted])

class BuiltinLocation(object):
    def get(self):
        return "<builtin type>"

    def __str__(self):
        return self.get()

class Builtin(object):
    kind = 'builtin'
    location = BuiltinLocation

    def __init__(self, name, nativename, signed=False, maybeConst=False):
        self.name = name
        self.nativename = nativename
        self.signed = signed
        self.maybeConst = maybeConst

    def isScriptable(self):
        return True

    def nativeType(self, calltype, shared=False, const=False):
        if const:
            print >>sys.stderr, IDLError("[const] doesn't make sense on builtin types.", self.location, warning=True)
            const = 'const '
        elif calltype == 'in' and self.nativename.endswith('*'):
            const = 'const '
        elif shared:
            if not self.nativename.endswith('*'):
                raise IDLError("[shared] not applicable to non-pointer types.", self.location)
            const = 'const '
        else:
            const = ''
        return "%s%s %s" % (const, self.nativename,
                            calltype != 'in' and '*' or '')

builtinNames = [
    Builtin('boolean', 'PRBool'),
    Builtin('void', 'void'),
    Builtin('octet', 'PRUint8'),
    Builtin('short', 'PRInt16', True, True),
    Builtin('long', 'PRInt32', True, True),
    Builtin('long long', 'PRInt64', True, False),
    Builtin('unsigned short', 'PRUint16', False, True),
    Builtin('unsigned long', 'PRUint32', False, True),
    Builtin('unsigned long long', 'PRUint64', False, False),
    Builtin('float', 'float', True, False),
    Builtin('double', 'double', True, False),
    Builtin('char', 'char', True, False),
    Builtin('string', 'char *', False, False),
    Builtin('wchar', 'PRUnichar', False, False),
    Builtin('wstring', 'PRUnichar *', False, False),
]

builtinMap = {}
for b in builtinNames:
    builtinMap[b.name] = b

class Location(object):
    _line = None

    def __init__(self, lexer, lineno, lexpos):
        self._lineno = lineno
        self._lexpos = lexpos
        self._lexdata = lexer.lexdata
        self._file = getattr(lexer, 'filename', "<unknown>")

    def __eq__(self, other):
        return self._lexpos == other._lexpos and \
               self._file == other._file

    def resolve(self):
        if self._line:
            return

        startofline = self._lexdata.rfind('\n', 0, self._lexpos) + 1
        endofline = self._lexdata.find('\n', self._lexpos, self._lexpos + 80)
        self._line = self._lexdata[startofline:endofline]
        self._colno = self._lexpos - startofline

    def pointerline(self):
        def i():
            for i in xrange(0, self._colno):
                yield " "
            yield "^"

        return "".join(i())

    def get(self):
        self.resolve()
        return "%s line %s:%s" % (self._file, self._lineno, self._colno)

    def __str__(self):
        self.resolve()
        return "%s line %s:%s\n%s\n%s" % (self._file, self._lineno, self._colno,
                                          self._line, self.pointerline())

class NameMap(object):
    """Map of name -> object. Each object must have a .name and .location property.
    Setting the same name twice throws an error."""
    def __init__(self):
        self._d = {}

    def __getitem__(self, key):
        if key in builtinMap:
            return builtinMap[key]
        return self._d[key]

    def __iter__(self):
        return self._d.itervalues()

    def __contains__(self, key):
        return key in builtinMap or key in self._d

    def set(self, object):
        if object.name in builtinMap:
            raise IDLError("name '%s' is a builtin and cannot be redeclared" % (object.name), object.location)
        if object.name in self._d:
            old = self._d[object.name]
            if old == object: return
            if isinstance(old, Forward) and isinstance(object, Interface):
                self._d[object.name] = object
            elif isinstance(old, Interface) and isinstance(object, Forward):
                pass
            else:
                raise IDLError("name '%s' specified twice. Previous location: %s" % (object.name, self._d[object.name].location), object.location)
        else:
            self._d[object.name] = object

    def get(self, id, location):
        try:
            return self[id]
        except KeyError:
            raise IDLError("Name '%s' not found", location)

class IDLError(Exception):
    def __init__(self, message, location, warning=False):
        self.message = message
        self.location = location
        self.warning = warning

    def __str__(self):
        return "%s: %s, %s" % (self.warning and 'warning' or 'error',
                               self.message, self.location)

class Include(object):
    kind = 'include'

    def __init__(self, filename, location):
        self.filename = filename
        self.location = location

    def __str__(self):
        return "".join(["include '%s'\n" % self.filename])

    def resolve(self, parent):
        def incfiles():
            yield self.filename
            for dir in parent.incdirs:
                yield os.path.join(dir, self.filename)

        for file in incfiles():
            if not os.path.exists(file): continue

            self.IDL = parent.parser.parse(open(file).read(), filename=file)
            self.IDL.resolve(parent.incdirs, parent.parser)
            for type in self.IDL.getNames():
                parent.setName(type)
            return

        raise IDLError("File '%s' not found" % self.filename, self.location)

class IDL(object):
    def __init__(self, productions):
        self.productions = productions

    def setName(self, object):
        self.namemap.set(object)

    def getName(self, id, location):
        try:
            return self.namemap[id]
        except KeyError:
            raise IDLError("type '%s' not found" % id, location)

    def hasName(self, id):
        return id in self.namemap

    def getNames(self):
        return iter(self.namemap)

    def __str__(self):
        return "".join([str(p) for p in self.productions])

    def resolve(self, incdirs, parser):
        self.namemap = NameMap()
        self.incdirs = incdirs
        self.parser = parser
        for p in self.productions:
            p.resolve(self)

    def includes(self):
        for p in self.productions:
            if p.kind == 'include':
                yield p

class CDATA(object):
    kind = 'cdata'
    _re = re.compile(r'\n+')

    def __init__(self, data, location):
        self.data = self._re.sub('\n', data)
        self.location = location

    def resolve(self, parent):
        pass

    def __str__(self):
        return "cdata: %s\n\t%r\n" % (self.location.get(), self.data)

class Typedef(object):
    kind = 'typedef'

    def __init__(self, type, name, location, doccomments):
        self.type = type
        self.name = name
        self.location = location
        self.doccomments = doccomments

    def __eq__(self, other):
        return self.name == other.name and self.type == other.type

    def resolve(self, parent):
        parent.setName(self)
        self.realtype = parent.getName(self.type, self.location)

    def isScriptable(self):
        return self.realtype.isScriptable()

    def nativeType(self, calltype):
        return "%s %s" % (self.name,
                          calltype != 'in' and '*' or '')

    def __str__(self):
        return "typedef %s %s\n" % (self.type, self.name)

class Forward(object):
    kind = 'forward'

    def __init__(self, name, location, doccomments):
        self.name = name
        self.location = location
        self.doccomments = doccomments

    def __eq__(self, other):
        return other.kind == 'forward' and other.name == self.name

    def resolve(self, parent):
        # Hack alert: if an identifier is already present, move the doccomments
        # forward.
        if parent.hasName(self.name):
            for i in xrange(0, len(parent.productions)):
                if parent.productions[i] is self: break
            for i in xrange(i + 1, len(parent.productions)):
                if hasattr(parent.productions[i], 'doccomments'):
                    parent.productions[i].doccomments[0:0] = self.doccomments
                    break

        parent.setName(self)

    def isScriptable(self):
        return True

    def nativeType(self, calltype):
        return "%s %s" % (self.name,
                          calltype != 'in' and '* *' or '*')

    def __str__(self):
        return "forward-declared %s\n" % self.name

class Native(object):
    kind = 'native'

    modifier = None
    specialtype = None

    specialtypes = {
        'nsid': None,
        'domstring': 'nsAString',
        'utf8string': 'nsACString',
        'cstring': 'nsACString',
        'astring': 'nsAString'
        }

    def __init__(self, name, nativename, attlist, location):
        self.name = name
        self.nativename = nativename
        self.location = location

        for name, value, aloc in attlist:
            if value is not None:
                raise IDLError("Unexpected attribute value", aloc)
            if name in ('ptr', 'ref'):
                if self.modifier is not None:
                    raise IDLError("More than one ptr/ref modifier", aloc)
                self.modifier = name
            elif name in self.specialtypes.keys():
                if self.specialtype is not None:
                    raise IDLError("More than one special type", aloc)
                self.specialtype = name
                if self.specialtypes[name] is not None:
                    self.nativename = self.specialtypes[name]
            else:
                raise IDLError("Unexpected attribute", aloc)

    def __eq__(self, other):
        return self.name == other.name and \
               self.nativename == other.nativename and \
               self.modifier == other.modifier and \
               self.specialtype == other.specialtype

    def resolve(self, parent):
        parent.setName(self)

    def isScriptable(self):
        if self.specialtype is None:
            return False

        if self.specialtype == 'nsid':
            return self.modifier is not None

        return self.modifier == 'ref'

    def nativeType(self, calltype, const=False, shared=False):
        if shared:
            if calltype != 'out':
                raise IDLError("[shared] only applies to out parameters.")
            const = True

        if self.specialtype is not None and calltype == 'in':
            const = True

        if self.modifier == 'ptr':
            m = '*' + (calltype != 'in' and '*' or '')
        elif self.modifier == 'ref':
            m = '& '
        else:
            m = calltype != 'in' and '*' or ''
        return "%s%s %s" % (const and 'const ' or '', self.nativename, m)

    def __str__(self):
        return "native %s(%s)\n" % (self.name, self.nativename)

class Interface(object):
    kind = 'interface'

    def __init__(self, name, attlist, base, members, location, doccomments):
        self.name = name
        self.attributes = InterfaceAttributes(attlist, location)
        self.base = base
        self.members = members
        self.location = location
        self.namemap = NameMap()
        self.doccomments = doccomments
        self.nativename = name

        for m in members:
            if not isinstance(m, CDATA):
                self.namemap.set(m)

    def __eq__(self, other):
        return self.name == other.name and self.location == other.location

    def resolve(self, parent):
        self.idl = parent

        # Hack alert: if an identifier is already present, libIDL assigns
        # doc comments incorrectly. This is quirks-mode extraordinaire!
        if parent.hasName(self.name):
            for member in self.members:
                if hasattr(member, 'doccomments'):
                    member.doccomments[0:0] = self.doccomments
                    break
            self.doccomments = parent.getName(self.name, None).doccomments

        parent.setName(self)
        if self.base is None:
            if self.name != 'nsISupports':
                print >>sys.stderr, IDLError("interface '%s' not derived from nsISupports",
                                             self.location, warning=True)
        else:
            realbase = parent.getName(self.base, self.location)
            if realbase.kind != 'interface':
                raise IDLError("interface '%s' inherits from non-interface type '%s'" % (self.name, self.base), self.location)

            if self.attributes.scriptable and not realbase.attributes.scriptable:
                print >>sys.stderr, IDLError("interface '%s' is scriptable but derives from non-scriptable '%s'" % (self.name, self.base), self.location, warning=True)

        for member in self.members:
            member.resolve(self)

    def isScriptable(self):
        # NOTE: this is not whether *this* interface is scriptable... it's
        # whether, when used as a type, it's scriptable, which is true of all
        # interfaces.
        return True

    def nativeType(self, calltype, const=False):
        return "%s%s %s" % (const and 'const ' or '',
                            self.name,
                            calltype != 'in' and '* *' or '*')

    def __str__(self):
        l = ["interface %s\n" % self.name]
        if self.base is not None:
            l.append("\tbase %s\n" % self.base)
        l.append(str(self.attributes))
        if self.members is None:
            l.append("\tincomplete type\n")
        else:
            for m in self.members:
                l.append(str(m))
        return "".join(l)

    def getConst(self, name, location):
        c = self.namemap.get(name, location)
        if c.kind != 'const':
            raise IDLError("symbol '%s' is not a constant", c.location)

        return c.getValue()

class InterfaceAttributes(object):
    uuid = None
    scriptable = False
    function = False
    deprecated = False
    noscript = False

    def setuuid(self, value):
        self.uuid = value.lower()

    def setscriptable(self):
        self.scriptable = True

    def setfunction(self):
        self.function = True

    def setnoscript(self):
        self.noscript = True

    def setdeprecated(self):
        self.deprecated = True

    actions = {
        'uuid':       (True, setuuid),
        'scriptable': (False, setscriptable),
        'function':   (False, setfunction),
        'noscript':   (False, setnoscript),
        'deprecated': (False, setdeprecated),
        'object':     (False, lambda self: True),
        }

    def __init__(self, attlist, location):
        def badattribute(self):
            raise IDLError("Unexpected interface attribute '%s'" % name, location)

        for name, val, aloc in attlist:
            hasval, action = self.actions.get(name, (False, badattribute))
            if hasval:
                if val is None:
                    raise IDLError("Expected value for attribute '%s'" % name,
                                   aloc)

                action(self, val)
            else:
                if val is not None:
                    raise IDLError("Unexpected value for attribute '%s'" % name,
                                   aloc)

                action(self)

        if self.uuid is None:
            raise IDLError("interface has no uuid", location)

    def __str__(self):
        l = []
        if self.uuid:
            l.append("\tuuid: %s\n" % self.uuid)
        if self.scriptable:
            l.append("\tscriptable\n")
        if self.function:
            l.append("\tfunction\n")
        return "".join(l)

class ConstMember(object):
    kind = 'const'
    def __init__(self, type, name, value, location, doccomments):
        self.type = type
        self.name = name
        self.value = value
        self.location = location
        self.doccomments = doccomments

    def resolve(self, parent):
        self.realtype = parent.idl.getName(self.type, self.location)
        self.iface = parent
        basetype = self.realtype
        while isinstance(basetype, Typedef):
            basetype = basetype.realtype
        if not isinstance(basetype, Builtin) or not basetype.maybeConst:
            raise IDLError("const may only be a short or long type, not %s" % self.type, self.location)

        self.basetype = basetype

    def getValue(self):
        return self.value(self.iface)

    def __str__(self):
        return "\tconst %s %s = %s\n" % (self.type, self.name, self.getValue())

class Attribute(object):
    kind = 'attribute'
    noscript = False
    notxpcom = False
    readonly = False
    binaryname = None

    def __init__(self, type, name, attlist, readonly, location, doccomments):
        self.type = type
        self.name = name
        self.attlist = attlist
        self.readonly = readonly
        self.location = location
        self.doccomments = doccomments

        for name, value, aloc in attlist:
            if name == 'binaryname':
                if value is None:
                    raise IDLError("binaryname attribute requires a value",
                                   aloc)

                self.binaryname = value
                continue

            if value is not None:
                raise IDLError("Unexpected attribute value", aloc)

            if name == 'noscript':
                self.noscript = True
            elif name == 'notxpcom':
                self.notxpcom = True
            else:
                raise IDLError("Unexpected attribute '%s'", aloc)

    def resolve(self, iface):
        self.iface = iface
        self.realtype = iface.idl.getName(self.type, self.location)

    def toIDL(self):
        attribs = attlistToIDL(self.attlist)
        readonly = self.readonly and 'readonly ' or ''
        return "%s%sattribute %s %s;" % (attribs, readonly, self.type, self.name)
        
    def isScriptable(self):
        if not self.iface.attributes.scriptable: return False
        return not (self.noscript or self.notxpcom)

    def __str__(self):
        return "\t%sattribute %s %s\n" % (self.readonly and 'readonly ' or '',
                                          self.type, self.name)

class Method(object):
    kind = 'method'
    noscript = False
    notxpcom = False
    binaryname = None

    def __init__(self, type, name, attlist, paramlist, location, doccomments, raises):
        self.type = type
        self.name = name
        self.attlist = attlist
        self.params = paramlist
        self.location = location
        self.doccomments = doccomments
        self.raises = raises

        for name, value, aloc in attlist:
            if name == 'binaryname':
                if value is None:
                    raise IDLError("binaryname attribute requires a value",
                                   aloc)

                self.binaryname = value
                continue

            if value is not None:
                raise IDLError("Unexpected attribute value", aloc)

            if name == 'noscript':
                self.noscript = True
            elif name == 'notxpcom':
                self.notxpcom = True
            else:
                raise IDLError("Unexpected attribute '%s'", aloc)

        self.namemap = NameMap()
        for p in paramlist:
            self.namemap.set(p)

    def resolve(self, iface):
        self.iface = iface
        self.realtype = self.iface.idl.getName(self.type, self.location)
        for p in self.params:
            p.resolve(self)

    def isScriptable(self):
        if not self.iface.attributes.scriptable: return False
        return not (self.noscript or self.notxpcom)

    def __str__(self):
        return "\t%s %s(%s)\n" % (self.type, self.name, ", ".join([p.name for p in self.params]))

    def toIDL(self):
        if len(self.raises):
            raises = ' raises (%s)' % ','.join(self.raises)
        else:
            raises = ''

        return "%s%s %s (%s)%s;" % (attlistToIDL(self.attlist),
                                    self.type,
                                    self.name,
                                    ", ".join([p.toIDL()
                                               for p in self.params]),
                                    raises)

class Param(object):
    size_is = None
    iid_is = None
    const = False
    array = False
    retval = False
    shared = False
    optional = False

    def __init__(self, paramtype, type, name, attlist, location, realtype=None):
        self.paramtype = paramtype
        self.type = type
        self.name = name
        self.attlist = attlist
        self.location = location
        self.realtype = realtype

        for name, value, aloc in attlist:
            # Put the value-taking attributes first!
            if name == 'size_is':
                if value is None:
                    raise IDLError("'size_is' must specify a parameter", aloc)
                self.size_is = value
            elif name == 'iid_is':
                if value is None:
                    raise IDLError("'iid_is' must specify a parameter", aloc)
                self.iid_is = value
            else:
                if value is not None:
                    raise IDLError("Unexpected value for attribute '%s'" % name,
                                   aloc)

                if name == 'const':
                    self.const = True
                elif name == 'array':
                    self.array = True
                elif name == 'retval':
                    self.retval = True
                elif name == 'shared':
                    self.shared = True
                elif name == 'optional':
                    self.optional = True
                else:
                    raise IDLError("Unexpected attribute '%s'" % name, aloc)

    def resolve(self, method):
        self.realtype = method.iface.idl.getName(self.type, self.location)
        if self.array:
            self.realtype = Array(self.realtype)

    def nativeType(self):
        kwargs = {}
        if self.shared: kwargs['shared'] = True
        if self.const: kwargs['const'] = True

        try:
            return self.realtype.nativeType(self.paramtype, **kwargs)
        except IDLError, e:
            raise IDLError(e.message, self.location)
        except TypeError, e:
            raise IDLError("Unexpected parameter attribute", self.location)

    def toIDL(self):
        return "%s%s %s %s" % (paramAttlistToIDL(self.attlist),
                               self.paramtype,
                               self.type,
                               self.name)

class Array(object):
    def __init__(self, basetype):
        self.type = basetype

    def isScriptable(self):
        return self.type.isScriptable()

    def nativeType(self, calltype, const=False):
        return "%s%s*" % (const and 'const ' or '',
                          self.type.nativeType(calltype))

class IDLParser(object):
    keywords = {
        'const': 'CONST',
        'interface': 'INTERFACE',
        'in': 'IN',
        'inout': 'INOUT',
        'out': 'OUT',
        'attribute': 'ATTRIBUTE',
        'raises': 'RAISES',
        'readonly': 'READONLY',
        'native': 'NATIVE',
        'typedef': 'TYPEDEF'
        }

    tokens = [
        'IDENTIFIER',
        'CDATA',
        'INCLUDE',
        'IID',
        'NUMBER',
        'HEXNUM',
        'LSHIFT',
        'RSHIFT',
        'NATIVEID',
        ]

    tokens.extend(keywords.values())

    states = (
        ('nativeid', 'exclusive'),
    )

    hexchar = r'[a-fA-F0-9]'

    t_NUMBER = r'-?\d+'
    t_HEXNUM = r'0x%s+' % hexchar
    t_LSHIFT = r'<<'
    t_RSHIFT=  r'>>'

    literals = '"(){}[],;:=|+-*'

    t_ignore = ' \t'

    def t_multilinecomment(self, t):
        r'/\*(?s).*?\*/'
        t.lexer.lineno += t.value.count('\n')
        if t.value.startswith("/**"):
            self._doccomments.append(t.value)

    def t_singlelinecomment(self, t):
        r'(?m)//.*?$'

    def t_IID(self, t):
        return t
    t_IID.__doc__ = r'%(c)s{8}-%(c)s{4}-%(c)s{4}-%(c)s{4}-%(c)s{12}' % {'c': hexchar}

    def t_IDENTIFIER(self, t):
        r'unsigned\ long\ long|unsigned\ short|unsigned\ long|long\ long|[A-Za-z][A-Za-z_0-9]*'
        t.type = self.keywords.get(t.value, 'IDENTIFIER')
        return t

    def t_LCDATA(self, t):
        r'(?s)%\{[ ]*C\+\+[ ]*\n(?P<cdata>.*?\n?)%\}[ ]*(C\+\+)?'
        t.type = 'CDATA'
        t.value = t.lexer.lexmatch.group('cdata')
        t.lexer.lineno += t.value.count('\n')
        return t

    def t_INCLUDE(self, t):
        r'\#include[ \t]+"[^"\n]+"'
        inc, value, end = t.value.split('"')
        t.value = value
        return t

    def t_directive(self, t):
        r'\#(?P<directive>[a-zA-Z]+)[^\n]+'
        print >>sys.stderr, IDLError("Unrecognized directive %s" % t.lexer.lexmatch.group('directive'),
                                     Location(lexer=self.lexer,
                                              lineno=self.lexer.lineno,
                                              lexpos=self.lexer.lexpos))

    def t_newline(self, t):
        r'\n+'
        t.lexer.lineno += len(t.value)

    def t_nativeid_NATIVEID(self, t):
        r'[^()\n]+(?=\))'
        t.lexer.begin('INITIAL')
        return t

    t_nativeid_ignore = ''

    def t_ANY_error(self, t):
        raise IDLError("unrecognized input",
                       Location(lexer=self.lexer,
                                lineno=self.lexer.lineno,
                                lexpos=self.lexer.lexpos))

    precedence = (
        ('left', '|'),
        ('left', 'LSHIFT', 'RSHIFT'),
        ('left', '+', '-'),
        ('left', '*'),
        ('left', 'UMINUS'),
    )

    def p_idlfile(self, p):
        """idlfile : productions"""
        p[0] = IDL(p[1])

    def p_productions_start(self, p):
        """productions : """
        p[0] = []

    def p_productions_cdata(self, p):
        """productions : CDATA productions"""
        p[0] = list(p[2])
        p[0].insert(0, CDATA(p[1], self.getLocation(p, 1)))

    def p_productions_include(self, p):
        """productions : INCLUDE productions"""
        p[0] = list(p[2])
        p[0].insert(0, Include(p[1], self.getLocation(p, 1)))

    def p_productions_interface(self, p):
        """productions : interface productions
                       | typedef productions
                       | native productions"""
        p[0] = list(p[2])
        p[0].insert(0, p[1])

    def p_typedef(self, p):
        """typedef : TYPEDEF IDENTIFIER IDENTIFIER ';'"""
        p[0] = Typedef(type=p[2],
                       name=p[3],
                       location=self.getLocation(p, 1),
                       doccomments=p.slice[1].doccomments)

    def p_native(self, p):
        """native : attributes NATIVE IDENTIFIER afternativeid '(' NATIVEID ')' ';'"""
        p[0] = Native(name=p[3],
                      nativename=p[6],
                      attlist=p[1]['attlist'],
                      location=self.getLocation(p, 2))

    def p_afternativeid(self, p):
        """afternativeid : """
        # this is a place marker: we switch the lexer into literal identifier
        # mode here, to slurp up everything until the closeparen
        self.lexer.begin('nativeid')

    def p_anyident(self, p):
        """anyident : IDENTIFIER
                    | CONST"""
        p[0] = {'value': p[1],
                'location': self.getLocation(p, 1)}

    def p_attributes(self, p):
        """attributes : '[' attlist ']'
                      | """
        if len(p) == 1:
            p[0] = {'attlist': []}
        else:
            p[0] = {'attlist': p[2],
                    'doccomments': p.slice[1].doccomments}

    def p_attlist_start(self, p):
        """attlist : attribute"""
        p[0] = [p[1]]

    def p_attlist_continue(self, p):
        """attlist : attribute ',' attlist"""
        p[0] = list(p[3])
        p[0].insert(0, p[1])

    def p_attribute(self, p):
        """attribute : anyident attributeval"""
        p[0] = (p[1]['value'], p[2], p[1]['location'])

    def p_attributeval(self, p):
        """attributeval : '(' IDENTIFIER ')'
                        | '(' IID ')'
                        | """
        if len(p) > 1:
            p[0] = p[2]

    def p_interface(self, p):
        """interface : attributes INTERFACE IDENTIFIER ifacebase ifacebody ';'"""
        atts, INTERFACE, name, base, body, SEMI = p[1:]
        attlist = atts['attlist']
        doccomments = []
        if 'doccomments' in atts:
            doccomments.extend(atts['doccomments'])
        doccomments.extend(p.slice[2].doccomments)

        l = lambda: self.getLocation(p, 2)

        if body is None:
            # forward-declared interface... must not have attributes!
            if len(attlist) != 0:
                raise IDLError("Forward-declared interface must not have attributes",
                               list[0][3])

            if base is not None:
                raise IDLError("Forward-declared interface must not have a base",
                               l())
            p[0] = Forward(name=name, location=l(), doccomments=doccomments)
        else:
            p[0] = Interface(name=name,
                             attlist=attlist,
                             base=base,
                             members=body,
                             location=l(),
                             doccomments=doccomments)

    def p_ifacebody(self, p):
        """ifacebody : '{' members '}'
                     | """
        if len(p) > 1:
            p[0] = p[2]

    def p_ifacebase(self, p):
        """ifacebase : ':' IDENTIFIER
                     | """
        if len(p) == 3:
            p[0] = p[2]

    def p_members_start(self, p):
        """members : """
        p[0] = []

    def p_members_continue(self, p):
        """members : member members"""
        p[0] = list(p[2])
        p[0].insert(0, p[1])

    def p_member_cdata(self, p):
        """member : CDATA"""
        p[0] = CDATA(p[1], self.getLocation(p, 1))

    def p_member_const(self, p):
        """member : CONST IDENTIFIER IDENTIFIER '=' number ';' """
        p[0] = ConstMember(type=p[2], name=p[3],
                           value=p[5], location=self.getLocation(p, 1),
                           doccomments=p.slice[1].doccomments)

# All "number" products return a function(interface)

    def p_number_decimal(self, p):
        """number : NUMBER"""
        n = int(p[1])
        p[0] = lambda i: n

    def p_number_hex(self, p):
        """number : HEXNUM"""
        n = int(p[1], 16)
        p[0] = lambda i: n

    def p_number_identifier(self, p):
        """number : IDENTIFIER"""
        id = p[1]
        loc = self.getLocation(p, 1)
        p[0] = lambda i: i.getConst(id, loc)

    def p_number_paren(self, p):
        """number : '(' number ')'"""
        p[0] = p[2]

    def p_number_neg(self, p):
        """number : '-' number %prec UMINUS"""
        n = p[2]
        p[0] = lambda i: - n(i)

    def p_number_add(self, p):
        """number : number '+' number
                  | number '-' number
                  | number '*' number"""
        n1 = p[1]
        n2 = p[3]
        if p[2] == '+':
            p[0] = lambda i: n1(i) + n2(i)
        elif p[2] == '-':
            p[0] = lambda i: n1(i) - n2(i)
        else:
            p[0] = lambda i: n1(i) * n2(i)

    def p_number_shift(self, p):
        """number : number LSHIFT number
                  | number RSHIFT number"""
        n1 = p[1]
        n2 = p[3]
        if p[2] == '<<':
            p[0] = lambda i: n1(i) << n2(i)
        else:
            p[0] = lambda i: n1(i) >> n2(i)

    def p_number_bitor(self, p):
        """number : number '|' number"""
        n1 = p[1]
        n2 = p[3]
        p[0] = lambda i: n1(i) | n2(i)

    def p_member_att(self, p):
        """member : attributes optreadonly ATTRIBUTE IDENTIFIER IDENTIFIER ';'"""
        if 'doccomments' in p[1]:
            doccomments = p[1]['doccomments']
        elif p[2] is not None:
            doccomments = p[2]
        else:
            doccomments = p.slice[3].doccomments

        p[0] = Attribute(type=p[4],
                         name=p[5],
                         attlist=p[1]['attlist'],
                         readonly=p[2] is not None,
                         location=self.getLocation(p, 3),
                         doccomments=doccomments)

    def p_member_method(self, p):
        """member : attributes IDENTIFIER IDENTIFIER '(' paramlist ')' raises ';'"""
        if 'doccomments' in p[1]:
            doccomments = p[1]['doccomments']
        else:
            doccomments = p.slice[2].doccomments

        p[0] = Method(type=p[2],
                      name=p[3],
                      attlist=p[1]['attlist'],
                      paramlist=p[5],
                      location=self.getLocation(p, 3),
                      doccomments=doccomments,
                      raises=p[7])

    def p_paramlist(self, p):
        """paramlist : param moreparams
                     | """
        if len(p) == 1:
            p[0] = []
        else:
            p[0] = list(p[2])
            p[0].insert(0, p[1])

    def p_moreparams_start(self, p):
        """moreparams :"""
        p[0] = []

    def p_moreparams_continue(self, p):
        """moreparams : ',' param moreparams"""
        p[0] = list(p[3])
        p[0].insert(0, p[2])

    def p_param(self, p):
        """param : attributes paramtype IDENTIFIER IDENTIFIER"""
        p[0] = Param(paramtype=p[2],
                     type=p[3],
                     name=p[4],
                     attlist=p[1]['attlist'],
                     location=self.getLocation(p, 3))

    def p_paramtype(self, p):
        """paramtype : IN
                     | INOUT
                     | OUT"""
        p[0] = p[1]

    def p_optreadonly(self, p):
        """optreadonly : READONLY
                       | """
        if len(p) > 1:
            p[0] = p.slice[1].doccomments
        else:
            p[0] = None

    def p_raises(self, p):
        """raises : RAISES '(' idlist ')'
                  | """
        if len(p) == 1:
            p[0] = []
        else:
            p[0] = p[3]

    def p_idlist(self, p):
        """idlist : IDENTIFIER"""
        p[0] = [p[1]]

    def p_idlist_continue(self, p):
        """idlist : IDENTIFIER ',' idlist"""
        p[0] = list(p[3])
        p[0].insert(0, p[1])

    def p_error(self, t):
        location = Location(self.lexer, t.lineno, t.lexpos)
        raise IDLError("invalid syntax", location)

    def __init__(self, outputdir=''):
        self._doccomments = []
        self.lexer = lex.lex(object=self,
                             outputdir=outputdir,
                             lextab='xpidllex',
                             optimize=1)
        self.parser = yacc.yacc(module=self,
                                outputdir=outputdir,
                                debugfile='xpidl_debug',
                                tabmodule='xpidlyacc',
                                optimize=1)

    def clearComments(self):
        self._doccomments = []

    def token(self):
        t = self.lexer.token()
        if t is not None and t.type != 'CDATA':
            t.doccomments = self._doccomments
            self._doccomments = []
        return t

    def parse(self, data, filename=None):
        if filename is not None:
            self.lexer.filename = filename
        self.lexer.lineno = 1
        self.lexer.input(data)
        return self.parser.parse(lexer=self)

    def getLocation(self, p, i):
        return Location(self.lexer, p.lineno(i), p.lexpos(i))

if __name__ == '__main__':
    p = IDLParser()
    for f in sys.argv[1:]:
        print "Parsing %s" % f
        p.parse(open(f).read(), filename=f)