From cfb1405ad44130f40f01cd52e7f330aa02a8e52b Mon Sep 17 00:00:00 2001
From: Ryan VanderMeulen <ryanvm@gmail.com>
Date: Tue, 19 Mar 2013 15:50:18 -0400
Subject: [PATCH] Backed out changeset d1b71de5bbc1 (bug 747831) for Windows
 bustage.

---
 js/src/frontend/BytecodeEmitter.cpp  |  56 +++----
 js/src/frontend/BytecodeEmitter.h    |  10 +-
 js/src/frontend/FullParseHandler.h   |  12 +-
 js/src/frontend/ParseNode.h          |  19 ++-
 js/src/frontend/Parser.cpp           |  57 +++----
 js/src/frontend/SyntaxParseHandler.h |  14 +-
 js/src/frontend/TokenStream.cpp      | 239 ++++++++++-----------------
 js/src/frontend/TokenStream.h        | 142 +++++++---------
 js/src/ion/AsmJS.cpp                 |   3 +-
 js/src/jsfun.cpp                     |   2 +-
 js/src/jsprvtd.h                     |   1 +
 js/src/jsreflect.cpp                 |  23 +--
 12 files changed, 236 insertions(+), 342 deletions(-)

diff --git a/js/src/frontend/BytecodeEmitter.cpp b/js/src/frontend/BytecodeEmitter.cpp
index 657dc2d4aeaa..44bdcf37f0bf 100644
--- a/js/src/frontend/BytecodeEmitter.cpp
+++ b/js/src/frontend/BytecodeEmitter.cpp
@@ -90,12 +90,12 @@ struct frontend::StmtInfoBCE : public StmtInfoBase
 BytecodeEmitter::BytecodeEmitter(BytecodeEmitter *parent,
                                  Parser<FullParseHandler> *parser, SharedContext *sc,
                                  HandleScript script, HandleScript evalCaller, bool hasGlobalScope,
-                                 uint32_t lineNum, bool selfHostingMode)
+                                 unsigned lineno, bool selfHostingMode)
   : sc(sc),
     parent(parent),
     script(sc->context, script),
-    prolog(sc->context, lineNum),
-    main(sc->context, lineNum),
+    prolog(sc->context, lineno),
+    main(sc->context, lineno),
     current(&main),
     parser(parser),
     evalCaller(evalCaller),
@@ -103,7 +103,7 @@ BytecodeEmitter::BytecodeEmitter(BytecodeEmitter *parent,
     topScopeStmt(NULL),
     blockChain(sc->context),
     atomIndices(sc->context),
-    firstLine(lineNum),
+    firstLine(lineno),
     stackDepth(0), maxStackDepth(0),
     tryNoteList(sc->context),
     arrayCompDepth(0),
@@ -341,13 +341,10 @@ EmitBackPatchOp(JSContext *cx, BytecodeEmitter *bce, ptrdiff_t *lastp)
 
 /* Updates line number notes, not column notes. */
 static inline bool
-UpdateLineNumberNotes(JSContext *cx, BytecodeEmitter *bce, uint32_t offset)
+UpdateLineNumberNotes(JSContext *cx, BytecodeEmitter *bce, unsigned line)
 {
-    TokenStream *ts = &bce->parser->tokenStream;
-    if (!ts->srcCoords.isOnThisLine(offset, bce->currentLine())) {
-        unsigned line = ts->srcCoords.lineNum(offset);
-        unsigned delta = line - bce->currentLine();
-
+    unsigned delta = line - bce->currentLine();
+    if (delta != 0) {
         /*
          * Encode any change in the current source line number by using
          * either several SRC_NEWLINE notes or just one SRC_SETLINE note,
@@ -376,13 +373,13 @@ UpdateLineNumberNotes(JSContext *cx, BytecodeEmitter *bce, uint32_t offset)
 
 /* A function, so that we avoid macro-bloating all the other callsites. */
 static bool
-UpdateSourceCoordNotes(JSContext *cx, BytecodeEmitter *bce, uint32_t offset)
+UpdateSourceCoordNotes(JSContext *cx, BytecodeEmitter *bce, TokenPtr pos)
 {
-    if (!UpdateLineNumberNotes(cx, bce, offset))
+    if (!UpdateLineNumberNotes(cx, bce, pos.lineno))
         return false;
 
-    uint32_t columnIndex = bce->parser->tokenStream.srcCoords.columnIndex(offset);
-    ptrdiff_t colspan = ptrdiff_t(columnIndex) - ptrdiff_t(bce->current->lastColumn);
+    ptrdiff_t colspan = ptrdiff_t(pos.index) -
+                        ptrdiff_t(bce->current->lastColumn);
     if (colspan != 0) {
         if (colspan < 0) {
             colspan += SN_COLSPAN_DOMAIN;
@@ -396,7 +393,7 @@ UpdateSourceCoordNotes(JSContext *cx, BytecodeEmitter *bce, uint32_t offset)
         }
         if (NewSrcNote2(cx, bce, SRC_COLSPAN, colspan) < 0)
             return false;
-        bce->current->lastColumn = columnIndex;
+        bce->current->lastColumn = pos.index;
     }
     return true;
 }
@@ -1672,8 +1669,7 @@ BytecodeEmitter::reportError(ParseNode *pn, unsigned errorNumber, ...)
 
     va_list args;
     va_start(args, errorNumber);
-    bool result = tokenStream()->reportCompileErrorNumberVA(pos.begin, JSREPORT_ERROR,
-                                                            errorNumber, args);
+    bool result = tokenStream()->reportCompileErrorNumberVA(pos, JSREPORT_ERROR, errorNumber, args);
     va_end(args);
     return result;
 }
@@ -1685,7 +1681,7 @@ BytecodeEmitter::reportStrictWarning(ParseNode *pn, unsigned errorNumber, ...)
 
     va_list args;
     va_start(args, errorNumber);
-    bool result = tokenStream()->reportStrictWarningErrorNumberVA(pos.begin, errorNumber, args);
+    bool result = tokenStream()->reportStrictWarningErrorNumberVA(pos, errorNumber, args);
     va_end(args);
     return result;
 }
@@ -1697,8 +1693,7 @@ BytecodeEmitter::reportStrictModeError(ParseNode *pn, unsigned errorNumber, ...)
 
     va_list args;
     va_start(args, errorNumber);
-    bool result = tokenStream()->reportStrictModeErrorNumberVA(pos.begin, sc->strict,
-                                                               errorNumber, args);
+    bool result = tokenStream()->reportStrictModeErrorNumberVA(pos, sc->strict, errorNumber, args);
     va_end(args);
     return result;
 }
@@ -4315,11 +4310,11 @@ EmitNormalFor(JSContext *cx, BytecodeEmitter *bce, ParseNode *pn, ptrdiff_t top)
             return false;
 
         /* Restore the absolute line number for source note readers. */
-        uint32_t lineNum = bce->parser->tokenStream.srcCoords.lineNum(pn->pn_pos.end);
-        if (bce->currentLine() != lineNum) {
-            if (NewSrcNote2(cx, bce, SRC_SETLINE, ptrdiff_t(lineNum)) < 0)
+        ptrdiff_t lineno = pn->pn_pos.end.lineno;
+        if (bce->currentLine() != (unsigned) lineno) {
+            if (NewSrcNote2(cx, bce, SRC_SETLINE, lineno) < 0)
                 return false;
-            bce->current->currentLine = lineNum;
+            bce->current->currentLine = (unsigned) lineno;
             bce->current->lastColumn = 0;
         }
     }
@@ -4412,9 +4407,8 @@ EmitFunc(JSContext *cx, BytecodeEmitter *bce, ParseNode *pn)
             return false;
 #endif
 
-        uint32_t lineNum = bce->parser->tokenStream.srcCoords.lineNum(pn->pn_pos.begin);
         BytecodeEmitter bce2(bce, bce->parser, funbox, script, bce->evalCaller,
-                             bce->hasGlobalScope, lineNum, bce->selfHostingMode);
+                             bce->hasGlobalScope, pn->pn_pos.begin.lineno, bce->selfHostingMode);
         if (!bce2.init())
             return false;
 
@@ -4746,7 +4740,7 @@ EmitStatement(JSContext *cx, BytecodeEmitter *bce, ParseNode *pn)
         }
     } else if (!pn->isDirectivePrologueMember()) {
         /* Don't complain about directive prologue members; just don't emit their code. */
-        bce->current->currentLine = bce->parser->tokenStream.srcCoords.lineNum(pn2->pn_pos.begin);
+        bce->current->currentLine = pn2->pn_pos.begin.lineno;
         bce->current->lastColumn = 0;
         if (!bce->reportStrictWarning(pn2, JSMSG_USELESS_EXPR))
             return false;
@@ -4961,10 +4955,8 @@ EmitCallOrNew(JSContext *cx, BytecodeEmitter *bce, ParseNode *pn)
     if (Emit3(cx, bce, pn->getOp(), ARGC_HI(argc), ARGC_LO(argc)) < 0)
         return false;
     CheckTypeSet(cx, bce, pn->getOp());
-    if (pn->isOp(JSOP_EVAL)) {
-        uint32_t lineNum = bce->parser->tokenStream.srcCoords.lineNum(pn->pn_pos.begin);
-        EMIT_UINT16_IMM_OP(JSOP_LINENO, lineNum);
-    }
+    if (pn->isOp(JSOP_EVAL))
+        EMIT_UINT16_IMM_OP(JSOP_LINENO, pn->pn_pos.begin.lineno);
     if (pn->pn_xflags & PNX_SETCALL) {
         if (Emit1(cx, bce, JSOP_SETCALL) < 0)
             return false;
@@ -5482,7 +5474,7 @@ frontend::EmitTree(JSContext *cx, BytecodeEmitter *bce, ParseNode *pn)
     pn->pn_offset = top;
 
     /* Emit notes to tell the current bytecode's source line number. */
-    if (!UpdateLineNumberNotes(cx, bce, pn->pn_pos.begin))
+    if (!UpdateLineNumberNotes(cx, bce, pn->pn_pos.begin.lineno))
         return false;
 
     switch (pn->getKind()) {
diff --git a/js/src/frontend/BytecodeEmitter.h b/js/src/frontend/BytecodeEmitter.h
index de36be36b112..c1e209d39bb1 100644
--- a/js/src/frontend/BytecodeEmitter.h
+++ b/js/src/frontend/BytecodeEmitter.h
@@ -78,12 +78,12 @@ struct BytecodeEmitter
         BytecodeVector code;        /* bytecode */
         SrcNotesVector notes;       /* source notes, see below */
         ptrdiff_t   lastNoteOffset; /* code offset for last source note */
-        uint32_t    currentLine;    /* line number for tree-based srcnote gen */
-        uint32_t    lastColumn;     /* zero-based column index on currentLine of
+        unsigned    currentLine;    /* line number for tree-based srcnote gen */
+        unsigned    lastColumn;     /* zero-based column index on currentLine of
                                        last SRC_COLSPAN-annotated opcode */
 
-        EmitSection(JSContext *cx, uint32_t lineNum)
-          : code(cx), notes(cx), lastNoteOffset(0), currentLine(lineNum), lastColumn(0)
+        EmitSection(JSContext *cx, unsigned lineno)
+          : code(cx), notes(cx), lastNoteOffset(0), currentLine(lineno), lastColumn(0)
         {}
     };
     EmitSection prolog, main, *current;
@@ -141,7 +141,7 @@ struct BytecodeEmitter
      */
     BytecodeEmitter(BytecodeEmitter *parent, Parser<FullParseHandler> *parser, SharedContext *sc,
                     HandleScript script, HandleScript evalCaller, bool hasGlobalScope,
-                    uint32_t lineNum, bool selfHostingMode = false);
+                    unsigned lineno, bool selfHostingMode = false);
     bool init();
 
     bool isAliasedName(ParseNode *pn);
diff --git a/js/src/frontend/FullParseHandler.h b/js/src/frontend/FullParseHandler.h
index a8990c612535..f8e560208ee1 100644
--- a/js/src/frontend/FullParseHandler.h
+++ b/js/src/frontend/FullParseHandler.h
@@ -135,19 +135,19 @@ class FullParseHandler
         return new_<TernaryNode>(kind, op, first, second, third);
     }
 
-    ParseNode *newBreak(PropertyName *label, uint32_t begin, uint32_t end) {
+    ParseNode *newBreak(PropertyName *label, const TokenPtr &begin, const TokenPtr &end) {
         return new_<BreakStatement>(label, begin, end);
     }
-    ParseNode *newContinue(PropertyName *label, uint32_t begin, uint32_t end) {
+    ParseNode *newContinue(PropertyName *label, const TokenPtr &begin, const TokenPtr &end) {
         return new_<ContinueStatement>(label, begin, end);
     }
     ParseNode *newDebuggerStatement(const TokenPos &pos) {
         return new_<DebuggerStatement>(pos);
     }
-    ParseNode *newPropertyAccess(ParseNode *pn, PropertyName *name, uint32_t end) {
+    ParseNode *newPropertyAccess(ParseNode *pn, PropertyName *name, const TokenPtr &end) {
         return new_<PropertyAccess>(pn, name, pn->pn_pos.begin, end);
     }
-    ParseNode *newPropertyByValue(ParseNode *pn, ParseNode *kid, uint32_t end) {
+    ParseNode *newPropertyByValue(ParseNode *pn, ParseNode *kid, const TokenPtr &end) {
         return new_<PropertyByValue>(pn, kid, pn->pn_pos.begin, end);
     }
 
@@ -177,7 +177,7 @@ class FullParseHandler
     void setBeginPosition(ParseNode *pn, ParseNode *oth) {
         setBeginPosition(pn, oth->pn_pos.begin);
     }
-    void setBeginPosition(ParseNode *pn, uint32_t begin) {
+    void setBeginPosition(ParseNode *pn, const TokenPtr &begin) {
         pn->pn_pos.begin = begin;
         JS_ASSERT(pn->pn_pos.begin <= pn->pn_pos.end);
     }
@@ -185,7 +185,7 @@ class FullParseHandler
     void setEndPosition(ParseNode *pn, ParseNode *oth) {
         setEndPosition(pn, oth->pn_pos.end);
     }
-    void setEndPosition(ParseNode *pn, uint32_t end) {
+    void setEndPosition(ParseNode *pn, const TokenPtr &end) {
         pn->pn_pos.end = end;
         JS_ASSERT(pn->pn_pos.begin <= pn->pn_pos.end);
     }
diff --git a/js/src/frontend/ParseNode.h b/js/src/frontend/ParseNode.h
index 81742675bca2..eca05c14e7f2 100644
--- a/js/src/frontend/ParseNode.h
+++ b/js/src/frontend/ParseNode.h
@@ -430,8 +430,10 @@ struct ParseNode {
         pn_offset(0), pn_next(NULL), pn_link(NULL)
     {
         JS_ASSERT(kind < PNK_LIMIT);
-        pn_pos.begin = 0;
-        pn_pos.end = 0;
+        pn_pos.begin.index = 0;
+        pn_pos.begin.lineno = 0;
+        pn_pos.end.index = 0;
+        pn_pos.end.lineno = 0;
         memset(&pn_u, 0, sizeof pn_u);
     }
 
@@ -960,7 +962,8 @@ struct LexicalScopeNode : public ParseNode {
 
 class LoopControlStatement : public ParseNode {
   protected:
-    LoopControlStatement(ParseNodeKind kind, PropertyName *label, uint32_t begin, uint32_t end)
+    LoopControlStatement(ParseNodeKind kind, PropertyName *label,
+                         const TokenPtr &begin, const TokenPtr &end)
       : ParseNode(kind, JSOP_NOP, PN_NULLARY, TokenPos::make(begin, end))
     {
         JS_ASSERT(kind == PNK_BREAK || kind == PNK_CONTINUE);
@@ -983,7 +986,7 @@ class LoopControlStatement : public ParseNode {
 
 class BreakStatement : public LoopControlStatement {
   public:
-    BreakStatement(PropertyName *label, uint32_t begin, uint32_t end)
+    BreakStatement(PropertyName *label, const TokenPtr &begin, const TokenPtr &end)
       : LoopControlStatement(PNK_BREAK, label, begin, end)
     { }
 
@@ -997,7 +1000,7 @@ class BreakStatement : public LoopControlStatement {
 
 class ContinueStatement : public LoopControlStatement {
   public:
-    ContinueStatement(PropertyName *label, uint32_t begin, uint32_t end)
+    ContinueStatement(PropertyName *label, TokenPtr &begin, TokenPtr &end)
       : LoopControlStatement(PNK_CONTINUE, label, begin, end)
     { }
 
@@ -1069,7 +1072,8 @@ class BooleanLiteral : public ParseNode {
 
 class PropertyAccess : public ParseNode {
   public:
-    PropertyAccess(ParseNode *lhs, PropertyName *name, uint32_t begin, uint32_t end)
+    PropertyAccess(ParseNode *lhs, PropertyName *name,
+                   const TokenPtr &begin, const TokenPtr &end)
       : ParseNode(PNK_DOT, JSOP_GETPROP, PN_NAME, TokenPos::make(begin, end))
     {
         JS_ASSERT(lhs != NULL);
@@ -1095,7 +1099,8 @@ class PropertyAccess : public ParseNode {
 
 class PropertyByValue : public ParseNode {
   public:
-    PropertyByValue(ParseNode *lhs, ParseNode *propExpr, uint32_t begin, uint32_t end)
+    PropertyByValue(ParseNode *lhs, ParseNode *propExpr,
+                    const TokenPtr &begin, const TokenPtr &end)
       : ParseNode(PNK_ELEM, JSOP_GETELEM, PN_BINARY, TokenPos::make(begin, end))
     {
         pn_u.binary.left = lhs;
diff --git a/js/src/frontend/Parser.cpp b/js/src/frontend/Parser.cpp
index dd3e14386468..947b6c6204f8 100644
--- a/js/src/frontend/Parser.cpp
+++ b/js/src/frontend/Parser.cpp
@@ -334,24 +334,23 @@ template <typename ParseHandler>
 bool
 Parser<ParseHandler>::report(ParseReportKind kind, bool strict, Node pn, unsigned errorNumber, ...)
 {
-    uint32_t offset = (pn ? handler.getPosition(pn) : tokenStream.currentToken().pos).begin;
+    TokenPos pos = pn ? handler.getPosition(pn) : tokenStream.currentToken().pos;
 
     va_list args;
     va_start(args, errorNumber);
     bool result = false;
     switch (kind) {
       case ParseError:
-        result = tokenStream.reportCompileErrorNumberVA(offset, JSREPORT_ERROR, errorNumber, args);
+        result = tokenStream.reportCompileErrorNumberVA(pos, JSREPORT_ERROR, errorNumber, args);
         break;
       case ParseWarning:
-        result =
-            tokenStream.reportCompileErrorNumberVA(offset, JSREPORT_WARNING, errorNumber, args);
+        result = tokenStream.reportCompileErrorNumberVA(pos, JSREPORT_WARNING, errorNumber, args);
         break;
       case ParseStrictWarning:
-        result = tokenStream.reportStrictWarningErrorNumberVA(offset, errorNumber, args);
+        result = tokenStream.reportStrictWarningErrorNumberVA(pos, errorNumber, args);
         break;
       case ParseStrictError:
-        result = tokenStream.reportStrictModeErrorNumberVA(offset, strict, errorNumber, args);
+        result = tokenStream.reportStrictModeErrorNumberVA(pos, strict, errorNumber, args);
         break;
     }
     va_end(args);
@@ -1513,7 +1512,7 @@ Parser<ParseHandler>::functionArguments(FunctionSyntaxKind kind, Node *listp, No
 
         // Record the start of function source (for FunctionToString). If we
         // are parenFreeArrow, we will set this below, after consuming the NAME.
-        funbox->bufStart = tokenStream.currentToken().pos.begin;
+        funbox->bufStart = tokenStream.offsetOfToken(tokenStream.currentToken());
     }
 
     hasRest = false;
@@ -1614,7 +1613,7 @@ Parser<ParseHandler>::functionArguments(FunctionSyntaxKind kind, Node *listp, No
               case TOK_NAME:
               {
                 if (parenFreeArrow)
-                    funbox->bufStart = tokenStream.currentToken().pos.begin;
+                    funbox->bufStart = tokenStream.offsetOfToken(tokenStream.currentToken());
 
                 RootedPropertyName name(context, tokenStream.currentToken().name());
                 bool disallowDuplicateArgs = destructuringArg || hasDefaults;
@@ -1981,12 +1980,13 @@ Parser<ParseHandler>::functionArgsAndBody(Node pn, HandleFunction fun, HandlePro
             report(ParseError, false, null(), JSMSG_CURLY_AFTER_BODY);
             return false;
         }
-        funbox->bufEnd = tokenStream.currentToken().pos.begin + 1;
+        funbox->bufEnd = tokenStream.offsetOfToken(tokenStream.currentToken()) + 1;
 #if JS_HAS_EXPR_CLOSURES
     } else {
+        // We shouldn't call endOffset if the tokenizer got an error.
         if (tokenStream.hadError())
             return false;
-        funbox->bufEnd = tokenStream.currentToken().pos.end;
+        funbox->bufEnd = tokenStream.endOffset(tokenStream.currentToken());
         if (kind == Statement && !MatchOrInsertSemicolon(context, &tokenStream))
             return false;
     }
@@ -2097,7 +2097,8 @@ IsEscapeFreeStringLiteral(const TokenPos &pos, JSAtom *str)
      * accounting for the quotes, then it must not contain any escape
      * sequences or line continuations.
      */
-    return pos.begin + str->length() + 2 == pos.end;
+    return (pos.begin.lineno == pos.end.lineno &&
+            pos.begin.index + str->length() + 2 == pos.end.index);
 }
 
 /*
@@ -3025,7 +3026,7 @@ Parser<ParseHandler>::letBlock(LetContext letContext)
     if (!blockObj)
         return null();
 
-    uint32_t begin = tokenStream.currentToken().pos.begin;
+    TokenPtr begin = tokenStream.currentToken().pos.begin;
 
     MUST_MATCH_TOKEN(TOK_LP, JSMSG_PAREN_BEFORE_LET);
 
@@ -3780,7 +3781,7 @@ typename ParseHandler::Node
 Parser<ParseHandler>::tryStatement()
 {
     JS_ASSERT(tokenStream.isCurrentTokenType(TOK_TRY));
-    uint32_t begin = tokenStream.currentToken().pos.begin;
+    TokenPtr begin = tokenStream.currentToken().pos.begin;
 
     /*
      * try nodes are ternary.
@@ -3950,7 +3951,7 @@ typename ParseHandler::Node
 Parser<ParseHandler>::withStatement()
 {
     JS_ASSERT(tokenStream.isCurrentTokenType(TOK_WITH));
-    uint32_t begin = tokenStream.currentToken().pos.begin;
+    TokenPtr begin = tokenStream.currentToken().pos.begin;
 
     // In most cases, we want the constructs forbidden in strict mode code to be
     // a subset of those that JSOPTION_STRICT warns about, and we should use
@@ -4187,7 +4188,7 @@ Parser<ParseHandler>::statement()
 
       case TOK_IF:
       {
-        uint32_t begin = tokenStream.currentToken().pos.begin;
+        TokenPtr begin = tokenStream.currentToken().pos.begin;
 
         /* An IF node has three kids: condition, then, and optional else. */
         Node cond = condition();
@@ -4229,7 +4230,7 @@ Parser<ParseHandler>::statement()
 
       case TOK_WHILE:
       {
-        uint32_t begin = tokenStream.currentToken().pos.begin;
+        TokenPtr begin = tokenStream.currentToken().pos.begin;
         StmtInfoPC stmtInfo(context);
         PushStatementPC(pc, &stmtInfo, STMT_WHILE_LOOP);
         Node cond = condition();
@@ -4248,7 +4249,7 @@ Parser<ParseHandler>::statement()
 
       case TOK_DO:
       {
-        uint32_t begin = tokenStream.currentToken().pos.begin;
+        TokenPtr begin = tokenStream.currentToken().pos.begin;
         StmtInfoPC stmtInfo(context);
         PushStatementPC(pc, &stmtInfo, STMT_DO_LOOP);
         Node body = statement();
@@ -4285,7 +4286,7 @@ Parser<ParseHandler>::statement()
 
       case TOK_THROW:
       {
-        uint32_t begin = tokenStream.currentToken().pos.begin;
+        TokenPtr begin = tokenStream.currentToken().pos.begin;
 
         /* ECMA-262 Edition 3 says 'throw [no LineTerminator here] Expr'. */
         TokenKind tt = tokenStream.peekTokenSameLine(TSF_OPERAND);
@@ -4318,11 +4319,11 @@ Parser<ParseHandler>::statement()
 
       case TOK_BREAK:
       {
-        uint32_t begin = tokenStream.currentToken().pos.begin;
+        TokenPtr begin = tokenStream.currentToken().pos.begin;
         RootedPropertyName label(context);
         if (!MatchLabel(context, &tokenStream, &label))
             return null();
-        uint32_t end = tokenStream.currentToken().pos.end;
+        TokenPtr end = tokenStream.currentToken().pos.end;
         pn = handler.newBreak(label, begin, end);
         if (!pn)
             return null();
@@ -4351,11 +4352,11 @@ Parser<ParseHandler>::statement()
 
       case TOK_CONTINUE:
       {
-        uint32_t begin = tokenStream.currentToken().pos.begin;
+        TokenPtr begin = tokenStream.currentToken().pos.begin;
         RootedPropertyName label(context);
         if (!MatchLabel(context, &tokenStream, &label))
             return null();
-        uint32_t end = tokenStream.currentToken().pos.end;
+        TokenPtr end = tokenStream.currentToken().pos.begin;
         pn = handler.newContinue(label, begin, end);
         if (!pn)
             return null();
@@ -4964,7 +4965,7 @@ Parser<ParseHandler>::assignExpr()
 
         if (tokenStream.getToken() == TOK_ERROR)
             return null();
-        size_t offset = tokenStream.currentToken().pos.begin;
+        size_t offset = tokenStream.offsetOfToken(tokenStream.currentToken());
         tokenStream.ungetToken();
 
         return functionDef(NullPtr(), start, offset, Normal, Arrow);
@@ -5138,7 +5139,7 @@ Parser<ParseHandler>::unaryExpr()
       case TOK_INC:
       case TOK_DEC:
       {
-        uint32_t begin = tokenStream.currentToken().pos.begin;
+        TokenPtr begin = tokenStream.currentToken().pos.begin;
         pn2 = memberExpr(true);
         if (!pn2)
             return null();
@@ -5153,7 +5154,7 @@ Parser<ParseHandler>::unaryExpr()
 
       case TOK_DELETE:
       {
-        uint32_t begin = tokenStream.currentToken().pos.begin;
+        TokenPtr begin = tokenStream.currentToken().pos.begin;
         pn2 = unaryExpr();
         if (!pn2)
             return null();
@@ -6067,7 +6068,7 @@ Parser<ParseHandler>::memberExpr(bool allowCallSyntax)
                 return null();
             if (tt == TOK_NAME) {
                 PropertyName *field = tokenStream.currentToken().name();
-                uint32_t end = tokenStream.currentToken().pos.end;
+                TokenPtr end = tokenStream.currentToken().pos.end;
                 nextMember = handler.newPropertyAccess(lhs, field, end);
                 if (!nextMember)
                     return null();
@@ -6091,7 +6092,7 @@ Parser<ParseHandler>::memberExpr(bool allowCallSyntax)
 
             PropertyName *name = foldPropertyByValue(propExpr);
 
-            uint32_t end = tokenStream.currentToken().pos.end;
+            TokenPtr end = tokenStream.currentToken().pos.end;
             if (name)
                 nextMember = handler.newPropertyAccess(lhs, name, end);
             else
@@ -6694,7 +6695,7 @@ typename ParseHandler::Node
 Parser<ParseHandler>::parenExpr(bool *genexp)
 {
     JS_ASSERT(tokenStream.currentToken().type == TOK_LP);
-    uint32_t begin = tokenStream.currentToken().pos.begin;
+    TokenPtr begin = tokenStream.currentToken().pos.begin;
 
     if (genexp)
         *genexp = false;
diff --git a/js/src/frontend/SyntaxParseHandler.h b/js/src/frontend/SyntaxParseHandler.h
index 085dec56800f..e01250028bed 100644
--- a/js/src/frontend/SyntaxParseHandler.h
+++ b/js/src/frontend/SyntaxParseHandler.h
@@ -51,7 +51,7 @@ class SyntaxParseHandler
         return NodeString;
     }
     Node newNumber(double value, DecimalPoint decimalPoint = NoDecimal) { return NodeGeneric; }
-    Node newNumber(Token tok) { return NodeGeneric; }
+    Node newNumber(const Token &tok) { return NodeGeneric; }
     Node newBooleanLiteral(bool cond, const TokenPos &pos) { return NodeGeneric; }
     Node newThisLiteral(const TokenPos &pos) { return NodeGeneric; }
     Node newNullLiteral(const TokenPos &pos) { return NodeGeneric; }
@@ -82,15 +82,15 @@ class SyntaxParseHandler
         return NodeGeneric;
     }
 
-    Node newBreak(PropertyName *label, uint32_t begin, uint32_t end) {
+    Node newBreak(PropertyName *label, const TokenPtr &begin, const TokenPtr &end) {
         return NodeGeneric;
     }
-    Node newContinue(PropertyName *label, uint32_t begin, uint32_t end) {
+    Node newContinue(PropertyName *label, const TokenPtr &begin, const TokenPtr &end) {
         return NodeGeneric;
     }
     Node newDebuggerStatement(const TokenPos &pos) { return NodeGeneric; }
-    Node newPropertyAccess(Node pn, PropertyName *name, uint32_t end) { return NodeLValue; }
-    Node newPropertyByValue(Node pn, Node kid, uint32_t end) { return NodeLValue; }
+    Node newPropertyAccess(Node pn, PropertyName *name, const TokenPtr &end) { return NodeLValue; }
+    Node newPropertyByValue(Node pn, Node kid, const TokenPtr &end) { return NodeLValue; }
 
     bool addCatchBlock(Node catchList, Node letBlock,
                        Node catchName, Node catchGuard, Node catchBody) { return true; }
@@ -114,10 +114,10 @@ class SyntaxParseHandler
     bool finishInitializerAssignment(Node pn, Node init, JSOp op) { return true; }
 
     void setBeginPosition(Node pn, Node oth) {}
-    void setBeginPosition(Node pn, uint32_t begin) {}
+    void setBeginPosition(Node pn, const TokenPtr &begin) {}
 
     void setEndPosition(Node pn, Node oth) {}
-    void setEndPosition(Node pn, uint32_t end) {}
+    void setEndPosition(Node pn, const TokenPtr &end) {}
 
     TokenPos getPosition(Node pn) {
         return tokenStream.currentToken().pos;
diff --git a/js/src/frontend/TokenStream.cpp b/js/src/frontend/TokenStream.cpp
index 6488ea8f646b..0e14a004d7bb 100644
--- a/js/src/frontend/TokenStream.cpp
+++ b/js/src/frontend/TokenStream.cpp
@@ -107,122 +107,6 @@ frontend::IsIdentifier(JSLinearString *str)
     return true;
 }
 
-const uint32_t TokenStream::SourceCoords::MAX_PTR;
-
-TokenStream::SourceCoords::SourceCoords(JSContext *cx, uint32_t ln)
-  : lineStartOffsets_(cx), initialLineNum_(ln), lastLineIndex_(0)
-{
-    // The first line begins at buffer offset 0.  MAX_PTR is the sentinel.  The
-    // appends cannot fail because |lineStartOffsets_| has statically-allocated
-    // elements.
-    JS_ASSERT(lineStartOffsets_.capacity() >= 2);
-    (void)lineStartOffsets_.reserve(2);
-    lineStartOffsets_.infallibleAppend(0);
-    lineStartOffsets_.infallibleAppend(MAX_PTR);
-}
-
-JS_ALWAYS_INLINE void
-TokenStream::SourceCoords::add(uint32_t lineNum, uint32_t lineStartOffset)
-{
-    uint32_t lineIndex = lineNumToIndex(lineNum);
-    uint32_t sentinelIndex = lineStartOffsets_.length() - 1;
-
-    JS_ASSERT(lineStartOffsets_[0] == 0 && lineStartOffsets_[sentinelIndex] == MAX_PTR);
-
-    if (lineIndex == sentinelIndex) {
-        // We haven't seen this newline before.  Update lineStartOffsets_.
-        // We ignore any failures due to OOM -- because we always have a
-        // sentinel node, it'll just be like the newline wasn't present.  I.e.
-        // the line numbers will be wrong, but the code won't crash or anything
-        // like that.
-        lineStartOffsets_[lineIndex] = lineStartOffset;
-        (void)lineStartOffsets_.append(MAX_PTR);
-
-    } else {
-        // We have seen this newline before (and ungot it).  Do nothing (other
-        // than checking it hasn't mysteriously changed).
-        JS_ASSERT(lineStartOffsets_[lineIndex] == lineStartOffset);
-    }
-}
-
-JS_ALWAYS_INLINE uint32_t
-TokenStream::SourceCoords::lineIndexOf(uint32_t offset) const
-{
-    uint32_t iMin, iMax, iMid;
-
-    if (lineStartOffsets_[lastLineIndex_] <= offset) {
-        // If we reach here, offset is on a line the same as or higher than
-        // last time.  Check first for the +0, +1, +2 cases, because they
-        // typically cover 85--98% of cases.
-        if (offset < lineStartOffsets_[lastLineIndex_ + 1])
-            return lastLineIndex_;      // lineIndex is same as last time
-
-        // If we reach here, there must be at least one more entry (plus the
-        // sentinel).  Try it.
-        lastLineIndex_++;
-        if (offset < lineStartOffsets_[lastLineIndex_ + 1])
-            return lastLineIndex_;      // lineIndex is one higher than last time
-
-        // The same logic applies here.
-        lastLineIndex_++;
-        if (offset < lineStartOffsets_[lastLineIndex_ + 1]) {
-            return lastLineIndex_;      // lineIndex is two higher than last time
-        }
-
-        // No luck.  Oh well, we have a better-than-default starting point for
-        // the binary search.
-        iMin = lastLineIndex_ + 1;
-        JS_ASSERT(iMin < lineStartOffsets_.length() - 1);   // -1 due to the sentinel
-
-    } else {
-        iMin = 0;
-    }
-
-    // This is a binary search with deferred detection of equality, which was
-    // marginally faster in this case than a standard binary search.
-    // The -2 is because |lineStartOffsets_.length() - 1| is the sentinel, and we
-    // want one before that.
-    iMax = lineStartOffsets_.length() - 2;
-    while (iMax > iMin) {
-        iMid = (iMin + iMax) / 2;
-        if (offset >= lineStartOffsets_[iMid + 1])
-            iMin = iMid + 1;    // offset is above lineStartOffsets_[iMid]
-        else
-            iMax = iMid;        // offset is below or within lineStartOffsets_[iMid]
-    }
-    JS_ASSERT(iMax == iMin);
-    JS_ASSERT(lineStartOffsets_[iMin] <= offset && offset < lineStartOffsets_[iMin + 1]);
-    lastLineIndex_ = iMin;
-    return iMin;
-}
-
-uint32_t
-TokenStream::SourceCoords::lineNum(uint32_t offset) const
-{
-    uint32_t lineIndex = lineIndexOf(offset);
-    return lineIndexToNum(lineIndex);
-}
-
-uint32_t
-TokenStream::SourceCoords::columnIndex(uint32_t offset) const
-{
-    uint32_t lineIndex = lineIndexOf(offset);
-    uint32_t lineStartOffset = lineStartOffsets_[lineIndex];
-    JS_ASSERT(offset >= lineStartOffset);
-    return offset - lineStartOffset;
-}
-
-void
-TokenStream::SourceCoords::lineNumAndColumnIndex(uint32_t offset, uint32_t *lineNum,
-                                                 uint32_t *columnIndex) const
-{
-    uint32_t lineIndex = lineIndexOf(offset);
-    *lineNum = lineIndexToNum(lineIndex);
-    uint32_t lineStartOffset = lineStartOffsets_[lineIndex];
-    JS_ASSERT(offset >= lineStartOffset);
-    *columnIndex = offset - lineStartOffset;
-}
-
 #ifdef _MSC_VER
 #pragma warning(push)
 #pragma warning(disable:4351)
@@ -231,8 +115,7 @@ TokenStream::SourceCoords::lineNumAndColumnIndex(uint32_t offset, uint32_t *line
 /* Initialize members that aren't initialized in |init|. */
 TokenStream::TokenStream(JSContext *cx, const CompileOptions &options,
                          const jschar *base, size_t length, StrictModeGetter *smg)
-  : srcCoords(cx, options.lineno),
-    tokens(),
+  : tokens(),
     cursor(),
     lookahead(),
     lineno(options.lineno),
@@ -265,9 +148,9 @@ TokenStream::TokenStream(JSContext *cx, const CompileOptions &options,
     /*
      * This table holds all the token kinds that satisfy these properties:
      * - A single char long.
-     * - Cannot be a prefix of any longer token (e.g. '+' is excluded because
+     * - Cannot be a prefix of any longer token (eg. '+' is excluded because
      *   '+=' is a valid token).
-     * - Doesn't need tp->t_op set (e.g. this excludes '~').
+     * - Doesn't need tp->t_op set (eg. this excludes '~').
      *
      * The few token kinds satisfying these properties cover roughly 35--45%
      * of the tokens seen in practice.
@@ -304,6 +187,18 @@ TokenStream::TokenStream(JSContext *cx, const CompileOptions &options,
     maybeStrSpecial[unsigned(LINE_SEPARATOR & 0xff)] = true;
     maybeStrSpecial[unsigned(PARA_SEPARATOR & 0xff)] = true;
     maybeStrSpecial[unsigned(EOF & 0xff)] = true;
+
+    /*
+     * Set |ln| as the beginning line number of the ungot "current token", so
+     * that js::Parser::statements (and potentially other such methods, in the
+     * future) can create parse nodes with good source coordinates before they
+     * explicitly get any tokens.
+     *
+     * Switching the parser/lexer so we always get the next token ahead of the
+     * parser needing it (the so-called "pump-priming" model) might be a better
+     * way to address the dependency from statements on the current token.
+     */
+    tokens[0].pos.begin.lineno = tokens[0].pos.end.lineno = options.lineno;
 }
 
 #ifdef _MSC_VER
@@ -333,7 +228,6 @@ TokenStream::updateLineInfoForEOL()
     prevLinebase = linebase;
     linebase = userbuf.addressOfNextRawChar();
     lineno++;
-    srcCoords.add(lineno, linebase - userbuf.base());
 }
 
 JS_ALWAYS_INLINE void
@@ -523,7 +417,7 @@ TokenStream::positionAfterLastFunctionKeyword(Position &pos)
 }
 
 bool
-TokenStream::reportStrictModeErrorNumberVA(uint32_t offset, bool strictMode, unsigned errorNumber,
+TokenStream::reportStrictModeErrorNumberVA(const TokenPos &pos, bool strictMode, unsigned errorNumber,
                                            va_list args)
 {
     /* In strict mode code, this is an error, not merely a warning. */
@@ -534,8 +428,8 @@ TokenStream::reportStrictModeErrorNumberVA(uint32_t offset, bool strictMode, uns
         flags |= JSREPORT_WARNING;
     else
         return true;
-
-    return reportCompileErrorNumberVA(offset, flags, errorNumber, args);
+ 
+    return reportCompileErrorNumberVA(pos, flags, errorNumber, args);
 }
 
 void
@@ -591,7 +485,7 @@ CompileError::~CompileError()
 }
 
 bool
-TokenStream::reportCompileErrorNumberVA(uint32_t offset, unsigned flags, unsigned errorNumber,
+TokenStream::reportCompileErrorNumberVA(const TokenPos &pos, unsigned flags, unsigned errorNumber,
                                         va_list args)
 {
     bool warning = JSREPORT_IS_WARNING(flags);
@@ -607,7 +501,7 @@ TokenStream::reportCompileErrorNumberVA(uint32_t offset, unsigned flags, unsigne
     err.report.errorNumber = errorNumber;
     err.report.filename = filename;
     err.report.originPrincipals = originPrincipals;
-    err.report.lineno = srcCoords.lineNum(offset);
+    err.report.lineno = pos.begin.lineno;
 
     err.argumentsType = (flags & JSREPORT_UC) ? ArgumentsAreUnicode : ArgumentsAreASCII;
 
@@ -624,11 +518,11 @@ TokenStream::reportCompileErrorNumberVA(uint32_t offset, unsigned flags, unsigne
      * T's (starting) line for context.
      *
      * So we don't even try, leaving report.linebuf and friends zeroed.  This
-     * means that any error involving a multi-line token (e.g. an unterminated
+     * means that any error involving a multi-line token (eg. an unterminated
      * multi-line string literal) won't have a context printed.
      */
     if (err.report.lineno == lineno) {
-        const jschar *tokenStart = userbuf.base() + offset;
+        const jschar *tokptr = linebase + pos.begin.index;
 
         // We show only a portion (a "window") of the line around the erroneous
         // token -- the first char in the token, plus |windowRadius| chars
@@ -638,13 +532,14 @@ TokenStream::reportCompileErrorNumberVA(uint32_t offset, unsigned flags, unsigne
         static const size_t windowRadius = 60;
 
         // Truncate at the front if necessary.
-        const jschar *windowBase = (linebase + windowRadius < tokenStart)
-                                 ? tokenStart - windowRadius
+        const jschar *windowBase = (linebase + windowRadius < tokptr)
+                                 ? tokptr - windowRadius
                                  : linebase;
-        uint32_t windowOffset = tokenStart - windowBase;
+        size_t nTrunc = windowBase - linebase;
+        uint32_t windowIndex = pos.begin.index - nTrunc;
 
         // Find EOL, or truncate at the back if necessary.
-        const jschar *windowLimit = userbuf.findEOLMax(tokenStart, windowRadius);
+        const jschar *windowLimit = userbuf.findEOLMax(tokptr, windowRadius);
         size_t windowLength = windowLimit - windowBase;
         JS_ASSERT(windowLength <= windowRadius * 2);
 
@@ -663,8 +558,10 @@ TokenStream::reportCompileErrorNumberVA(uint32_t offset, unsigned flags, unsigne
         if (!err.report.linebuf)
             return false;
 
-        err.report.tokenptr = err.report.linebuf + windowOffset;
-        err.report.uctokenptr = err.report.uclinebuf + windowOffset;
+        // The lineno check above means we should only see single-line tokens here.
+        JS_ASSERT(pos.begin.lineno == pos.end.lineno);
+        err.report.tokenptr = err.report.linebuf + windowIndex;
+        err.report.uctokenptr = err.report.uclinebuf + windowIndex;
     }
 
     err.throwError();
@@ -677,8 +574,7 @@ TokenStream::reportStrictModeError(unsigned errorNumber, ...)
 {
     va_list args;
     va_start(args, errorNumber);
-    bool result = reportStrictModeErrorNumberVA(currentToken().pos.begin, strictMode(),
-                                                errorNumber, args);
+    bool result = reportStrictModeErrorNumberVA(currentToken().pos, strictMode(), errorNumber, args);
     va_end(args);
     return result;
 }
@@ -688,8 +584,7 @@ TokenStream::reportError(unsigned errorNumber, ...)
 {
     va_list args;
     va_start(args, errorNumber);
-    bool result = reportCompileErrorNumberVA(currentToken().pos.begin, JSREPORT_ERROR, errorNumber,
-                                             args);
+    bool result = reportCompileErrorNumberVA(currentToken().pos, JSREPORT_ERROR, errorNumber, args);
     va_end(args);
     return result;
 }
@@ -699,27 +594,26 @@ TokenStream::reportWarning(unsigned errorNumber, ...)
 {
     va_list args;
     va_start(args, errorNumber);
-    bool result = reportCompileErrorNumberVA(currentToken().pos.begin, JSREPORT_WARNING,
-                                             errorNumber, args);
+    bool result = reportCompileErrorNumberVA(currentToken().pos, JSREPORT_WARNING, errorNumber, args);
     va_end(args);
     return result;
 }
 
 bool
-TokenStream::reportStrictWarningErrorNumberVA(uint32_t offset, unsigned errorNumber, va_list args)
+TokenStream::reportStrictWarningErrorNumberVA(const TokenPos &pos, unsigned errorNumber, va_list args)
 {
     if (!cx->hasStrictOption())
         return true;
 
-    return reportCompileErrorNumberVA(offset, JSREPORT_STRICT|JSREPORT_WARNING, errorNumber, args);
+    return reportCompileErrorNumberVA(pos, JSREPORT_STRICT | JSREPORT_WARNING, errorNumber, args);
 }
 
 void
-TokenStream::reportAsmJSError(uint32_t offset, unsigned errorNumber, ...)
+TokenStream::reportAsmJSError(ParseNode *pn, unsigned errorNumber, ...)
 {
     va_list args;
     va_start(args, errorNumber);
-    reportCompileErrorNumberVA(offset, JSREPORT_WARNING, errorNumber, args);
+    reportCompileErrorNumberVA(pn->pn_pos, JSREPORT_WARNING, errorNumber, args);
     va_end(args);
 }
 
@@ -767,6 +661,31 @@ TokenStream::matchUnicodeEscapeIdent(int32_t *cp)
     return false;
 }
 
+size_t
+TokenStream::endOffset(const Token &tok)
+{
+    uint32_t lineno = tok.pos.begin.lineno;
+    JS_ASSERT(lineno <= tok.pos.end.lineno);
+    const jschar *end;
+    if (lineno < tok.pos.end.lineno) {
+        TokenBuf buf(cx, tok.ptr, userbuf.addressOfNextRawChar() - userbuf.base());
+        for (; lineno < tok.pos.end.lineno; lineno++) {
+            jschar c;
+            do {
+                JS_ASSERT(buf.hasRawChars());
+                c = buf.getRawChar();
+            } while (!TokenBuf::isRawEOLChar(c));
+            if (c == '\r' && buf.hasRawChars())
+                buf.matchRawChar('\n');
+        }
+        end = buf.addressOfNextRawChar() + tok.pos.end.index;
+    } else {
+        end = tok.ptr + (tok.pos.end.index - tok.pos.begin.index);
+    }
+    JS_ASSERT(end <= userbuf.addressOfNextRawChar());
+    return end - userbuf.base();
+}
+
 /*
  * Helper function which returns true if the first length(q) characters in p are
  * the same as the characters in q.
@@ -837,11 +756,9 @@ TokenStream::newToken(ptrdiff_t adjust)
 {
     cursor = (cursor + 1) & ntokensMask;
     Token *tp = &tokens[cursor];
-    tp->pos.begin = userbuf.addressOfNextRawChar() + adjust - userbuf.base();
-
-    // NOTE: tp->pos.end is not set until the very end of getTokenInternal().
-    MOZ_MAKE_MEM_UNDEFINED(&tp->pos.end, sizeof(tp->pos.end));
-
+    tp->ptr = userbuf.addressOfNextRawChar() + adjust;
+    tp->pos.begin.index = tp->ptr - linebase;
+    tp->pos.begin.lineno = tp->pos.end.lineno = lineno;
     return tp;
 }
 
@@ -862,9 +779,14 @@ IsTokenSane(Token *tp)
     if (tp->type < TOK_ERROR || tp->type >= TOK_LIMIT || tp->type == TOK_EOL)
         return false;
 
-    if (tp->pos.end < tp->pos.begin)
-        return false;
-
+    if (tp->pos.begin.lineno == tp->pos.end.lineno) {
+        if (tp->pos.begin.index > tp->pos.end.index)
+            return false;
+    } else {
+        /* Only string tokens can be multi-line. */
+        if (tp->type != TOK_STRING)
+            return false;
+    }
     return true;
 }
 #endif
@@ -1266,6 +1188,7 @@ TokenStream::getTokenInternal()
         JSAtom *atom = atomize(cx, tokenbuf);
         if (!atom)
             goto error;
+        tp->pos.end.lineno = lineno;
         tp->setAtom(JSOP_STRING, atom);
         tt = TOK_STRING;
         goto out;
@@ -1614,7 +1537,7 @@ TokenStream::getTokenInternal()
             c = peekChar();
             if (JS7_ISLET(c)) {
                 char buf[2] = { '\0', '\0' };
-                tp->pos.begin += length + 1;
+                tp->pos.begin.index += length + 1;
                 buf[0] = char(c);
                 reportError(JSMSG_BAD_REGEXP_FLAG, buf);
                 (void) getChar();
@@ -1663,14 +1586,20 @@ TokenStream::getTokenInternal()
 
   out:
     flags |= TSF_DIRTYLINE;
-    tp->pos.end = userbuf.addressOfNextRawChar() - userbuf.base();
+    tp->pos.end.index = userbuf.addressOfNextRawChar() - linebase;
     tp->type = tt;
     JS_ASSERT(IsTokenSane(tp));
     return tt;
 
   error:
+    /*
+     * For erroneous multi-line tokens we won't have changed end.lineno (it'll
+     * still be equal to begin.lineno) so we revert end.index to be equal to
+     * begin.index + 1 (as if it's a 1-char token) to avoid having inconsistent
+     * begin/end positions.  end.index isn't used in error messages anyway.
+     */
     flags |= TSF_DIRTYLINE;
-    tp->pos.end = userbuf.addressOfNextRawChar() - userbuf.base();
+    tp->pos.end.index = tp->pos.begin.index + 1;
     tp->type = TOK_ERROR;
     JS_ASSERT(IsTokenSane(tp));
     onError();
diff --git a/js/src/frontend/TokenStream.h b/js/src/frontend/TokenStream.h
index 8690048154c0..4b804cb6b6be 100644
--- a/js/src/frontend/TokenStream.h
+++ b/js/src/frontend/TokenStream.h
@@ -178,11 +178,42 @@ TokenKindIsDecl(TokenKind tt)
 #endif
 }
 
-struct TokenPos {
-    uint32_t          begin;          /* offset of the token's first char */
-    uint32_t          end;            /* offset of 1 past the token's last char */
+struct TokenPtr {
+    uint32_t            index;          /* index of char in physical line */
+    uint32_t            lineno;         /* physical line number */
 
-    static TokenPos make(uint32_t begin, uint32_t end) {
+    bool operator==(const TokenPtr& bptr) const {
+        return index == bptr.index && lineno == bptr.lineno;
+    }
+
+    bool operator!=(const TokenPtr& bptr) const {
+        return index != bptr.index || lineno != bptr.lineno;
+    }
+
+    bool operator <(const TokenPtr& bptr) const {
+        return lineno < bptr.lineno ||
+               (lineno == bptr.lineno && index < bptr.index);
+    }
+
+    bool operator <=(const TokenPtr& bptr) const {
+        return lineno < bptr.lineno ||
+               (lineno == bptr.lineno && index <= bptr.index);
+    }
+
+    bool operator >(const TokenPtr& bptr) const {
+        return !(*this <= bptr);
+    }
+
+    bool operator >=(const TokenPtr& bptr) const {
+        return !(*this < bptr);
+    }
+};
+
+struct TokenPos {
+    TokenPtr          begin;          /* first character and line of token */
+    TokenPtr          end;            /* index 1 past last char, last line */
+
+    static TokenPos make(const TokenPtr &begin, const TokenPtr &end) {
         JS_ASSERT(begin <= end);
         TokenPos pos = {begin, end};
         return pos;
@@ -231,6 +262,7 @@ enum DecimalPoint { NoDecimal = false, HasDecimal = true };
 struct Token {
     TokenKind           type;           /* char value or above enumerator */
     TokenPos            pos;            /* token position in file */
+    const jschar        *ptr;           /* beginning of token in line buffer */
     union {
         struct {                        /* name or string literal */
             JSOp        op;             /* operator, for minimal parser */
@@ -402,7 +434,7 @@ class TokenStream
 
     /* Accessors. */
     JSContext *getContext() const { return cx; }
-    bool onCurrentLine(const TokenPos &pos) const { return srcCoords.isOnThisLine(pos.end, lineno); }
+    bool onCurrentLine(const TokenPos &pos) const { return lineno == pos.end.lineno; }
     const Token &currentToken() const { return tokens[cursor]; }
     bool isCurrentTokenType(TokenKind type) const {
         return currentToken().type == type;
@@ -411,6 +443,9 @@ class TokenStream
         TokenKind type = currentToken().type;
         return type == type1 || type == type2;
     }
+    size_t offsetOfToken(const Token &tok) const {
+        return tok.ptr - userbuf.base();
+    }
     const CharBuffer &getTokenbuf() const { return tokenbuf; }
     const char *getFilename() const { return filename; }
     unsigned getLineno() const { return lineno; }
@@ -448,15 +483,15 @@ class TokenStream
     // General-purpose error reporters.  You should avoid calling these
     // directly, and instead use the more succinct alternatives (e.g.
     // reportError()) in TokenStream, Parser, and BytecodeEmitter.
-    bool reportCompileErrorNumberVA(uint32_t offset, unsigned flags, unsigned errorNumber,
+    bool reportCompileErrorNumberVA(const TokenPos &pos, unsigned flags, unsigned errorNumber,
                                     va_list args);
-    bool reportStrictModeErrorNumberVA(uint32_t offset, bool strictMode, unsigned errorNumber,
+    bool reportStrictModeErrorNumberVA(const TokenPos &pos, bool strictMode, unsigned errorNumber,
                                        va_list args);
-    bool reportStrictWarningErrorNumberVA(uint32_t offset, unsigned errorNumber,
+    bool reportStrictWarningErrorNumberVA(const TokenPos &pos, unsigned errorNumber,
                                           va_list args);
 
     // asm.js reporter
-    void reportAsmJSError(uint32_t offset, unsigned errorNumber, ...);
+    void reportAsmJSError(ParseNode *pn, unsigned errorNumber, ...);
 
   private:
     // These are private because they should only be called by the tokenizer
@@ -525,8 +560,10 @@ class TokenStream
     }
 
     TokenKind peekToken() {
-        if (lookahead != 0)
+        if (lookahead != 0) {
+            JS_ASSERT(lookahead <= maxLookahead);
             return tokens[(cursor + 1) & ntokensMask].type;
+        }
         TokenKind tt = getTokenInternal();
         ungetToken();
         return tt;
@@ -538,12 +575,17 @@ class TokenStream
     }
 
     TokenKind peekTokenSameLine(unsigned withFlags = 0) {
+        if (lookahead != 0) {
+            JS_ASSERT(lookahead <= maxLookahead);
+            Token &nextToken = tokens[(cursor + 1) & ntokensMask];
+            return currentToken().pos.end.lineno == nextToken.pos.begin.lineno
+                   ? nextToken.type
+                   : TOK_EOL;
+        }
+
         if (!onCurrentLine(currentToken().pos))
             return TOK_EOL;
 
-        if (lookahead != 0)
-            return tokens[(cursor + 1) & ntokensMask].type;
-
         /*
          * This is the only place TOK_EOL is produced.  No token with TOK_EOL
          * is created, just a TOK_EOL TokenKind is returned.
@@ -593,6 +635,11 @@ class TokenStream
     void seek(const Position &pos);
     void positionAfterLastFunctionKeyword(Position &pos);
 
+    /*
+     * Return the offset into the source buffer of the end of the token.
+     */
+    size_t endOffset(const Token &tok);
+
     size_t positionToOffset(const Position &pos) const {
         return pos.buf - userbuf.base();
     }
@@ -627,75 +674,6 @@ class TokenStream
      */
     bool checkForKeyword(const jschar *s, size_t length, TokenKind *ttp, JSOp *topp);
 
-    // This class maps a userbuf offset (which is 0-indexed) to a line number
-    // (which is 1-indexed) and a column index (which is 0-indexed).
-    class SourceCoords
-    {
-        // For a given buffer holding source code, |lineStartOffsets_| has one
-        // element per line of source code, plus one sentinel element.  Each
-        // non-sentinel element holds the buffer offset for the start of the
-        // corresponding line of source code.  For this example script:
-        //
-        // 1  // xyz            [line starts at offset 0]
-        // 2  var x;            [line starts at offset 7]
-        // 3                    [line starts at offset 14]
-        // 4  var y;            [line starts at offset 15]
-        //
-        // |lineStartOffsets_| is:
-        //
-        //   [0, 7, 14, 15, MAX_PTR]
-        //
-        // To convert a "line number" to a "line index" (i.e. an index into
-        // |lineStartOffsets_|), subtract |initialLineNum_|.  E.g. line 3's
-        // line index is (3 - initialLineNum_), which is 2.  Therefore
-        // lineStartOffsets_[2] holds the buffer offset for the start of line 3,
-        // which is 14.  (Note that |initialLineNum_| is often 1, but not
-        // always.)
-        //
-        // The first element is always 0, and the last element is always the
-        // MAX_PTR sentinel.
-        //
-        // offset-to-line/column lookups are O(log n) in the worst case (binary
-        // search), but in practice they're heavily clustered and we do better
-        // than that by using the previous lookup's result (lastLineIndex_) as
-        // a starting point.
-        //
-        // Checking if an offset lies within a particular line number
-        // (isOnThisLine()) is O(1).
-        //
-        Vector<uint32_t, 128> lineStartOffsets_;
-        uint32_t            initialLineNum_;
-
-        // This is mutable because it's modified on every search, but that fact
-        // isn't visible outside this class.
-        mutable uint32_t    lastLineIndex_;
-
-        uint32_t lineIndexOf(uint32_t offset) const;
-
-        static const uint32_t MAX_PTR = UINT32_MAX;
-
-        uint32_t lineIndexToNum(uint32_t lineIndex) const { return lineIndex + initialLineNum_; }
-        uint32_t lineNumToIndex(uint32_t lineNum)   const { return lineNum   - initialLineNum_; }
-
-      public:
-        SourceCoords(JSContext *cx, uint32_t ln);
-
-        void add(uint32_t lineNum, uint32_t lineStartOffset);
-
-        bool isOnThisLine(uint32_t offset, uint32_t lineNum) const {
-            uint32_t lineIndex = lineNumToIndex(lineNum);
-            JS_ASSERT(lineIndex + 1 < lineStartOffsets_.length());  // +1 due to sentinel
-            return lineStartOffsets_[lineIndex] <= offset &&
-                   offset < lineStartOffsets_[lineIndex + 1];
-        }
-
-        uint32_t lineNum(uint32_t offset) const;
-        uint32_t columnIndex(uint32_t offset) const;
-        void lineNumAndColumnIndex(uint32_t offset, uint32_t *lineNum, uint32_t *columnIndex) const;
-    };
-
-    SourceCoords srcCoords;
-
   private:
     /*
      * This is the low-level interface to the JS source code buffer.  It just
diff --git a/js/src/ion/AsmJS.cpp b/js/src/ion/AsmJS.cpp
index fe79689f8798..d27003655a16 100644
--- a/js/src/ion/AsmJS.cpp
+++ b/js/src/ion/AsmJS.cpp
@@ -1092,8 +1092,7 @@ class ModuleCompiler
 
     ~ModuleCompiler() {
         if (errorString_)
-            tokenStream_.reportAsmJSError(errorNode_->pn_pos.begin, JSMSG_USE_ASM_TYPE_FAIL,
-                                          errorString_);
+            tokenStream_.reportAsmJSError(errorNode_, JSMSG_USE_ASM_TYPE_FAIL, errorString_);
 
         // Avoid spurious Label assertions on compilation failure.
         if (!stackOverflowLabel_.bound())
diff --git a/js/src/jsfun.cpp b/js/src/jsfun.cpp
index 81c6f5114f1d..665ce6516078 100644
--- a/js/src/jsfun.cpp
+++ b/js/src/jsfun.cpp
@@ -601,7 +601,7 @@ FindBody(JSContext *cx, HandleFunction fun, StableCharPtr chars, size_t length,
         return false;
     bool braced = tt == TOK_LC;
     JS_ASSERT_IF(fun->isExprClosure(), !braced);
-    *bodyStart = ts.currentToken().pos.begin;
+    *bodyStart = ts.offsetOfToken(ts.currentToken());
     if (braced)
         *bodyStart += 1;
     StableCharPtr end(chars.get() + length, chars.get(), length);
diff --git a/js/src/jsprvtd.h b/js/src/jsprvtd.h
index a98120be4350..e0e8c39d9f6c 100644
--- a/js/src/jsprvtd.h
+++ b/js/src/jsprvtd.h
@@ -158,6 +158,7 @@ class FunctionBox;
 class ObjectBox;
 struct Token;
 struct TokenPos;
+struct TokenPtr;
 class TokenStream;
 class ParseMapPool;
 struct ParseNode;
diff --git a/js/src/jsreflect.cpp b/js/src/jsreflect.cpp
index 1718e96b9ce8..3a30e114f415 100644
--- a/js/src/jsreflect.cpp
+++ b/js/src/jsreflect.cpp
@@ -134,7 +134,6 @@ typedef AutoValueVector NodeVector;
 class NodeBuilder
 {
     JSContext   *cx;
-    TokenStream *tokenStream;
     bool        saveLoc;               /* save source location information?     */
     char const  *src;                  /* source filename or null               */
     RootedValue srcval;                /* source filename JS value or null      */
@@ -145,7 +144,7 @@ class NodeBuilder
 
   public:
     NodeBuilder(JSContext *c, bool l, char const *s)
-        : cx(c), tokenStream(NULL), saveLoc(l), src(s), srcval(c),
+        : cx(c), saveLoc(l), src(s), srcval(c),
           callbacksRoots(c, callbacks, AST_LIMIT), userv(c), undefinedVal(c, UndefinedValue())
     {
         MakeRangeGCSafe(callbacks, mozilla::ArrayLength(callbacks));
@@ -197,10 +196,6 @@ class NodeBuilder
         return true;
     }
 
-    void setTokenStream(TokenStream *ts) {
-        tokenStream = ts;
-    }
-
   private:
     bool callback(HandleValue fun, TokenPos *pos, MutableHandleValue dst) {
         if (saveLoc) {
@@ -681,20 +676,15 @@ NodeBuilder::newNodeLoc(TokenPos *pos, MutableHandleValue dst)
 
     dst.setObject(*loc);
 
-    uint32_t startLineNum, startColumnIndex;
-    uint32_t endLineNum, endColumnIndex;
-    tokenStream->srcCoords.lineNumAndColumnIndex(pos->begin, &startLineNum, &startColumnIndex);
-    tokenStream->srcCoords.lineNumAndColumnIndex(pos->end, &endLineNum, &endColumnIndex);
-
     if (!newObject(&to))
         return false;
     val.setObject(*to);
     if (!setProperty(loc, "start", val))
         return false;
-    val.setNumber(startLineNum);
+    val.setNumber(pos->begin.lineno);
     if (!setProperty(to, "line", val))
         return false;
-    val.setNumber(startColumnIndex);
+    val.setNumber(pos->begin.index);
     if (!setProperty(to, "column", val))
         return false;
 
@@ -703,10 +693,10 @@ NodeBuilder::newNodeLoc(TokenPos *pos, MutableHandleValue dst)
     val.setObject(*to);
     if (!setProperty(loc, "end", val))
         return false;
-    val.setNumber(endLineNum);
+    val.setNumber(pos->end.lineno);
     if (!setProperty(to, "line", val))
         return false;
-    val.setNumber(endColumnIndex);
+    val.setNumber(pos->end.index);
     if (!setProperty(to, "column", val))
         return false;
 
@@ -1563,7 +1553,6 @@ class ASTSerializer
 
     void setParser(Parser<FullParseHandler> *p) {
         parser = p;
-        builder.setTokenStream(&p->tokenStream);
     }
 
     bool program(ParseNode *pn, MutableHandleValue dst);
@@ -1730,7 +1719,7 @@ ASTSerializer::blockStatement(ParseNode *pn, MutableHandleValue dst)
 bool
 ASTSerializer::program(ParseNode *pn, MutableHandleValue dst)
 {
-    JS_ASSERT(parser->tokenStream.srcCoords.lineNum(pn->pn_pos.begin) == lineno);
+    JS_ASSERT(pn->pn_pos.begin.lineno == lineno);
 
     NodeVector stmts(cx);
     return statements(pn, stmts) &&