Bug 1467336 - Convert TokenStreamSpecific::getDirective act on code units, and remove TokenStreamSpecific::peekChar. r=arai

--HG--
extra : rebase_source : d553724fa06e0a9e0810140f46ae3b88f7270143

js/src/frontend/TokenStream.cpp

@@ -1225,31 +1225,50 @@ TokenStreamSpecific<CharT, AnyCharsAccess>::getDirective(bool isMultiline,
     tokenbuf.clear();
 
     do {
-        int32_t c;
-        if (!peekChar(&c))
-            return false;
-
-        if (c == EOF || unicode::IsSpaceOrBOM2(c))
+        int32_t unit = peekCodeUnit();
+        if (unit == EOF)
             break;
 
-        consumeKnownChar(c);
+        if (MOZ_LIKELY(isAsciiCodePoint(unit))) {
+            if (unicode::IsSpaceOrBOM2(unit))
+                break;
+
+            consumeKnownCodeUnit(unit);
+
+            // Debugging directives can occur in both single- and multi-line
+            // comments. If we're currently inside a multi-line comment, we
+            // also must recognize multi-line comment terminators.
+            if (isMultiline && unit == '*' && peekCodeUnit() == '/') {
+                ungetCodeUnit('*');
+                break;
+            }
+
+            if (!tokenbuf.append(unit))
+                return false;
+
+            continue;
+        }
+
+        int32_t codePoint;
+        if (!getCodePoint(&codePoint))
+            return false;
+
+        if (unicode::IsSpaceOrBOM2(codePoint)) {
+            ungetCodePointIgnoreEOL(codePoint);
+
+            if (codePoint == unicode::LINE_SEPARATOR || codePoint == unicode::PARA_SEPARATOR)
+                anyCharsAccess().undoInternalUpdateLineInfoForEOL();
 
-        // Debugging directives can occur in both single- and multi-line
-        // comments. If we're currently inside a multi-line comment, we also
-        // need to recognize multi-line comment terminators.
-        if (isMultiline && c == '*' && matchCodeUnit('/')) {
-            ungetCodeUnit('/');
-            ungetCodeUnit('*');
             break;
         }
 
-        if (!tokenbuf.append(c))
+        if (!appendCodePointToTokenbuf(codePoint))
             return false;
     } while (true);
 
     if (tokenbuf.empty()) {
-        // The directive's URL was missing, but this is not quite an
-        // exception that we should stop and drop everything for.
+        // The directive's URL was missing, but comments can contain anything,
+        // so it isn't an error.
         return true;
     }
 

js/src/frontend/TokenStream.h

@@ -1848,12 +1848,6 @@ class MOZ_STACK_CLASS TokenStreamSpecific
     MOZ_MUST_USE bool getDisplayURL(bool isMultiline, bool shouldWarnDeprecated);
     MOZ_MUST_USE bool getSourceMappingURL(bool isMultiline, bool shouldWarnDeprecated);
 
-    void consumeKnownChar(int32_t expect) {
-        int32_t c;
-        MOZ_ALWAYS_TRUE(getChar(&c));
-        MOZ_ASSERT(c == expect);
-    }
-
     void consumeKnownCharIgnoreEOL(int32_t expect) {
 #ifdef DEBUG
         auto c =
@@ -1861,13 +1855,6 @@ class MOZ_STACK_CLASS TokenStreamSpecific
             getCodeUnit();
         MOZ_ASSERT(c == expect);
     }
-
-    MOZ_MUST_USE bool peekChar(int32_t* c) {
-        if (!getChar(c))
-            return false;
-        ungetChar(*c);
-        return true;
-    }
 };
 
 // It's preferable to define this in TokenStream.cpp, but its template-ness