gecko-dev/layout/style/nsCSSScanner.cpp

/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */


/* tokenization of CSS style sheets */

#include "nsCSSScanner.h"
#include "nsStyleUtil.h"
#include "nsISupportsImpl.h"
#include "mozilla/ArrayUtils.h"
#include "mozilla/css/ErrorReporter.h"
#include "mozilla/Likely.h"
#include <algorithm>

/* Character class tables and related helper functions. */

static const uint8_t IS_HEX_DIGIT  = 0x01;
static const uint8_t IS_IDSTART    = 0x02;
static const uint8_t IS_IDCHAR     = 0x04;
static const uint8_t IS_URL_CHAR   = 0x08;
static const uint8_t IS_HSPACE     = 0x10;
static const uint8_t IS_VSPACE     = 0x20;
static const uint8_t IS_SPACE      = IS_HSPACE|IS_VSPACE;
static const uint8_t IS_STRING     = 0x40;

#define H    IS_HSPACE
#define V    IS_VSPACE
#define I    IS_IDCHAR
#define J    IS_IDSTART
#define U    IS_URL_CHAR
#define S    IS_STRING
#define X    IS_HEX_DIGIT

#define SH    S|H
#define SU    S|U
#define SUI   S|U|I
#define SUIJ  S|U|I|J
#define SUIX  S|U|I|X
#define SUIJX S|U|I|J|X

static const uint8_t gLexTable[] = {
// 00    01    02    03    04    05    06    07
    0,    S,    S,    S,    S,    S,    S,    S,
// 08   TAB    LF    0B    FF    CR    0E    0F
    S,   SH,    V,    S,    V,    V,    S,    S,
// 10    11    12    13    14    15    16    17
    S,    S,    S,    S,    S,    S,    S,    S,
// 18    19    1A    1B    1C    1D    1E    1F
    S,    S,    S,    S,    S,    S,    S,    S,
//SPC     !     "     #     $     %     &     '
   SH,   SU,    0,   SU,   SU,   SU,   SU,    0,
//  (     )     *     +     ,     -     .     /
    S,    S,   SU,   SU,   SU,  SUI,   SU,   SU,
//  0     1     2     3     4     5     6     7
 SUIX, SUIX, SUIX, SUIX, SUIX, SUIX, SUIX, SUIX,
//  8     9     :     ;     <     =     >     ?
 SUIX, SUIX,   SU,   SU,   SU,   SU,   SU,   SU,
//  @     A     B     C     D     E     F     G
   SU,SUIJX,SUIJX,SUIJX,SUIJX,SUIJX,SUIJX, SUIJ,
//  H     I     J     K     L     M     N     O
 SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ,
//  P     Q     R     S     T     U     V     W
 SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ,
//  X     Y     Z     [     \     ]     ^     _
 SUIJ, SUIJ, SUIJ,   SU,    J,   SU,   SU, SUIJ,
//  `     a     b     c     d     e     f     g
   SU,SUIJX,SUIJX,SUIJX,SUIJX,SUIJX,SUIJX, SUIJ,
//  h     i     j     k     l     m     n     o
 SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ,
//  p     q     r     s     t     u     v     w
 SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ,
//  x     y     z     {     |     }     ~    7F
 SUIJ, SUIJ, SUIJ,   SU,   SU,   SU,   SU,    S,
};

static_assert(MOZ_ARRAY_LENGTH(gLexTable) == 128,
              "gLexTable expected to cover all 128 ASCII characters");

#undef I
#undef J
#undef U
#undef S
#undef X
#undef SH
#undef SU
#undef SUI
#undef SUIJ
#undef SUIX
#undef SUIJX

/**
 * True if 'ch' is in character class 'cls', which should be one of
 * the constants above or some combination of them.  All characters
 * above U+007F are considered to be in 'cls'.  EOF is never in 'cls'.
 */
static inline bool
IsOpenCharClass(int32_t ch, uint8_t cls) {
  return ch >= 0 && (ch >= 128 || (gLexTable[ch] & cls) != 0);
}

/**
 * True if 'ch' is in character class 'cls', which should be one of
 * the constants above or some combination of them.  No characters
 * above U+007F are considered to be in 'cls'. EOF is never in 'cls'.
 */
static inline bool
IsClosedCharClass(int32_t ch, uint8_t cls) {
  return uint32_t(ch) < 128 && (gLexTable[ch] & cls) != 0;
}

/**
 * True if 'ch' is CSS whitespace, i.e. any of the ASCII characters
 * TAB, LF, FF, CR, or SPC.
 */
static inline bool
IsWhitespace(int32_t ch) {
  return IsClosedCharClass(ch, IS_SPACE);
}

/**
 * True if 'ch' is horizontal whitespace, i.e. TAB or SPC.
 */
static inline bool
IsHorzSpace(int32_t ch) {
  return IsClosedCharClass(ch, IS_HSPACE);
}

/**
 * True if 'ch' is vertical whitespace, i.e. LF, FF, or CR.  Vertical
 * whitespace requires special handling when consumed, see AdvanceLine.
 */
static inline bool
IsVertSpace(int32_t ch) {
  return IsClosedCharClass(ch, IS_VSPACE);
}

/**
 * True if 'ch' is a character that can appear in the middle of an identifier.
 * This includes U+0000 since it is handled as U+FFFD, but for purposes of
 * GatherText it should not be included in IsOpenCharClass.
 */
static inline bool
IsIdentChar(int32_t ch) {
  return IsOpenCharClass(ch, IS_IDCHAR) || ch == 0;
}

/**
 * True if 'ch' is a character that by itself begins an identifier.
 * This includes U+0000 since it is handled as U+FFFD, but for purposes of
 * GatherText it should not be included in IsOpenCharClass.
 * (This is a subset of IsIdentChar.)
 */
static inline bool
IsIdentStart(int32_t ch) {
  return IsOpenCharClass(ch, IS_IDSTART) || ch == 0;
}

/**
 * True if the two-character sequence aFirstChar+aSecondChar begins an
 * identifier.
 */
static inline bool
StartsIdent(int32_t aFirstChar, int32_t aSecondChar)
{
  return IsIdentStart(aFirstChar) ||
    (aFirstChar == '-' && IsIdentStart(aSecondChar));
}

/**
 * True if 'ch' is a decimal digit.
 */
static inline bool
IsDigit(int32_t ch) {
  return (ch >= '0') && (ch <= '9');
}

/**
 * True if 'ch' is a hexadecimal digit.
 */
static inline bool
IsHexDigit(int32_t ch) {
  return IsClosedCharClass(ch, IS_HEX_DIGIT);
}

/**
 * Assuming that 'ch' is a decimal digit, return its numeric value.
 */
static inline uint32_t
DecimalDigitValue(int32_t ch)
{
  return ch - '0';
}

/**
 * Assuming that 'ch' is a hexadecimal digit, return its numeric value.
 */
static inline uint32_t
HexDigitValue(int32_t ch)
{
  if (IsDigit(ch)) {
    return DecimalDigitValue(ch);
  } else {
    // Note: c&7 just keeps the low three bits which causes
    // upper and lower case alphabetics to both yield their
    // "relative to 10" value for computing the hex value.
    return (ch & 0x7) + 9;
  }
}

/**
 * If 'ch' can be the first character of a two-character match operator
 * token, return the token type code for that token, otherwise return
 * eCSSToken_Symbol to indicate that it can't.
 */
static inline nsCSSTokenType
MatchOperatorType(int32_t ch)
{
  switch (ch) {
  case '~': return eCSSToken_Includes;
  case '|': return eCSSToken_Dashmatch;
  case '^': return eCSSToken_Beginsmatch;
  case '$': return eCSSToken_Endsmatch;
  case '*': return eCSSToken_Containsmatch;
  default:  return eCSSToken_Symbol;
  }
}

/* Out-of-line nsCSSToken methods. */

/**
 * Append the textual representation of |this| to |aBuffer|.
 */
void
nsCSSToken::AppendToString(nsString& aBuffer) const
{
  switch (mType) {
    case eCSSToken_Ident:
      nsStyleUtil::AppendEscapedCSSIdent(mIdent, aBuffer);
      break;

    case eCSSToken_AtKeyword:
      aBuffer.Append('@');
      nsStyleUtil::AppendEscapedCSSIdent(mIdent, aBuffer);
      break;

    case eCSSToken_ID:
    case eCSSToken_Hash:
      aBuffer.Append('#');
      nsStyleUtil::AppendEscapedCSSIdent(mIdent, aBuffer);
      break;

    case eCSSToken_Function:
      nsStyleUtil::AppendEscapedCSSIdent(mIdent, aBuffer);
      aBuffer.Append('(');
      break;

    case eCSSToken_URL:
    case eCSSToken_Bad_URL:
      aBuffer.AppendLiteral("url(");
      if (mSymbol != char16_t(0)) {
        nsStyleUtil::AppendEscapedCSSString(mIdent, aBuffer, mSymbol);
      } else {
        aBuffer.Append(mIdent);
      }
      if (mType == eCSSToken_URL) {
        aBuffer.Append(char16_t(')'));
      }
      break;

    case eCSSToken_Number:
      if (mIntegerValid) {
        aBuffer.AppendInt(mInteger, 10);
      } else {
        aBuffer.AppendFloat(mNumber);
      }
      break;

    case eCSSToken_Percentage:
      aBuffer.AppendFloat(mNumber * 100.0f);
      aBuffer.Append(char16_t('%'));
      break;

    case eCSSToken_Dimension:
      if (mIntegerValid) {
        aBuffer.AppendInt(mInteger, 10);
      } else {
        aBuffer.AppendFloat(mNumber);
      }
      nsStyleUtil::AppendEscapedCSSIdent(mIdent, aBuffer);
      break;

    case eCSSToken_Bad_String:
      nsStyleUtil::AppendEscapedCSSString(mIdent, aBuffer, mSymbol);
      // remove the trailing quote character
      aBuffer.Truncate(aBuffer.Length() - 1);
      break;

    case eCSSToken_String:
      nsStyleUtil::AppendEscapedCSSString(mIdent, aBuffer, mSymbol);
      break;

    case eCSSToken_Symbol:
      aBuffer.Append(mSymbol);
      break;

    case eCSSToken_Whitespace:
      aBuffer.Append(' ');
      break;

    case eCSSToken_HTMLComment:
    case eCSSToken_URange:
      aBuffer.Append(mIdent);
      break;

    case eCSSToken_Includes:
      aBuffer.AppendLiteral("~=");
      break;
    case eCSSToken_Dashmatch:
      aBuffer.AppendLiteral("|=");
      break;
    case eCSSToken_Beginsmatch:
      aBuffer.AppendLiteral("^=");
      break;
    case eCSSToken_Endsmatch:
      aBuffer.AppendLiteral("$=");
      break;
    case eCSSToken_Containsmatch:
      aBuffer.AppendLiteral("*=");
      break;

    default:
      NS_ERROR("invalid token type");
      break;
  }
}

/* nsCSSScanner methods. */

nsCSSScanner::nsCSSScanner(const nsAString& aBuffer, uint32_t aLineNumber)
  : mBuffer(aBuffer.BeginReading())
  , mOffset(0)
  , mCount(aBuffer.Length())
  , mLineNumber(aLineNumber)
  , mLineOffset(0)
  , mTokenLineNumber(aLineNumber)
  , mTokenLineOffset(0)
  , mTokenOffset(0)
  , mRecordStartOffset(0)
  , mEOFCharacters(eEOFCharacters_None)
  , mReporter(nullptr)
  , mSVGMode(false)
  , mRecording(false)
  , mSeenBadToken(false)
  , mSeenVariableReference(false)
{
  MOZ_COUNT_CTOR(nsCSSScanner);
}

nsCSSScanner::~nsCSSScanner()
{
  MOZ_COUNT_DTOR(nsCSSScanner);
}

void
nsCSSScanner::StartRecording()
{
  MOZ_ASSERT(!mRecording, "already started recording");
  mRecording = true;
  mRecordStartOffset = mOffset;
}

void
nsCSSScanner::StopRecording()
{
  MOZ_ASSERT(mRecording, "haven't started recording");
  mRecording = false;
}

void
nsCSSScanner::StopRecording(nsString& aBuffer)
{
  MOZ_ASSERT(mRecording, "haven't started recording");
  mRecording = false;
  aBuffer.Append(mBuffer + mRecordStartOffset,
                 mOffset - mRecordStartOffset);
}

uint32_t
nsCSSScanner::RecordingLength() const
{
  MOZ_ASSERT(mRecording, "haven't started recording");
  return mOffset - mRecordStartOffset;
}

#ifdef DEBUG
bool
nsCSSScanner::IsRecording() const
{
  return mRecording;
}
#endif

nsDependentSubstring
nsCSSScanner::GetCurrentLine() const
{
  uint32_t end = mTokenOffset;
  while (end < mCount && !IsVertSpace(mBuffer[end])) {
    end++;
  }
  return nsDependentSubstring(mBuffer + mTokenLineOffset,
                              mBuffer + end);
}

/**
 * Return the raw UTF-16 code unit at position |mOffset + n| within
 * the read buffer.  If that is beyond the end of the buffer, returns
 * -1 to indicate end of input.
 */
inline int32_t
nsCSSScanner::Peek(uint32_t n)
{
  if (mOffset + n >= mCount) {
    return -1;
  }
  return mBuffer[mOffset + n];
}

/**
 * Advance |mOffset| over |n| code units.  Advance(0) is a no-op.
 * If |n| is greater than the distance to end of input, will silently
 * stop at the end.  May not be used to advance over a line boundary;
 * AdvanceLine() must be used instead.
 */
inline void
nsCSSScanner::Advance(uint32_t n)
{
#ifdef DEBUG
  while (mOffset < mCount && n > 0) {
    MOZ_ASSERT(!IsVertSpace(mBuffer[mOffset]),
               "may not Advance() over a line boundary");
    mOffset++;
    n--;
  }
#else
  if (mOffset + n >= mCount || mOffset + n < mOffset)
    mOffset = mCount;
  else
    mOffset += n;
#endif
}

/**
 * Advance |mOffset| over a line boundary.
 */
void
nsCSSScanner::AdvanceLine()
{
  MOZ_ASSERT(IsVertSpace(mBuffer[mOffset]),
             "may not AdvanceLine() over a horizontal character");
  // Advance over \r\n as a unit.
  if (mBuffer[mOffset]   == '\r' && mOffset + 1 < mCount &&
      mBuffer[mOffset+1] == '\n')
    mOffset += 2;
  else
    mOffset += 1;
  // 0 is a magical line number meaning that we don't know (i.e., script)
  if (mLineNumber != 0)
    mLineNumber++;
  mLineOffset = mOffset;
}

/**
 * Back up |mOffset| over |n| code units.  Backup(0) is a no-op.
 * If |n| is greater than the distance to beginning of input, will
 * silently stop at the beginning.  May not be used to back up over a
 * line boundary.
 */
void
nsCSSScanner::Backup(uint32_t n)
{
#ifdef DEBUG
  while (mOffset > 0 && n > 0) {
    MOZ_ASSERT(!IsVertSpace(mBuffer[mOffset-1]),
               "may not Backup() over a line boundary");
    mOffset--;
    n--;
  }
#else
  if (mOffset < n)
    mOffset = 0;
  else
    mOffset -= n;
#endif
}

void
nsCSSScanner::SavePosition(nsCSSScannerPosition& aState)
{
  aState.mOffset = mOffset;
  aState.mLineNumber = mLineNumber;
  aState.mLineOffset = mLineOffset;
  aState.mTokenLineNumber = mTokenLineNumber;
  aState.mTokenLineOffset = mTokenLineOffset;
  aState.mTokenOffset = mTokenOffset;
  aState.mInitialized = true;
}

void
nsCSSScanner::RestoreSavedPosition(const nsCSSScannerPosition& aState)
{
  MOZ_ASSERT(aState.mInitialized, "have not saved state");
  if (aState.mInitialized) {
    mOffset = aState.mOffset;
    mLineNumber = aState.mLineNumber;
    mLineOffset = aState.mLineOffset;
    mTokenLineNumber = aState.mTokenLineNumber;
    mTokenLineOffset = aState.mTokenLineOffset;
    mTokenOffset = aState.mTokenOffset;
  }
}

/**
 * Skip over a sequence of whitespace characters (vertical or
 * horizontal) starting at the current read position.
 */
void
nsCSSScanner::SkipWhitespace()
{
  for (;;) {
    int32_t ch = Peek();
    if (!IsWhitespace(ch)) { // EOF counts as non-whitespace
      break;
    }
    if (IsVertSpace(ch)) {
      AdvanceLine();
    } else {
      Advance();
    }
  }
}

/**
 * Skip over one CSS comment starting at the current read position.
 */
void
nsCSSScanner::SkipComment()
{
  MOZ_ASSERT(Peek() == '/' && Peek(1) == '*', "should not have been called");
  Advance(2);
  for (;;) {
    int32_t ch = Peek();
    if (ch < 0) {
      if (mReporter)
        mReporter->ReportUnexpectedEOF("PECommentEOF");
      SetEOFCharacters(eEOFCharacters_Asterisk | eEOFCharacters_Slash);
      return;
    }
    if (ch == '*') {
      Advance();
      ch = Peek();
      if (ch < 0) {
        if (mReporter)
          mReporter->ReportUnexpectedEOF("PECommentEOF");
        SetEOFCharacters(eEOFCharacters_Slash);
        return;
      }
      if (ch == '/') {
        Advance();
        return;
      }
    } else if (IsVertSpace(ch)) {
      AdvanceLine();
    } else {
      Advance();
    }
  }
}

/**
 * If there is a valid escape sequence starting at the current read
 * position, consume it, decode it, append the result to |aOutput|,
 * and return true.  Otherwise, consume nothing, leave |aOutput|
 * unmodified, and return false.  If |aInString| is true, accept the
 * additional form of escape sequence allowed within string-like tokens.
 */
bool
nsCSSScanner::GatherEscape(nsString& aOutput, bool aInString)
{
  MOZ_ASSERT(Peek() == '\\', "should not have been called");
  int32_t ch = Peek(1);
  if (ch < 0) {
    // If we are in a string (or a url() containing a string), we want to drop
    // the backslash on the floor.  Otherwise, we want to treat it as a U+FFFD
    // character.
    Advance();
    if (aInString) {
      SetEOFCharacters(eEOFCharacters_DropBackslash);
    } else {
      aOutput.Append(UCS2_REPLACEMENT_CHAR);
      SetEOFCharacters(eEOFCharacters_ReplacementChar);
    }
    return true;
  }
  if (IsVertSpace(ch)) {
    if (aInString) {
      // In strings (and in url() containing a string), escaped
      // newlines are completely removed, to allow splitting over
      // multiple lines.
      Advance();
      AdvanceLine();
      return true;
    }
    // Outside of strings, backslash followed by a newline is not an escape.
    return false;
  }

  if (!IsHexDigit(ch)) {
    // "Any character (except a hexadecimal digit, linefeed, carriage
    // return, or form feed) can be escaped with a backslash to remove
    // its special meaning." -- CSS2.1 section 4.1.3
    Advance(2);
    if (ch == 0) {
      aOutput.Append(UCS2_REPLACEMENT_CHAR);
    } else {
      aOutput.Append(ch);
    }
    return true;
  }

  // "[at most six hexadecimal digits following a backslash] stand
  // for the ISO 10646 character with that number, which must not be
  // zero. (It is undefined in CSS 2.1 what happens if a style sheet
  // does contain a character with Unicode codepoint zero.)"
  //   -- CSS2.1 section 4.1.3

  // At this point we know we have \ followed by at least one
  // hexadecimal digit, therefore the escape sequence is valid and we
  // can go ahead and consume the backslash.
  Advance();
  uint32_t val = 0;
  int i = 0;
  do {
    val = val * 16 + HexDigitValue(ch);
    i++;
    Advance();
    ch = Peek();
  } while (i < 6 && IsHexDigit(ch));

  // "Interpret the hex digits as a hexadecimal number. If this number is zero,
  // or is greater than the maximum allowed codepoint, return U+FFFD
  // REPLACEMENT CHARACTER" -- CSS Syntax Level 3
  if (MOZ_UNLIKELY(val == 0)) {
    aOutput.Append(UCS2_REPLACEMENT_CHAR);
  } else {
    AppendUCS4ToUTF16(ENSURE_VALID_CHAR(val), aOutput);
  }

  // Consume exactly one whitespace character after a
  // hexadecimal escape sequence.
  if (IsVertSpace(ch)) {
    AdvanceLine();
  } else if (IsHorzSpace(ch)) {
    Advance();
  }
  return true;
}

/**
 * Consume a run of "text" beginning with the current read position,
 * consisting of characters in the class |aClass| (which must be a
 * suitable argument to IsOpenCharClass) plus escape sequences.
 * Append the text to |aText|, after decoding escape sequences.
 *
 * Returns true if at least one character was appended to |aText|,
 * false otherwise.
 */
bool
nsCSSScanner::GatherText(uint8_t aClass, nsString& aText)
{
  // This is all of the character classes currently used with
  // GatherText.  If you have a need to use this function with a
  // different class, go ahead and add it.
  MOZ_ASSERT(aClass == IS_STRING ||
             aClass == IS_IDCHAR ||
             aClass == IS_URL_CHAR,
             "possibly-inappropriate character class");

  uint32_t start = mOffset;
  bool inString = aClass == IS_STRING;

  for (;;) {
    // Consume runs of unescaped characters in one go.
    uint32_t n = mOffset;
    while (n < mCount && IsOpenCharClass(mBuffer[n], aClass)) {
      n++;
    }
    if (n > mOffset) {
      aText.Append(&mBuffer[mOffset], n - mOffset);
      mOffset = n;
    }
    if (n == mCount) {
      break;
    }

    int32_t ch = Peek();
    MOZ_ASSERT(!IsOpenCharClass(ch, aClass),
               "should not have exited the inner loop");
    if (ch == 0) {
      Advance();
      aText.Append(UCS2_REPLACEMENT_CHAR);
      continue;
    }

    if (ch != '\\') {
      break;
    }
    if (!GatherEscape(aText, inString)) {
      break;
    }
  }

  return mOffset > start;
}

/**
 * Scan an Ident token.  This also handles Function and URL tokens,
 * both of which begin indistinguishably from an identifier.  It can
 * produce a Symbol token when an apparent identifier actually led
 * into an invalid escape sequence.
 */
bool
nsCSSScanner::ScanIdent(nsCSSToken& aToken)
{
  if (MOZ_UNLIKELY(!GatherText(IS_IDCHAR, aToken.mIdent))) {
    MOZ_ASSERT(Peek() == '\\',
               "unexpected IsIdentStart character that did not begin an ident");
    aToken.mSymbol = Peek();
    Advance();
    return true;
  }

  if (MOZ_LIKELY(Peek() != '(')) {
    aToken.mType = eCSSToken_Ident;
    return true;
  }

  Advance();
  aToken.mType = eCSSToken_Function;
  if (aToken.mIdent.LowerCaseEqualsLiteral("url")) {
    NextURL(aToken);
  } else if (aToken.mIdent.LowerCaseEqualsLiteral("var")) {
    mSeenVariableReference = true;
  }
  return true;
}

/**
 * Scan an AtKeyword token.  Also handles production of Symbol when
 * an '@' is not followed by an identifier.
 */
bool
nsCSSScanner::ScanAtKeyword(nsCSSToken& aToken)
{
  MOZ_ASSERT(Peek() == '@', "should not have been called");

  // Fall back for when '@' isn't followed by an identifier.
  aToken.mSymbol = '@';
  Advance();

  int32_t ch = Peek();
  if (StartsIdent(ch, Peek(1))) {
    if (GatherText(IS_IDCHAR, aToken.mIdent)) {
       aToken.mType = eCSSToken_AtKeyword;
     }
  }
  return true;
}

/**
 * Scan a Hash token.  Handles the distinction between eCSSToken_ID
 * and eCSSToken_Hash, and handles production of Symbol when a '#'
 * is not followed by identifier characters.
 */
bool
nsCSSScanner::ScanHash(nsCSSToken& aToken)
{
  MOZ_ASSERT(Peek() == '#', "should not have been called");

  // Fall back for when '#' isn't followed by identifier characters.
  aToken.mSymbol = '#';
  Advance();

  int32_t ch = Peek();
  if (IsIdentChar(ch) || ch == '\\') {
    nsCSSTokenType type =
      StartsIdent(ch, Peek(1)) ? eCSSToken_ID : eCSSToken_Hash;
    aToken.mIdent.SetLength(0);
    if (GatherText(IS_IDCHAR, aToken.mIdent)) {
      aToken.mType = type;
    }
  }

  return true;
}

/**
 * Scan a Number, Percentage, or Dimension token (all of which begin
 * like a Number).  Can produce a Symbol when a '.' is not followed by
 * digits, or when '+' or '-' are not followed by either a digit or a
 * '.' and then a digit.  Can also produce a HTMLComment when it
 * encounters '-->'.
 */
bool
nsCSSScanner::ScanNumber(nsCSSToken& aToken)
{
  int32_t c = Peek();
#ifdef DEBUG
  {
    int32_t c2 = Peek(1);
    int32_t c3 = Peek(2);
    MOZ_ASSERT(IsDigit(c) ||
               (IsDigit(c2) && (c == '.' || c == '+' || c == '-')) ||
               (IsDigit(c3) && (c == '+' || c == '-') && c2 == '.'),
               "should not have been called");
  }
#endif

  // Sign of the mantissa (-1 or 1).
  int32_t sign = c == '-' ? -1 : 1;
  // Absolute value of the integer part of the mantissa.  This is a double so
  // we don't run into overflow issues for consumers that only care about our
  // floating-point value while still being able to express the full int32_t
  // range for consumers who want integers.
  double intPart = 0;
  // Fractional part of the mantissa.  This is a double so that when we convert
  // to float at the end we'll end up rounding to nearest float instead of
  // truncating down (as we would if fracPart were a float and we just
  // effectively lost the last several digits).
  double fracPart = 0;
  // Absolute value of the power of 10 that we should multiply by (only
  // relevant for numbers in scientific notation).  Has to be a signed integer,
  // because multiplication of signed by unsigned converts the unsigned to
  // signed, so if we plan to actually multiply by expSign...
  int32_t exponent = 0;
  // Sign of the exponent.
  int32_t expSign = 1;

  aToken.mHasSign = (c == '+' || c == '-');
  if (aToken.mHasSign) {
    Advance();
    c = Peek();
  }

  bool gotDot = (c == '.');

  if (!gotDot) {
    // Scan the integer part of the mantissa.
    MOZ_ASSERT(IsDigit(c), "should have been excluded by logic above");
    do {
      intPart = 10*intPart + DecimalDigitValue(c);
      Advance();
      c = Peek();
    } while (IsDigit(c));

    gotDot = (c == '.') && IsDigit(Peek(1));
  }

  if (gotDot) {
    // Scan the fractional part of the mantissa.
    Advance();
    c = Peek();
    MOZ_ASSERT(IsDigit(c), "should have been excluded by logic above");
    // Power of ten by which we need to divide our next digit
    double divisor = 10;
    do {
      fracPart += DecimalDigitValue(c) / divisor;
      divisor *= 10;
      Advance();
      c = Peek();
    } while (IsDigit(c));
  }

  bool gotE = false;
  if (c == 'e' || c == 'E') {
    int32_t expSignChar = Peek(1);
    int32_t nextChar = Peek(2);
    if (IsDigit(expSignChar) ||
        ((expSignChar == '-' || expSignChar == '+') && IsDigit(nextChar))) {
      gotE = true;
      if (expSignChar == '-') {
        expSign = -1;
      }
      Advance(); // consumes the E
      if (expSignChar == '-' || expSignChar == '+') {
        Advance();
        c = nextChar;
      } else {
        c = expSignChar;
      }
      MOZ_ASSERT(IsDigit(c), "should have been excluded by logic above");
      do {
        exponent = 10*exponent + DecimalDigitValue(c);
        Advance();
        c = Peek();
      } while (IsDigit(c));
    }
  }

  nsCSSTokenType type = eCSSToken_Number;

  // Set mIntegerValid for all cases (except %, below) because we need
  // it for the "2n" in :nth-child(2n).
  aToken.mIntegerValid = false;

  // Time to reassemble our number.
  // Do all the math in double precision so it's truncated only once.
  double value = sign * (intPart + fracPart);
  if (gotE) {
    // Explicitly cast expSign*exponent to double to avoid issues with
    // overloaded pow() on Windows.
    value *= pow(10.0, double(expSign * exponent));
  } else if (!gotDot) {
    // Clamp values outside of integer range.
    if (sign > 0) {
      aToken.mInteger = int32_t(std::min(intPart, double(INT32_MAX)));
    } else {
      aToken.mInteger = int32_t(std::max(-intPart, double(INT32_MIN)));
    }
    aToken.mIntegerValid = true;
  }

  nsString& ident = aToken.mIdent;

  // Check for Dimension and Percentage tokens.
  if (c >= 0) {
    if (StartsIdent(c, Peek(1))) {
      if (GatherText(IS_IDCHAR, ident)) {
        type = eCSSToken_Dimension;
      }
    } else if (c == '%') {
      Advance();
      type = eCSSToken_Percentage;
      value = value / 100.0f;
      aToken.mIntegerValid = false;
    }
  }
  aToken.mNumber = value;
  aToken.mType = type;
  return true;
}

/**
 * Scan a string constant ('foo' or "foo").  Will always produce
 * either a String or a Bad_String token; the latter occurs when the
 * close quote is missing.  Always returns true (for convenience in Next()).
 */
bool
nsCSSScanner::ScanString(nsCSSToken& aToken)
{
  int32_t aStop = Peek();
  MOZ_ASSERT(aStop == '"' || aStop == '\'', "should not have been called");
  aToken.mType = eCSSToken_String;
  aToken.mSymbol = char16_t(aStop); // Remember how it's quoted.
  Advance();

  for (;;) {
    GatherText(IS_STRING, aToken.mIdent);

    int32_t ch = Peek();
    if (ch == -1) {
      AddEOFCharacters(aStop == '"' ? eEOFCharacters_DoubleQuote :
                                      eEOFCharacters_SingleQuote);
      break; // EOF ends a string token with no error.
    }
    if (ch == aStop) {
      Advance();
      break;
    }
    // Both " and ' are excluded from IS_STRING.
    if (ch == '"' || ch == '\'') {
      aToken.mIdent.Append(ch);
      Advance();
      continue;
    }

    mSeenBadToken = true;
    aToken.mType = eCSSToken_Bad_String;
    if (mReporter)
      mReporter->ReportUnexpected("SEUnterminatedString", aToken);
    break;
  }
  return true;
}

/**
 * Scan a unicode-range token.  These match the regular expression
 *
 *     u\+[0-9a-f?]{1,6}(-[0-9a-f]{1,6})?
 *
 * However, some such tokens are "invalid".  There are three valid forms:
 *
 *     u+[0-9a-f]{x}              1 <= x <= 6
 *     u+[0-9a-f]{x}\?{y}         1 <= x+y <= 6
 *     u+[0-9a-f]{x}-[0-9a-f]{y}  1 <= x <= 6, 1 <= y <= 6
 *
 * All unicode-range tokens have their text recorded in mIdent; valid ones
 * are also decoded into mInteger and mInteger2, and mIntegerValid is set.
 * Note that this does not validate the numeric range, only the syntactic
 * form.
 */
bool
nsCSSScanner::ScanURange(nsCSSToken& aResult)
{
  int32_t intro1 = Peek();
  int32_t intro2 = Peek(1);
  int32_t ch = Peek(2);

  MOZ_ASSERT((intro1 == 'u' || intro1 == 'U') &&
             intro2 == '+' &&
             (IsHexDigit(ch) || ch == '?'),
             "should not have been called");

  aResult.mIdent.Append(intro1);
  aResult.mIdent.Append(intro2);
  Advance(2);

  bool valid = true;
  bool haveQues = false;
  uint32_t low = 0;
  uint32_t high = 0;
  int i = 0;

  do {
    aResult.mIdent.Append(ch);
    if (IsHexDigit(ch)) {
      if (haveQues) {
        valid = false; // All question marks should be at the end.
      }
      low = low*16 + HexDigitValue(ch);
      high = high*16 + HexDigitValue(ch);
    } else {
      haveQues = true;
      low = low*16 + 0x0;
      high = high*16 + 0xF;
    }

    i++;
    Advance();
    ch = Peek();
  } while (i < 6 && (IsHexDigit(ch) || ch == '?'));

  if (ch == '-' && IsHexDigit(Peek(1))) {
    if (haveQues) {
      valid = false;
    }

    aResult.mIdent.Append(ch);
    Advance();
    ch = Peek();
    high = 0;
    i = 0;
    do {
      aResult.mIdent.Append(ch);
      high = high*16 + HexDigitValue(ch);

      i++;
      Advance();
      ch = Peek();
    } while (i < 6 && IsHexDigit(ch));
  }

  aResult.mInteger = low;
  aResult.mInteger2 = high;
  aResult.mIntegerValid = valid;
  aResult.mType = eCSSToken_URange;
  return true;
}

#ifdef DEBUG
/* static */ void
nsCSSScanner::AssertEOFCharactersValid(uint32_t c)
{
  MOZ_ASSERT(c == eEOFCharacters_None ||
             c == eEOFCharacters_ReplacementChar ||
             c == eEOFCharacters_Slash ||
             c == (eEOFCharacters_Asterisk |
                   eEOFCharacters_Slash) ||
             c == eEOFCharacters_DoubleQuote ||
             c == eEOFCharacters_SingleQuote ||
             c == (eEOFCharacters_DropBackslash |
                   eEOFCharacters_DoubleQuote) ||
             c == (eEOFCharacters_DropBackslash |
                   eEOFCharacters_SingleQuote) ||
             c == eEOFCharacters_CloseParen ||
             c == (eEOFCharacters_ReplacementChar |
                   eEOFCharacters_CloseParen) ||
             c == (eEOFCharacters_DoubleQuote |
                   eEOFCharacters_CloseParen) ||
             c == (eEOFCharacters_SingleQuote |
                   eEOFCharacters_CloseParen) ||
             c == (eEOFCharacters_DropBackslash |
                   eEOFCharacters_DoubleQuote |
                   eEOFCharacters_CloseParen) ||
             c == (eEOFCharacters_DropBackslash |
                   eEOFCharacters_SingleQuote |
                   eEOFCharacters_CloseParen),
             "invalid EOFCharacters value");
}
#endif

void
nsCSSScanner::SetEOFCharacters(uint32_t aEOFCharacters)
{
  mEOFCharacters = EOFCharacters(aEOFCharacters);
}

void
nsCSSScanner::AddEOFCharacters(uint32_t aEOFCharacters)
{
  mEOFCharacters = EOFCharacters(mEOFCharacters | aEOFCharacters);
}

static const char16_t kImpliedEOFCharacters[] = {
  UCS2_REPLACEMENT_CHAR, '*', '/', '"', '\'', ')', 0
};

/* static */ void
nsCSSScanner::AppendImpliedEOFCharacters(EOFCharacters aEOFCharacters,
                                         nsAString& aResult)
{
  // First, ignore eEOFCharacters_DropBackslash.
  uint32_t c = aEOFCharacters >> 1;

  // All of the remaining EOFCharacters bits represent appended characters,
  // and the bits are in the order that they need appending.
  for (const char16_t* p = kImpliedEOFCharacters; *p && c; p++, c >>= 1) {
    if (c & 1) {
      aResult.Append(*p);
    }
  }

  MOZ_ASSERT(c == 0, "too many bits in mEOFCharacters");
}

/**
 * Consume the part of an URL token after the initial 'url('.  Caller
 * is assumed to have consumed 'url(' already.  Will always produce
 * either an URL or a Bad_URL token.
 *
 * Exposed for use by nsCSSParser::ParseMozDocumentRule, which applies
 * the special lexical rules for URL tokens in a nonstandard context.
 */
void
nsCSSScanner::NextURL(nsCSSToken& aToken)
{
  SkipWhitespace();

  // aToken.mIdent may be "url" at this point; clear that out
  aToken.mIdent.Truncate();

  int32_t ch = Peek();
  // Do we have a string?
  if (ch == '"' || ch == '\'') {
    ScanString(aToken);
    if (MOZ_UNLIKELY(aToken.mType == eCSSToken_Bad_String)) {
      aToken.mType = eCSSToken_Bad_URL;
      return;
    }
    MOZ_ASSERT(aToken.mType == eCSSToken_String, "unexpected token type");

  } else {
    // Otherwise, this is the start of a non-quoted url (which may be empty).
    aToken.mSymbol = char16_t(0);
    GatherText(IS_URL_CHAR, aToken.mIdent);
  }

  // Consume trailing whitespace and then look for a close parenthesis.
  SkipWhitespace();
  ch = Peek();
  // ch can be less than zero indicating EOF
  if (MOZ_LIKELY(ch < 0 || ch == ')')) {
    Advance();
    aToken.mType = eCSSToken_URL;
    if (ch < 0) {
      AddEOFCharacters(eEOFCharacters_CloseParen);
    }
  } else {
    mSeenBadToken = true;
    aToken.mType = eCSSToken_Bad_URL;
  }
}

/**
 * Primary scanner entry point.  Consume one token and fill in
 * |aToken| accordingly.  Will skip over any number of comments first,
 * and will also skip over rather than return whitespace and comment
 * tokens, depending on the value of |aSkip|.
 *
 * Returns true if it successfully consumed a token, false if EOF has
 * been reached.  Will always advance the current read position by at
 * least one character unless called when already at EOF.
 */
bool
nsCSSScanner::Next(nsCSSToken& aToken, nsCSSScannerExclude aSkip)
{
  int32_t ch;

  // do this here so we don't have to do it in dozens of other places
  aToken.mIdent.Truncate();
  aToken.mType = eCSSToken_Symbol;

  for (;;) {
    // Consume any number of comments, and possibly also whitespace tokens,
    // in between other tokens.
    mTokenOffset = mOffset;
    mTokenLineOffset = mLineOffset;
    mTokenLineNumber = mLineNumber;

    ch = Peek();
    if (IsWhitespace(ch)) {
      SkipWhitespace();
      if (aSkip != eCSSScannerExclude_WhitespaceAndComments) {
        aToken.mType = eCSSToken_Whitespace;
        return true;
      }
      continue; // start again at the beginning
    }
    if (ch == '/' && !IsSVGMode() && Peek(1) == '*') {
      SkipComment();
      if (aSkip == eCSSScannerExclude_None) {
        aToken.mType = eCSSToken_Comment;
        return true;
      }
      continue; // start again at the beginning
    }
    break;
  }

  // EOF
  if (ch < 0) {
    return false;
  }

  // 'u' could be UNICODE-RANGE or an identifier-family token
  if (ch == 'u' || ch == 'U') {
    int32_t c2 = Peek(1);
    int32_t c3 = Peek(2);
    if (c2 == '+' && (IsHexDigit(c3) || c3 == '?')) {
      return ScanURange(aToken);
    }
    return ScanIdent(aToken);
  }

  // identifier family
  if (IsIdentStart(ch)) {
    return ScanIdent(aToken);
  }

  // number family
  if (IsDigit(ch)) {
    return ScanNumber(aToken);
  }

  if (ch == '.' && IsDigit(Peek(1))) {
    return ScanNumber(aToken);
  }

  if (ch == '+') {
    int32_t c2 = Peek(1);
    if (IsDigit(c2) || (c2 == '.' && IsDigit(Peek(2)))) {
      return ScanNumber(aToken);
    }
  }

  // '-' can start an identifier-family token, a number-family token,
  // or an HTML-comment
  if (ch == '-') {
    int32_t c2 = Peek(1);
    int32_t c3 = Peek(2);
    if (IsIdentStart(c2) || (c2 == '-' && c3 != '>')) {
      return ScanIdent(aToken);
    }
    if (IsDigit(c2) || (c2 == '.' && IsDigit(c3))) {
      return ScanNumber(aToken);
    }
    if (c2 == '-' && c3 == '>') {
      Advance(3);
      aToken.mType = eCSSToken_HTMLComment;
      aToken.mIdent.AssignLiteral("-->");
      return true;
    }
  }

  // the other HTML-comment token
  if (ch == '<' && Peek(1) == '!' && Peek(2) == '-' && Peek(3) == '-') {
    Advance(4);
    aToken.mType = eCSSToken_HTMLComment;
    aToken.mIdent.AssignLiteral("<!--");
    return true;
  }

  // AT_KEYWORD
  if (ch == '@') {
    return ScanAtKeyword(aToken);
  }

  // HASH
  if (ch == '#') {
    return ScanHash(aToken);
  }

  // STRING
  if (ch == '"' || ch == '\'') {
    return ScanString(aToken);
  }

  // Match operators: ~= |= ^= $= *=
  nsCSSTokenType opType = MatchOperatorType(ch);
  if (opType != eCSSToken_Symbol && Peek(1) == '=') {
    aToken.mType = opType;
    Advance(2);
    return true;
  }

  // Otherwise, a symbol (DELIM).
  aToken.mSymbol = ch;
  Advance();
  return true;
}

/* nsCSSGridTemplateAreaScanner methods. */

nsCSSGridTemplateAreaScanner::nsCSSGridTemplateAreaScanner(const nsAString& aBuffer)
  : mBuffer(aBuffer.BeginReading())
  , mOffset(0)
  , mCount(aBuffer.Length())
{
}

bool
nsCSSGridTemplateAreaScanner::Next(nsCSSGridTemplateAreaToken& aTokenResult)
{
  int32_t ch;
  // Skip whitespace
  do {
    if (mOffset >= mCount) {
      return false;
    }
    ch = mBuffer[mOffset];
    mOffset++;
  } while (IsWhitespace(ch));

  if (IsOpenCharClass(ch, IS_IDCHAR)) {
    // Named cell token
    uint32_t start = mOffset - 1;  // offset of |ch|
    while (mOffset < mCount && IsOpenCharClass(mBuffer[mOffset], IS_IDCHAR)) {
      mOffset++;
    }
    aTokenResult.mName.Assign(&mBuffer[start], mOffset - start);
    aTokenResult.isTrash = false;
  } else if (ch == '.') {
    // Null cell token
    // Skip any other '.'
    while (mOffset < mCount && mBuffer[mOffset] == '.') {
      mOffset++;
    }
    aTokenResult.mName.Truncate();
    aTokenResult.isTrash = false;
  } else {
    // Trash token
    aTokenResult.isTrash = true;
  }
  return true;
}