Bug 1434429 - Move TokenStreamSpecific::ungetChar into a new GeneralTokenStreamChars<CharT, AnyCharsAccess> inserted between TokenStreamCharsBase<CharT> and TokenStreamChars<CharT, AnyCharsAccess> in the token stream inheritance hierarchy. r=arai

--HG-- extra : rebase_source : a5ed08ccd92c29476b1219f5966aa3d63de539d2
2018-01-18 11:34:27 -08:00 · 2018-01-18 11:34:27 -08:00 · de083b3ca7
--- a/js/src/frontend/Parser.h
+++ b/js/src/frontend/Parser.h
@ -628,10 +628,10 @@ class ParserAnyCharsAccess
 {
  public:
    using TokenStreamSpecific = typename Parser::TokenStream;
-    using TokenStreamChars = typename TokenStreamSpecific::CharsBase;
+    using GeneralTokenStreamChars = typename TokenStreamSpecific::GeneralCharsBase;

-    static inline TokenStreamAnyChars& anyChars(TokenStreamChars* ts);
-    static inline const TokenStreamAnyChars& anyChars(const TokenStreamChars* ts);
+    static inline TokenStreamAnyChars& anyChars(GeneralTokenStreamChars* ts);
+    static inline const TokenStreamAnyChars& anyChars(const GeneralTokenStreamChars* ts);
 };

 // Specify a value for an ES6 grammar parametrization.  We have no enum for
@ -1518,8 +1518,8 @@ class Parser<FullParseHandler, CharT> final
 };

 template<class Parser>
-/* static */ inline TokenStreamAnyChars&
-ParserAnyCharsAccess<Parser>::anyChars(TokenStreamChars* ts)
+/* static */ inline const TokenStreamAnyChars&
+ParserAnyCharsAccess<Parser>::anyChars(const GeneralTokenStreamChars* ts)
 {
    // The structure we're walking through looks like this:
    //
@ -1529,35 +1529,21 @@ ParserAnyCharsAccess<Parser>::anyChars(TokenStreamChars* ts)
    //       TokenStreamAnyChars anyChars;
    //       ...;
    //   };
-    //   struct Parser : ParserBase
+    //   struct Parser : <class that ultimately inherits from ParserBase>
    //   {
    //       ...;
    //       TokenStreamSpecific tokenStream;
    //       ...;
    //   };
    //
-    // We're passed a TokenStreamChars* corresponding to a base class of
-    // Parser::tokenStream.  We cast that pointer to a TokenStreamSpecific*,
+    // We're passed a GeneralTokenStreamChars* (this being a base class of
+    // Parser::tokenStream).  We cast that pointer to a TokenStreamSpecific*,
    // then translate that to the enclosing Parser*, then return the |anyChars|
    // member within.

-    auto* tss = static_cast<TokenStreamSpecific*>(ts);
-
-    auto tssAddr = reinterpret_cast<uintptr_t>(tss);
-
-    using ActualTokenStreamType = decltype(static_cast<Parser*>(nullptr)->tokenStream);
-    static_assert(mozilla::IsSame<ActualTokenStreamType, TokenStreamSpecific>::value,
-                                  "Parser::tokenStream must have type TokenStreamSpecific");
-
-    uintptr_t parserAddr = tssAddr - offsetof(Parser, tokenStream);
-
-    return reinterpret_cast<Parser*>(parserAddr)->anyChars;
-}
-
-template<class Parser>
-/* static */ inline const TokenStreamAnyChars&
-ParserAnyCharsAccess<Parser>::anyChars(const typename Parser::TokenStream::CharsBase* ts)
-{
+    static_assert(mozilla::IsBaseOf<GeneralTokenStreamChars,
+                                    TokenStreamSpecific>::value,
+                  "the static_cast<> below assumes a base-class relationship");
    const auto* tss = static_cast<const TokenStreamSpecific*>(ts);

    auto tssAddr = reinterpret_cast<uintptr_t>(tss);
@ -1571,6 +1557,16 @@ ParserAnyCharsAccess<Parser>::anyChars(const typename Parser::TokenStream::Chars
    return reinterpret_cast<const Parser*>(parserAddr)->anyChars;
 }

+template<class Parser>
+/* static */ inline TokenStreamAnyChars&
+ParserAnyCharsAccess<Parser>::anyChars(GeneralTokenStreamChars* ts)
+{
+    const TokenStreamAnyChars& anyCharsConst =
+        anyChars(const_cast<const GeneralTokenStreamChars*>(ts));
+
+    return const_cast<TokenStreamAnyChars&>(anyCharsConst);
+}
+
 template <class ParseHandler, typename CharT>
 class MOZ_STACK_CLASS AutoAwaitIsKeyword
 {
--- a/js/src/frontend/TokenStream.cpp
+++ b/js/src/frontend/TokenStream.cpp
@ -567,7 +567,7 @@ TokenStreamAnyChars::undoGetChar()

 template<typename CharT, class AnyCharsAccess>
 void
-TokenStreamSpecific<CharT, AnyCharsAccess>::ungetChar(int32_t c)
+GeneralTokenStreamChars<CharT, AnyCharsAccess>::ungetChar(int32_t c)
 {
    if (c == EOF)
        return;
--- a/js/src/frontend/TokenStream.h
+++ b/js/src/frontend/TokenStream.h
@ -73,30 +73,62 @@
 *
 * All such functionality lives in TokenStreamCharsBase<CharT>.
 *
- * == TokenStreamChars<CharT, AnyCharsAccess> → TokenStreamCharsBase<CharT> ==
+ * == GeneralTokenStreamChars<CharT, AnyCharsAccess> →
+ *    TokenStreamCharsBase<CharT> ==
 *
- * Some functionality operates at a very low level upon character-type-specific
- * data, but in distinct ways.  For example, "is this character the start of a
- * multi-character codepoint?"  Consider how such functionality would work on
- * various encodings (hypothetically -- we haven't fully implemented any
- * particular single-byte encoding support yet):
+ * Some functionality operates differently on different character types, just
+ * as for TokenStreamCharsBase, but additionally requires access to character-
+ * type-agnostic information in TokenStreamAnyChars.  For example, getting the
+ * next character performs different steps for different character types and
+ * must access TokenStreamAnyChars to update line break information.
 *
- *   * For two-byte text, the character must pass |unicode::IsLeadSurrogate|.
- *   * For single-byte Latin-1 text, there are no multi-character codepoints.
- *   * For single-byte UTF-8 text, the answer depends on how many high bits of
- *     the character are set.
+ * Such functionality, if it can be defined using the same algorithm for all
+ * character types, lives in GeneralTokenStreamChars<CharT, AnyCharsAccess>.
+ * The AnyCharsAccess parameter provides a way for a GeneralTokenStreamChars
+ * instance to access its corresponding TokenStreamAnyChars, without inheriting
+ * from it.
+ *
+ * GeneralTokenStreamChars<CharT, AnyCharsAccess> is just functionality, no
+ * actual member data.
 *
 * Such functionality all lives in TokenStreamChars<CharT, AnyCharsAccess>, a
 * declared-but-not-defined template class whose specializations have a common
 * public interface (plus whatever private helper functions are desirable).
 *
- * Why the AnyCharsAccess parameter?  Some functionality along these lines
- * really wants TokenStreamSpecific, below, e.g. to report an error.  Providing
- * this parameter allows TokenStreamChars functions to statically cast to this
- * presumed superclass to access its functionality.
+ * == TokenStreamChars<CharT, AnyCharsAccess> →
+ *    GeneralTokenStreamChars<CharT, AnyCharsAccess> ==
 *
- * TokenStreamChars<CharT, AnyCharsAccess> is just functionality, no actual
- * member data.
+ * Some functionality is like that in GeneralTokenStreamChars, *but* it's
+ * defined entirely differently for different character types.
+ *
+ * For example, consider "match a multi-code unit code point" (hypothetically:
+ * we've only implemented two-byte tokenizing right now):
+ *
+ *   * For two-byte text, there must be two code units to get, the leading code
+ *     unit must be a UTF-16 lead surrogate, and the trailing code unit must be
+ *     a UTF-16 trailing surrogate.  (If any of these fail to hold, a next code
+ *     unit encodes that code point and is not multi-code unit.)
+ *   * For single-byte Latin-1 text, there are no multi-code unit code points.
+ *   * For single-byte UTF-8 text, the first code unit must have N > 1 of its
+ *     highest bits set (and the next unset), and |N - 1| successive code units
+ *     must have their high bit set and next-highest bit unset, *and*
+ *     concatenating all unconstrained bits together must not produce a code
+ *     point value that could have been encoded in fewer code units.
+ *
+ * This functionality can't be implemented as member functions in
+ * GeneralTokenStreamChars because we'd need to *partially specialize* those
+ * functions -- hold CharT constant while letting AnyCharsAccess vary.  But
+ * C++ forbids function template partial specialization like this: either you
+ * fix *all* parameters or you fix none of them.
+ *
+ * Fortunately, C++ *does* allow *class* template partial specialization.  So
+ * TokenStreamChars is a template class with one specialization per CharT.
+ * Functions can be defined differently in the different specializations,
+ * because AnyCharsAccess as the only template parameter on member functions
+ * *can* vary.
+ *
+ * All TokenStreamChars<CharT, AnyCharsAccess> specializations, one per CharT,
+ * are just functionality, no actual member data.
 *
 * == TokenStreamSpecific<CharT, AnyCharsAccess> →
 *    TokenStreamChars<CharT, AnyCharsAccess>, TokenStreamShared ==
@ -472,6 +504,7 @@ class TokenStreamAnyChars
    TokenStreamAnyChars(JSContext* cx, const ReadOnlyCompileOptions& options,
                        StrictModeGetter* smg);

+    template<typename CharT, class AnyCharsAccess> friend class GeneralTokenStreamChars;
    template<typename CharT, class AnyCharsAccess> friend class TokenStreamSpecific;

    // Accessors.
@ -959,36 +992,59 @@ TokenStreamCharsBase<char16_t>::atomizeChars(JSContext* cx, const char16_t* char
    return AtomizeChars(cx, chars, length);
 }

-template<typename CharT, class AnyCharsAccess> class TokenStreamChars;
-
-template<class AnyCharsAccess>
-class TokenStreamChars<char16_t, AnyCharsAccess>
-  : public TokenStreamCharsBase<char16_t>
+template<typename CharT, class AnyCharsAccess>
+class GeneralTokenStreamChars
+  : public TokenStreamCharsBase<CharT>
 {
-    using Self = TokenStreamChars<char16_t, AnyCharsAccess>;
-    using CharsBase = TokenStreamCharsBase<char16_t>;
+    using CharsSharedBase = TokenStreamCharsBase<CharT>;

-    using TokenStreamSpecific = frontend::TokenStreamSpecific<char16_t, AnyCharsAccess>;
+    using typename CharsSharedBase::TokenBuf;
+
+    using CharsSharedBase::userbuf;
+
+  public:
+    using CharsSharedBase::CharsSharedBase;
+
+    TokenStreamAnyChars& anyCharsAccess() {
+        return AnyCharsAccess::anyChars(this);
+    }
+
+    const TokenStreamAnyChars& anyCharsAccess() const {
+        return AnyCharsAccess::anyChars(this);
+    }
+
+    using TokenStreamSpecific = frontend::TokenStreamSpecific<CharT, AnyCharsAccess>;

    TokenStreamSpecific* asSpecific() {
-        static_assert(mozilla::IsBaseOf<Self, TokenStreamSpecific>::value,
+        static_assert(mozilla::IsBaseOf<GeneralTokenStreamChars, TokenStreamSpecific>::value,
                      "static_cast below presumes an inheritance relationship");

        return static_cast<TokenStreamSpecific*>(this);
    }

+    void ungetChar(int32_t c);
+};
+
+template<typename CharT, class AnyCharsAccess> class TokenStreamChars;
+
+template<class AnyCharsAccess>
+class TokenStreamChars<char16_t, AnyCharsAccess>
+  : public GeneralTokenStreamChars<char16_t, AnyCharsAccess>
+{
+    using Self = TokenStreamChars<char16_t, AnyCharsAccess>;
+    using GeneralCharsBase = GeneralTokenStreamChars<char16_t, AnyCharsAccess>;
+    using CharsSharedBase = TokenStreamCharsBase<char16_t>;
+
    bool matchTrailForLeadSurrogate(char16_t lead, uint32_t* codePoint);

  public:
-    using CharsBase::CharsBase;
+    using typename GeneralCharsBase::TokenStreamSpecific;

-    TokenStreamAnyChars& anyChars() {
-        return AnyCharsAccess::anyChars(this);
-    }
+    using GeneralCharsBase::asSpecific;
+    using GeneralCharsBase::anyCharsAccess;

-    const TokenStreamAnyChars& anyChars() const {
-        return AnyCharsAccess::anyChars(this);
-    }
+  public:
+    using GeneralCharsBase::GeneralCharsBase;

    MOZ_ALWAYS_INLINE bool isMultiUnitCodepoint(char16_t c, uint32_t* codepoint) {
        if (MOZ_LIKELY(!unicode::IsLeadSurrogate(c)))
@ -1046,6 +1102,7 @@ class MOZ_STACK_CLASS TokenStreamSpecific
 {
  public:
    using CharsBase = TokenStreamChars<CharT, AnyCharsAccess>;
+    using GeneralCharsBase = GeneralTokenStreamChars<CharT, AnyCharsAccess>;
    using CharsSharedBase = TokenStreamCharsBase<CharT>;

    // Anything inherited through a base class whose type depends upon this
@ -1063,6 +1120,7 @@ class MOZ_STACK_CLASS TokenStreamSpecific
    using typename CharsSharedBase::Position;

  public:
+    using GeneralCharsBase::anyCharsAccess;
    using CharsSharedBase::getTokenbuf;

  private:
@ -1075,20 +1133,13 @@ class MOZ_STACK_CLASS TokenStreamSpecific
    using CharsSharedBase::copyTokenbufTo;
    using CharsBase::isMultiUnitCodepoint;
    using CharsSharedBase::tokenbuf;
+    using GeneralCharsBase::ungetChar;
    using CharsSharedBase::userbuf;

  public:
    TokenStreamSpecific(JSContext* cx, const ReadOnlyCompileOptions& options,
                        const CharT* base, size_t length);

-    TokenStreamAnyChars& anyCharsAccess() {
-        return CharsBase::anyChars();
-    }
-
-    const TokenStreamAnyChars& anyCharsAccess() const {
-        return CharsBase::anyChars();
-    }
-
    // If there is an invalid escape in a template, report it and return false,
    // otherwise return true.
    bool checkForInvalidTemplateEscapeError() {
@ -1362,7 +1413,6 @@ class MOZ_STACK_CLASS TokenStreamSpecific
    MOZ_MUST_USE bool getChar(int32_t* cp);
    int32_t getCharIgnoreEOL();

-    void ungetChar(int32_t c);
    void ungetCharIgnoreEOL(int32_t c);
    Token* newToken(ptrdiff_t adjust);
    uint32_t peekUnicodeEscape(uint32_t* codePoint);