Bug 1340260 - Fix potentially wrong string returned from Tokenizer::ReadUntil, keep Record/Claim work after ReadUntil. r=froydnj

--HG--
extra : rebase_source : 1029f9b6743085b7865554fdedc1c96e9239b13a

xpcom/ds/Tokenizer.cpp

@@ -214,8 +214,9 @@ Tokenizer::ReadUntil(Token const& aToken, nsACString& aResult, ClaimInclusion aI
 bool
 Tokenizer::ReadUntil(Token const& aToken, nsDependentCSubstring& aResult, ClaimInclusion aInclude)
 {
+  nsACString::const_char_iterator record = mRecord;
   Record();
-  nsACString::const_char_iterator rollback = mCursor;
+  nsACString::const_char_iterator rollback = mRollback = mCursor;
 
   bool found = false;
   Token t;
@@ -224,10 +225,16 @@ Tokenizer::ReadUntil(Token const& aToken, nsDependentCSubstring& aResult, ClaimI
       found = true;
       break;
     }
+    if (t.Equals(Token::EndOfFile())) {
+      // We don't want to eat it.
+      Rollback();
+      break;
+    }
   }
 
   Claim(aResult, aInclude);
   mRollback = rollback;
+  mRecord = record;
   return found;
 }
 

xpcom/tests/gtest/TestTokenizer.cpp

@@ -1132,3 +1132,26 @@ TEST(Tokenizer, IncrementalBuffering2)
   i.FinishInput();
   EXPECT_TRUE(test == 6);
 }
+
+TEST(Tokenizer, RecordAndReadUntil)
+{
+  typedef TokenizerBase::Token Token;
+
+  Tokenizer t("aaaa,bbbb");
+  t.SkipWhites();
+  nsDependentCSubstring subject;
+
+  EXPECT_TRUE(t.ReadUntil(mozilla::Tokenizer::Token::Char(','), subject));
+  EXPECT_FALSE(t.CheckChar(','));
+  EXPECT_TRUE(subject.Length() == 4);
+  EXPECT_TRUE(subject == "aaaa");
+
+  EXPECT_FALSE(t.ReadUntil(mozilla::Tokenizer::Token::Char(','), subject));
+  EXPECT_TRUE(subject.Length() == 4);
+  EXPECT_TRUE(subject == "bbbb");
+
+  EXPECT_FALSE(t.ReadUntil(mozilla::Tokenizer::Token::Char(','), subject));
+  EXPECT_TRUE(subject.Length() == 0);
+
+  EXPECT_TRUE(t.CheckEOF());
+}