mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

WIP: push-based tokenization

This commit is contained in:
rickg 1998-05-07 07:19:47 +00:00
Родитель d1c80f42b5
Коммит c5fd8c61c7
22 изменённых файлов: 1032 добавлений и 512 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

171
htmlparser/src/CNavDelegate.cpp
Просмотреть файл

@ -85,36 +85,36 @@ nsIDTD* CNavDelegate::GetDTD(void) const{
* @param
* @return
*/
CToken* CNavDelegate::ConsumeTag(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode) {
CToken* result=0;
PRInt32 CNavDelegate::ConsumeTag(PRUnichar aChar,CScanner& aScanner,CToken*& aToken) {
nsAutoString empty("");
anErrorCode=anErrorCode=aScanner.GetChar(aChar);
PRInt32 result=aScanner.GetChar(aChar);
switch(aChar) {
case kForwardSlash:
PRUnichar ch;
anErrorCode=aScanner.Peek(ch);
result=aScanner.Peek(ch);
if(nsString::IsAlpha(ch))
result=new CEndToken(empty);
else result=new CCommentToken(empty); //Special case: </ ...> is treated as a comment
aToken=new CEndToken(empty);
else aToken=new CCommentToken(empty); //Special case: </ ...> is treated as a comment
break;
case kExclamation:
result=new CCommentToken(empty);
aToken=new CCommentToken(empty);
break;
default:
if(nsString::IsAlpha(aChar))
return ConsumeStartTag(aChar,aScanner,anErrorCode);
return ConsumeStartTag(aChar,aScanner,aToken);
else if(kEOF!=aChar) {
nsAutoString temp("<");
return ConsumeText(temp,aScanner,anErrorCode);
return ConsumeText(temp,aScanner,aToken);
}
} //switch
if(result!=0) {
anErrorCode= result->Consume(aChar,aScanner); //tell new token to finish consuming text...
if(anErrorCode) {
result=0;
delete result;
if(0!=aToken) {
result= aToken->Consume(aChar,aScanner); //tell new token to finish consuming text...
if(result) {
delete aToken;
aToken=0;
}
}
return result;
@ -129,23 +129,23 @@ CToken* CNavDelegate::ConsumeTag(PRUnichar aChar,CScanner& aScanner,PRInt32& anE
* @param aScanner: see nsScanner.h
* @return
*/
void CNavDelegate::ConsumeAttributes(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode) {
PRInt32 CNavDelegate::ConsumeAttributes(PRUnichar aChar,CScanner& aScanner) {
PRBool done=PR_FALSE;
nsAutoString as("");
anErrorCode=kNoError;
while((!done) && (anErrorCode==kNoError)) {
CToken* result = new CAttributeToken(as);
if(result){
anErrorCode= result->Consume(aChar,aScanner); //tell new token to finish consuming text...
mTokenDeque.Push(result);
PRInt32 result=kNoError;
while((!done) && (result==kNoError)) {
CToken* theToken= new CAttributeToken(as);
if(theToken){
result= theToken->Consume(aChar,aScanner); //tell new token to finish consuming text...
mTokenDeque.Push(theToken);
}
aScanner.Peek(aChar);
if(aChar==kGreaterThan) { //you just ate the '>'
aScanner.GetChar(aChar); //skip the '>'
done=PR_TRUE;
}
if(aChar==kGreaterThan) { //you just ate the '>'
aScanner.GetChar(aChar); //skip the '>'
done=PR_TRUE;
}
}
return;
return result;
}
/**
@ -157,7 +157,7 @@ void CNavDelegate::ConsumeAttributes(PRUnichar aChar,CScanner& aScanner,PRInt32&
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* CNavDelegate::ConsumeContentToEndTag(const nsString& aString,PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode){
PRInt32 CNavDelegate::ConsumeContentToEndTag(const nsString& aString,PRUnichar aChar,CScanner& aScanner,CToken*& aToken){
//In the case that we just read the given tag, we should go and
//consume all the input until we find a matching end tag.
@ -165,9 +165,9 @@ CToken* CNavDelegate::ConsumeContentToEndTag(const nsString& aString,PRUnichar a
nsAutoString endTag("</");
endTag.Append(aString);
endTag.Append(">");
CSkippedContentToken* sc=new CSkippedContentToken(endTag);
anErrorCode= sc->Consume(aChar,aScanner); //tell new token to finish consuming text...
return sc;
aToken=new CSkippedContentToken(endTag);
PRInt32 result= aToken->Consume(aChar,aScanner); //tell new token to finish consuming text...
return result;
}
/**
@ -180,30 +180,32 @@ CToken* CNavDelegate::ConsumeContentToEndTag(const nsString& aString,PRUnichar a
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* CNavDelegate::ConsumeStartTag(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode) {
CStartToken* result=new CStartToken(nsAutoString(""));
if(result) {
anErrorCode= result->Consume(aChar,aScanner); //tell new token to finish consuming text...
if(result->IsAttributed()) {
ConsumeAttributes(aChar,aScanner,anErrorCode);
PRInt32 CNavDelegate::ConsumeStartTag(PRUnichar aChar,CScanner& aScanner,CToken*& aToken) {
aToken=new CStartToken(nsAutoString(""));
PRInt32 result=kNoError;
if(aToken) {
result= aToken->Consume(aChar,aScanner); //tell new token to finish consuming text...
if(((CStartToken*)aToken)->IsAttributed()) {
result=ConsumeAttributes(aChar,aScanner);
}
//now that that's over with, we have one more problem to solve.
//In the case that we just read a <SCRIPT> or <STYLE> tags, we should go and
//consume all the content itself.
nsString& str=result->GetText();
nsString& str=aToken->GetText();
CToken* skippedToken=0;
if(str.EqualsIgnoreCase("SCRIPT") ||
str.EqualsIgnoreCase("STYLE") ||
str.EqualsIgnoreCase("TITLE") ||
str.EqualsIgnoreCase("TEXTAREA")) {
CToken* sc=ConsumeContentToEndTag(str,aChar,aScanner,anErrorCode);
result=ConsumeContentToEndTag(str,aChar,aScanner,skippedToken);
if(sc){
if(skippedToken){
//now we strip the ending sequence from our new SkippedContent token...
PRInt32 slen=str.Length()+3;
nsString& skippedText=sc->GetText();
nsString& skippedText=skippedToken->GetText();
skippedText.Cut(skippedText.Length()-slen,slen);
mTokenDeque.Push(sc);
mTokenDeque.Push(skippedToken);
//In the case that we just read a given tag, we should go and
//consume all the tag content itself (and throw it all away).
@ -226,22 +228,21 @@ CToken* CNavDelegate::ConsumeStartTag(PRUnichar aChar,CScanner& aScanner,PRInt32
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* CNavDelegate::ConsumeEntity(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode) {
CToken* result = 0;
PRInt32 CNavDelegate::ConsumeEntity(PRUnichar aChar,CScanner& aScanner,CToken*& aToken) {
PRUnichar ch;
anErrorCode=aScanner.GetChar(ch);
PRInt32 result=aScanner.GetChar(ch);
if(nsString::IsAlpha(ch)) { //handle common enity references &xxx; or &#000.
result = new CEntityToken(nsAutoString(""));
anErrorCode= result->Consume(ch,aScanner); //tell new token to finish consuming text...
aToken = new CEntityToken(nsAutoString(""));
result = aToken->Consume(ch,aScanner); //tell new token to finish consuming text...
}
else if(kHashsign==ch) {
result = new CEntityToken(nsAutoString(""));
anErrorCode=result->Consume(ch,aScanner);
aToken = new CEntityToken(nsAutoString(""));
result=aToken->Consume(ch,aScanner);
}
else {
//oops, we're actually looking at plain text...
nsAutoString temp("&");
return ConsumeText(temp,aScanner,anErrorCode);
result=ConsumeText(temp,aScanner,aToken);
}
return result;
}
@ -256,9 +257,12 @@ CToken* CNavDelegate::ConsumeEntity(PRUnichar aChar,CScanner& aScanner,PRInt32&
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* CNavDelegate::ConsumeWhitespace(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode) {
CToken* result = new CWhitespaceToken(nsAutoString(""));
anErrorCode=result->Consume(aChar,aScanner);
PRInt32 CNavDelegate::ConsumeWhitespace(PRUnichar aChar,CScanner& aScanner,CToken*& aToken) {
aToken = new CWhitespaceToken(nsAutoString(""));
PRInt32 result=kNoError;
if(aToken) {
result=aToken->Consume(aChar,aScanner);
}
return result;
}
@ -272,9 +276,12 @@ CToken* CNavDelegate::ConsumeWhitespace(PRUnichar aChar,CScanner& aScanner,PRInt
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* CNavDelegate::ConsumeComment(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode){
CToken* result= new CCommentToken(nsAutoString(""));
anErrorCode=result->Consume(aChar,aScanner);
PRInt32 CNavDelegate::ConsumeComment(PRUnichar aChar,CScanner& aScanner,CToken*& aToken){
aToken = new CCommentToken(nsAutoString(""));
PRInt32 result=kNoError;
if(aToken) {
result=aToken->Consume(aChar,aScanner);
}
return result;
}
@ -288,11 +295,12 @@ CToken* CNavDelegate::ConsumeComment(PRUnichar aChar,CScanner& aScanner,PRInt32&
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* CNavDelegate::ConsumeText(const nsString& aString,CScanner& aScanner,PRInt32& anErrorCode){
CToken* result=new CTextToken(aString);
if(result) {
PRInt32 CNavDelegate::ConsumeText(const nsString& aString,CScanner& aScanner,CToken*& aToken){
PRInt32 result=kNoError;
if(aToken=new CTextToken(aString)) {
PRUnichar ch;
anErrorCode=result->Consume(ch,aScanner);
result=aToken->Consume(ch,aScanner);
}
return result;
}
@ -306,10 +314,11 @@ CToken* CNavDelegate::ConsumeText(const nsString& aString,CScanner& aScanner,PRI
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* CNavDelegate::ConsumeNewline(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode){
CToken* result=new CNewlineToken(nsAutoString(""));
if(result) {
anErrorCode=result->Consume(aChar,aScanner);
PRInt32 CNavDelegate::ConsumeNewline(PRUnichar aChar,CScanner& aScanner,CToken*& aToken){
aToken=new CNewlineToken(nsAutoString(""));
PRInt32 result=kNoError;
if(aToken) {
result=aToken->Consume(aChar,aScanner);
}
return result;
}
@ -326,36 +335,36 @@ CToken* CNavDelegate::ConsumeNewline(PRUnichar aChar,CScanner& aScanner,PRInt32&
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* CNavDelegate::GetToken(CScanner& aScanner,PRInt32& anErrorCode){
CToken* result=0;
PRInt32 CNavDelegate::GetToken(CScanner& aScanner,CToken*& aToken){
PRInt32 result=kNoError;
PRUnichar aChar;
if(mTokenDeque.GetSize()>0) {
return (CToken*)mTokenDeque.Pop();
aToken=(CToken*)mTokenDeque.Pop();
return result;
}
while(!aScanner.Eof()) {
anErrorCode=aScanner.GetChar(aChar);
aToken=0;
while(!aScanner.Eof()) {
result=aScanner.GetChar(aChar);
switch(aChar) {
case kAmpersand:
return ConsumeEntity(aChar,aScanner,anErrorCode);
return ConsumeEntity(aChar,aScanner,aToken);
case kLessThan:
return ConsumeTag(aChar,aScanner,anErrorCode);
return ConsumeTag(aChar,aScanner,aToken);
case kCR: case kLF:
return ConsumeNewline(aChar,aScanner,anErrorCode);
return ConsumeNewline(aChar,aScanner,aToken);
case kNotFound:
break;
default:
if(nsString::IsSpace(aChar))
return ConsumeWhitespace(aChar,aScanner,anErrorCode);
else
{
if(!nsString::IsSpace(aChar)) {
nsAutoString temp(aChar);
return ConsumeText(temp,aScanner,anErrorCode);
return ConsumeText(temp,aScanner,aToken);
}
else return ConsumeWhitespace(aChar,aScanner,aToken);
break;
} //switch
if(anErrorCode==kEOF)
anErrorCode=0;
if(result==kEOF)
result=0;
} //while
return result;
}
@ -395,7 +404,7 @@ PRBool CNavDelegate::WillAddToken(CToken& /*aToken*/) {
* @update gess 3/25/98
* @return TRUE if preinitialization completed successfully
*/
PRBool CNavDelegate::WillTokenize() {
PRBool CNavDelegate::WillTokenize(PRBool aIncremental) {
PRBool result=PR_TRUE;
return result;
}
@ -408,7 +417,7 @@ PRBool CNavDelegate::WillTokenize() {
* @update gess 3/25/98
* @return TRUE if preinitialization completed successfully
*/
PRBool CNavDelegate::DidTokenize() {
PRBool CNavDelegate::DidTokenize(PRBool aIncremental) {
PRBool result=PR_TRUE;
return result;
}

24
htmlparser/src/CNavDelegate.h
Просмотреть файл

@ -49,11 +49,11 @@ class CNavDelegate : public ITokenizerDelegate {
CNavDelegate();
CNavDelegate(CNavDelegate& aDelegate);
virtual CToken* GetToken(CScanner& aScanner,PRInt32& anErrorCode);
virtual PRInt32 GetToken(CScanner& aScanner,CToken*& aToken);
virtual PRBool WillAddToken(CToken& aToken);
virtual PRBool WillTokenize();
virtual PRBool DidTokenize();
virtual PRBool WillTokenize(PRBool aIncremental);
virtual PRBool DidTokenize(PRBool aIncremental);
virtual eParseMode GetParseMode(void) const;
virtual nsIDTD* GetDTD(void) const;
@ -63,17 +63,17 @@ class CNavDelegate : public ITokenizerDelegate {
virtual CToken* CreateTokenOfType(eHTMLTokenTypes aType);
CToken* ConsumeTag(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeStartTag(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
void ConsumeAttributes(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeText(const nsString& aString,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeEntity(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeWhitespace(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeComment(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeNewline(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
PRInt32 ConsumeTag(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeStartTag(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeAttributes(PRUnichar aChar,CScanner& aScanner);
PRInt32 ConsumeText(const nsString& aString,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeEntity(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeWhitespace(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeComment(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeNewline(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
//the only special case method...
virtual CToken* ConsumeContentToEndTag(const nsString& aString,PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
virtual PRInt32 ConsumeContentToEndTag(const nsString& aString,PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
nsDeque mTokenDeque;

174
htmlparser/src/COtherDelegate.cpp
Просмотреть файл

@ -20,8 +20,7 @@
#include "COtherDelegate.h"
#include "nsScanner.h"
#include "nsParserTypes.h"
#include "COtherDTD.h"
#include "CNavDTD.h"
// Note: We already handle the following special case conditions:
@ -66,6 +65,7 @@ eParseMode COtherDelegate::GetParseMode(void) const {
return eParseMode_unknown;
}
/**
* Cause delegate to create and return a new DTD.
*
@ -73,7 +73,7 @@ eParseMode COtherDelegate::GetParseMode(void) const {
* @return new DTD or null
*/
nsIDTD* COtherDelegate::GetDTD(void) const{
return new COtherDTD();
return new CNavDTD();
}
/**
@ -85,36 +85,36 @@ nsIDTD* COtherDelegate::GetDTD(void) const{
* @param
* @return
*/
CToken* COtherDelegate::ConsumeTag(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode) {
CToken* result=0;
PRInt32 COtherDelegate::ConsumeTag(PRUnichar aChar,CScanner& aScanner,CToken*& aToken) {
nsAutoString empty("");
anErrorCode=anErrorCode=aScanner.GetChar(aChar);
PRInt32 result=aScanner.GetChar(aChar);
switch(aChar) {
case kForwardSlash:
PRUnichar ch;
anErrorCode=aScanner.Peek(ch);
result=aScanner.Peek(ch);
if(nsString::IsAlpha(ch))
result=new CEndToken(empty);
else result=new CCommentToken(empty); //Special case: </ ...> is treated as a comment
aToken=new CEndToken(empty);
else aToken=new CCommentToken(empty); //Special case: </ ...> is treated as a comment
break;
case kExclamation:
result=new CCommentToken(empty);
aToken=new CCommentToken(empty);
break;
default:
if(nsString::IsAlpha(aChar))
return ConsumeStartTag(aChar,aScanner,anErrorCode);
return ConsumeStartTag(aChar,aScanner,aToken);
else if(kEOF!=aChar) {
nsAutoString temp("<");
return ConsumeText(temp,aScanner,anErrorCode);
return ConsumeText(temp,aScanner,aToken);
}
} //switch
if(result!=0) {
anErrorCode= result->Consume(aChar,aScanner); //tell new token to finish consuming text...
if(anErrorCode) {
result=0;
delete result;
if(0!=aToken) {
result= aToken->Consume(aChar,aScanner); //tell new token to finish consuming text...
if(result) {
delete aToken;
aToken=0;
}
}
return result;
@ -129,23 +129,23 @@ CToken* COtherDelegate::ConsumeTag(PRUnichar aChar,CScanner& aScanner,PRInt32& a
* @param aScanner: see nsScanner.h
* @return
*/
void COtherDelegate::ConsumeAttributes(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode) {
PRInt32 COtherDelegate::ConsumeAttributes(PRUnichar aChar,CScanner& aScanner) {
PRBool done=PR_FALSE;
nsAutoString as("");
anErrorCode=kNoError;
while((!done) && (anErrorCode==kNoError)) {
CToken* result = new CAttributeToken(as);
if(result){
anErrorCode= result->Consume(aChar,aScanner); //tell new token to finish consuming text...
mTokenDeque.Push(result);
PRInt32 result=kNoError;
while((!done) && (result==kNoError)) {
CToken* theToken= new CAttributeToken(as);
if(theToken){
result= theToken->Consume(aChar,aScanner); //tell new token to finish consuming text...
mTokenDeque.Push(theToken);
}
aScanner.Peek(aChar);
if(aChar==kGreaterThan) { //you just ate the '>'
aScanner.GetChar(aChar); //skip the '>'
done=PR_TRUE;
}
if(aChar==kGreaterThan) { //you just ate the '>'
aScanner.GetChar(aChar); //skip the '>'
done=PR_TRUE;
}
}
return;
return result;
}
/**
@ -157,7 +157,7 @@ void COtherDelegate::ConsumeAttributes(PRUnichar aChar,CScanner& aScanner,PRInt3
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* COtherDelegate::ConsumeContentToEndTag(const nsString& aString,PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode){
PRInt32 COtherDelegate::ConsumeContentToEndTag(const nsString& aString,PRUnichar aChar,CScanner& aScanner,CToken*& aToken){
//In the case that we just read the given tag, we should go and
//consume all the input until we find a matching end tag.
@ -165,9 +165,9 @@ CToken* COtherDelegate::ConsumeContentToEndTag(const nsString& aString,PRUnichar
nsAutoString endTag("</");
endTag.Append(aString);
endTag.Append(">");
CSkippedContentToken* sc=new CSkippedContentToken(endTag);
anErrorCode= sc->Consume(aChar,aScanner); //tell new token to finish consuming text...
return sc;
aToken=new CSkippedContentToken(endTag);
PRInt32 result= aToken->Consume(aChar,aScanner); //tell new token to finish consuming text...
return result;
}
/**
@ -180,30 +180,31 @@ CToken* COtherDelegate::ConsumeContentToEndTag(const nsString& aString,PRUnichar
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* COtherDelegate::ConsumeStartTag(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode) {
CStartToken* result=new CStartToken(nsAutoString(""));
if(result) {
anErrorCode= result->Consume(aChar,aScanner); //tell new token to finish consuming text...
if(result->IsAttributed()) {
ConsumeAttributes(aChar,aScanner,anErrorCode);
PRInt32 COtherDelegate::ConsumeStartTag(PRUnichar aChar,CScanner& aScanner,CToken*& aToken) {
aToken=new CStartToken(nsAutoString(""));
PRInt32 result=kNoError;
if(aToken) {
result= aToken->Consume(aChar,aScanner); //tell new token to finish consuming text...
if(((CStartToken*)aToken)->IsAttributed()) {
result=ConsumeAttributes(aChar,aScanner);
}
//now that that's over with, we have one more problem to solve.
//In the case that we just read a <SCRIPT> or <STYLE> tags, we should go and
//consume all the content itself.
nsString& str=result->GetText();
nsString& str=aToken->GetText();
if(str.EqualsIgnoreCase("SCRIPT") ||
str.EqualsIgnoreCase("STYLE") ||
str.EqualsIgnoreCase("TITLE") ||
str.EqualsIgnoreCase("TEXTAREA")) {
CToken* sc=ConsumeContentToEndTag(str,aChar,aScanner,anErrorCode);
result=ConsumeContentToEndTag(str,aChar,aScanner,aToken);
if(sc){
if(aToken){
//now we strip the ending sequence from our new SkippedContent token...
PRInt32 slen=str.Length()+3;
nsString& skippedText=sc->GetText();
nsString& skippedText=aToken->GetText();
skippedText.Cut(skippedText.Length()-slen,slen);
mTokenDeque.Push(sc);
mTokenDeque.Push(aToken);
//In the case that we just read a given tag, we should go and
//consume all the tag content itself (and throw it all away).
@ -226,22 +227,21 @@ CToken* COtherDelegate::ConsumeStartTag(PRUnichar aChar,CScanner& aScanner,PRInt
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* COtherDelegate::ConsumeEntity(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode) {
CToken* result = 0;
PRInt32 COtherDelegate::ConsumeEntity(PRUnichar aChar,CScanner& aScanner,CToken*& aToken) {
PRUnichar ch;
anErrorCode=aScanner.GetChar(ch);
PRInt32 result=aScanner.GetChar(ch);
if(nsString::IsAlpha(ch)) { //handle common enity references &xxx; or &#000.
result = new CEntityToken(nsAutoString(""));
anErrorCode= result->Consume(ch,aScanner); //tell new token to finish consuming text...
aToken = new CEntityToken(nsAutoString(""));
result = aToken->Consume(ch,aScanner); //tell new token to finish consuming text...
}
else if(kHashsign==ch) {
result = new CEntityToken(nsAutoString(""));
anErrorCode=result->Consume(ch,aScanner);
aToken = new CEntityToken(nsAutoString(""));
result=aToken->Consume(ch,aScanner);
}
else {
//oops, we're actually looking at plain text...
nsAutoString temp("&");
return ConsumeText(temp,aScanner,anErrorCode);
result=ConsumeText(temp,aScanner,aToken);
}
return result;
}
@ -256,9 +256,12 @@ CToken* COtherDelegate::ConsumeEntity(PRUnichar aChar,CScanner& aScanner,PRInt32
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* COtherDelegate::ConsumeWhitespace(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode) {
CToken* result = new CWhitespaceToken(nsAutoString(""));
anErrorCode=result->Consume(aChar,aScanner);
PRInt32 COtherDelegate::ConsumeWhitespace(PRUnichar aChar,CScanner& aScanner,CToken*& aToken) {
aToken = new CWhitespaceToken(nsAutoString(""));
PRInt32 result=kNoError;
if(aToken) {
result=aToken->Consume(aChar,aScanner);
}
return result;
}
@ -272,9 +275,12 @@ CToken* COtherDelegate::ConsumeWhitespace(PRUnichar aChar,CScanner& aScanner,PRI
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* COtherDelegate::ConsumeComment(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode){
CToken* result= new CCommentToken(nsAutoString(""));
anErrorCode=result->Consume(aChar,aScanner);
PRInt32 COtherDelegate::ConsumeComment(PRUnichar aChar,CScanner& aScanner,CToken*& aToken){
aToken = new CCommentToken(nsAutoString(""));
PRInt32 result=kNoError;
if(aToken) {
result=aToken->Consume(aChar,aScanner);
}
return result;
}
@ -288,11 +294,12 @@ CToken* COtherDelegate::ConsumeComment(PRUnichar aChar,CScanner& aScanner,PRInt3
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* COtherDelegate::ConsumeText(const nsString& aString,CScanner& aScanner,PRInt32& anErrorCode){
CToken* result=new CTextToken(aString);
if(result) {
PRInt32 COtherDelegate::ConsumeText(const nsString& aString,CScanner& aScanner,CToken*& aToken){
aToken=new CTextToken(aString);
PRInt32 result=kNoError;
if(aToken) {
PRUnichar ch;
anErrorCode=result->Consume(ch,aScanner);
result=aToken->Consume(ch,aScanner);
}
return result;
}
@ -306,10 +313,11 @@ CToken* COtherDelegate::ConsumeText(const nsString& aString,CScanner& aScanner,P
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* COtherDelegate::ConsumeNewline(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode){
CToken* result=new CNewlineToken(nsAutoString(""));
if(result) {
anErrorCode=result->Consume(aChar,aScanner);
PRInt32 COtherDelegate::ConsumeNewline(PRUnichar aChar,CScanner& aScanner,CToken*& aToken){
aToken=new CNewlineToken(nsAutoString(""));
PRInt32 result=kNoError;
if(aToken) {
result=aToken->Consume(aChar,aScanner);
}
return result;
}
@ -326,36 +334,36 @@ CToken* COtherDelegate::ConsumeNewline(PRUnichar aChar,CScanner& aScanner,PRInt3
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* COtherDelegate::GetToken(CScanner& aScanner,PRInt32& anErrorCode){
CToken* result=0;
PRInt32 COtherDelegate::GetToken(CScanner& aScanner,CToken*& aToken){
PRInt32 result=kNoError;
PRUnichar aChar;
if(mTokenDeque.GetSize()>0) {
return (CToken*)mTokenDeque.Pop();
aToken=(CToken*)mTokenDeque.Pop();
return result;
}
while(!aScanner.Eof()) {
anErrorCode=aScanner.GetChar(aChar);
while(!aScanner.Eof()) {
result=aScanner.GetChar(aChar);
switch(aChar) {
case kAmpersand:
return ConsumeEntity(aChar,aScanner,anErrorCode);
return ConsumeEntity(aChar,aScanner,aToken);
case kLessThan:
return ConsumeTag(aChar,aScanner,anErrorCode);
return ConsumeTag(aChar,aScanner,aToken);
case kCR: case kLF:
return ConsumeNewline(aChar,aScanner,anErrorCode);
return ConsumeNewline(aChar,aScanner,aToken);
case kNotFound:
break;
default:
if(nsString::IsSpace(aChar))
return ConsumeWhitespace(aChar,aScanner,anErrorCode);
else
{
if(!nsString::IsSpace(aChar)) {
nsAutoString temp(aChar);
return ConsumeText(temp,aScanner,anErrorCode);
return ConsumeText(temp,aScanner,aToken);
}
else return ConsumeWhitespace(aChar,aScanner,aToken);
break;
} //switch
if(anErrorCode==kEOF)
anErrorCode=0;
if(result==kEOF)
result=0;
} //while
return result;
}
@ -395,7 +403,7 @@ PRBool COtherDelegate::WillAddToken(CToken& /*aToken*/) {
* @update gess 3/25/98
* @return TRUE if preinitialization completed successfully
*/
PRBool COtherDelegate::WillTokenize() {
PRBool COtherDelegate::WillTokenize(PRBool aIncremental) {
PRBool result=PR_TRUE;
return result;
}
@ -408,7 +416,7 @@ PRBool COtherDelegate::WillTokenize() {
* @update gess 3/25/98
* @return TRUE if preinitialization completed successfully
*/
PRBool COtherDelegate::DidTokenize() {
PRBool COtherDelegate::DidTokenize(PRBool aIncremental) {
PRBool result=PR_TRUE;
return result;
}

25
htmlparser/src/COtherDelegate.h
Просмотреть файл

@ -49,11 +49,11 @@ class COtherDelegate : public ITokenizerDelegate {
COtherDelegate();
COtherDelegate(COtherDelegate& aDelegate);
virtual CToken* GetToken(CScanner& aScanner,PRInt32& anErrorCode);
virtual PRInt32 GetToken(CScanner& aScanner,CToken*& aToken);
virtual PRBool WillAddToken(CToken& aToken);
virtual PRBool WillTokenize();
virtual PRBool DidTokenize();
virtual PRBool WillTokenize(PRBool aIncremental);
virtual PRBool DidTokenize(PRBool aIncremental);
virtual eParseMode GetParseMode(void) const;
virtual nsIDTD* GetDTD(void) const;
@ -63,17 +63,17 @@ class COtherDelegate : public ITokenizerDelegate {
virtual CToken* CreateTokenOfType(eHTMLTokenTypes aType);
CToken* ConsumeTag(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeStartTag(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
void ConsumeAttributes(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeText(const nsString& aString,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeEntity(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeWhitespace(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeComment(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeNewline(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
PRInt32 ConsumeTag(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeStartTag(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeAttributes(PRUnichar aChar,CScanner& aScanner);
PRInt32 ConsumeText(const nsString& aString,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeEntity(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeWhitespace(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeComment(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeNewline(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
//the only special case method...
virtual CToken* ConsumeContentToEndTag(const nsString& aString,PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
virtual PRInt32 ConsumeContentToEndTag(const nsString& aString,PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
nsDeque mTokenDeque;
@ -82,4 +82,3 @@ class COtherDelegate : public ITokenizerDelegate {
#endif

10
htmlparser/src/nsHTMLParser.cpp
Просмотреть файл

@ -16,6 +16,8 @@
* Reserved.
*/
//#define __INCREMENTAL 1
#include "nsHTMLParser.h"
#include "nsHTMLContentSink.h"
#include "nsTokenizer.h"
@ -513,7 +515,15 @@ PRBool nsHTMLParser::Parse(nsIURL* aURL,eParseMode aMode){
if(mDTD)
mDTD->SetParser(this);
mTokenizer=new CTokenizer(aURL, theDelegate, mParseMode);
#ifdef __INCREMENTAL
int iter=-1;
for(;;){
mTokenizer->TokenizeAvailable(++iter);
}
#else
mTokenizer->Tokenize();
#endif
result=IterateTokens();
}
return result;

2
htmlparser/src/nsHTMLTokens.cpp
Просмотреть файл

@ -560,7 +560,7 @@ CCommentToken::CCommentToken(const nsString& aName) : CHTMLToken(aName) {
PRInt32 CCommentToken::Consume(PRUnichar aChar, CScanner& aScanner) {
PRUnichar ch,ch2;
PRInt32 result;
PRInt32 result=kNoError;
static nsAutoString terminals(">");

6
htmlparser/src/nsITokenizerDelegate.h
Просмотреть файл

@ -48,10 +48,10 @@ class CToken;
class ITokenizerDelegate {
public:
virtual PRBool WillTokenize()=0;
virtual PRBool DidTokenize()=0;
virtual PRBool WillTokenize(PRBool aIncremental)=0;
virtual PRBool DidTokenize(PRBool aIncremental)=0;
virtual CToken* GetToken(CScanner& aScanner,PRInt32& anErrorCode)=0;
virtual PRInt32 GetToken(CScanner& aScanner,CToken*& aToken)=0;
virtual PRBool WillAddToken(CToken& aToken)=0;
virtual eParseMode GetParseMode(void) const=0;

56
htmlparser/src/nsScanner.cpp
Просмотреть файл

@ -16,6 +16,7 @@
* Reserved.
*/
//#define __INCREMENTAL 1
#include "nsScanner.h"
#include "nsIURL.h"
@ -25,6 +26,12 @@ const char* gURLRef;
const char* kBadHTMLText1="<HTML><BODY><H3>Oops...</H3>You just tried to read a non-existent document: <BR>";
const char* kBadHTMLText2="</BODY></HTML>";
#ifdef __INCREMENTAL
const int kBufsize=1;
#else
const int kBufsize=64;
#endif
/**
* default constructor
*
@ -39,9 +46,15 @@ CScanner::CScanner(nsIURL* aURL,eParseMode aMode) : mBuffer("") {
mTotalRead=0;
mParseMode=aMode;
if(aURL) {
PRInt32 error;
gURLRef=aURL->GetSpec();
#ifdef __INCREMENTAL
mStream=new fstream("c:/temp/temp.html",ios::in|ios::binary);
#else
int error;
mStream=aURL->Open(&error);
#endif
}
}
@ -53,22 +66,30 @@ CScanner::CScanner(nsIURL* aURL,eParseMode aMode) : mBuffer("") {
* @return
*/
CScanner::~CScanner() {
#ifdef __INCREMENTAL
mStream->close();
delete mStream;
mStream=0;
#else
if(mStream) {
mStream->Close();
mStream->Release();
mStream=0;
}
#endif
}
/*-------------------------------------------------------
*
/**
* Grab data from underlying stream.
*
* @update gess4/3/98
* @param
* @return
* @return error code
*/
PRInt32 CScanner::FillBuffer(PRInt32& anError) {
mBuffer.Cut(0,mBuffer.Length());
PRInt32 CScanner::FillBuffer(void) {
PRInt32 anError=0;
mBuffer.Truncate();
if(!mStream) {
//This is DEBUG code!!!!!! XXX DEBUG XXX
//If you're here, it means someone tried to load a
@ -82,16 +103,21 @@ PRInt32 CScanner::FillBuffer(PRInt32& anError) {
else return 0;
}
else {
anError=0;
PRInt32 numread=0;
char buf[64];
buf[sizeof(buf)-1]=0;
numread=mStream->Read(&anError,buf,0,sizeof(buf)-1);
char buf[kBufsize+1];
buf[kBufsize]=0;
#ifdef __INCREMENTAL
mStream->read(buf,kBufsize);
numread=mStream->gcount();
#else
numread=mStream->Read(&anError,buf,0,kBufsize);
#endif
if((0<numread) && (0==anError))
mBuffer.Append((const char*)buf,numread);
}
mTotalRead+=mBuffer.Length();
return mBuffer.Length();
return anError;
}
/**
@ -104,16 +130,13 @@ PRInt32 CScanner::FillBuffer(PRInt32& anError) {
PRBool CScanner::Eof() {
PRInt32 theError=0;
if(mOffset>=mBuffer.Length()) {
PRInt32 numread=FillBuffer(theError);
theError=FillBuffer();
mOffset=0;
}
PRBool result=PR_TRUE;
if(0==theError) {
result=PRBool(0==mBuffer.Length());
}
else {
result=PR_TRUE;
}
return result;
}
@ -163,7 +186,6 @@ PRInt32 CScanner::PutBack(PRUnichar aChar) {
}
/**
* Skip whitespace on scanner input stream
*

138
htmlparser/src/nsScanner.h
Просмотреть файл

@ -28,44 +28,150 @@
* readWhile() and SkipWhite().
*/
#ifndef SCANNER
#define SCANNER
#include "nsString.h"
#include "nsParserTypes.h"
#include "prtypes.h"
#include "nsIInputStream.h"
#include <fstream.h>
class nsIURL;
class ifstream;
class CScanner {
public:
CScanner(nsIURL* aURL,eParseMode aMode=eParseMode_navigator);
~CScanner();
CScanner(nsIURL* aURL,eParseMode aMode=eParseMode_navigator);
~CScanner();
PRInt32 GetChar(PRUnichar& ch);
PRInt32 Peek(PRUnichar& ch);
PRInt32 PutBack(PRUnichar ch);
PRInt32 SkipOver(nsString& SkipChars);
PRInt32 SkipOver(PRUnichar aSkipChar);
PRInt32 SkipPast(nsString& aSequence);
PRInt32 SkipWhite(void);
PRBool Eof(void);
/**
* retrieve next char from internal input stream
*
* @update gess 3/25/98
* @param ch is the char to accept new value
* @return error code reflecting read status
*/
PRInt32 GetChar(PRUnichar& ch);
PRInt32 ReadUntil(nsString& aString,PRUnichar aTerminal,PRBool addTerminal);
PRInt32 ReadUntil(nsString& aString,nsString& terminals,PRBool addTerminal);
PRInt32 ReadWhile(nsString& aString,nsString& validChars,PRBool addTerminal);
/**
* peek ahead to consume next char from scanner's internal
* input buffer
*
* @update gess 3/25/98
* @param ch is the char to accept new value
* @return error code reflecting read status
*/
PRInt32 Peek(PRUnichar& ch);
/**
* Push the given char back onto the scanner
*
* @update gess 3/25/98
* @param character to be pushed
* @return error code
*/
PRInt32 PutBack(PRUnichar ch);
/**
* Skip over chars as long as they're in aSkipSet
*
* @update gess 3/25/98
* @param set of chars to be skipped
* @return error code
*/
PRInt32 SkipOver(nsString& SkipChars);
/**
* Skip over chars as long as they equal given char
*
* @update gess 3/25/98
* @param char to be skipped
* @return error code
*/
PRInt32 SkipOver(PRUnichar aSkipChar);
/**
* Skip over chars as long as they're in aSequence
*
* @update gess 3/25/98
* @param contains sequence to be skipped
* @return error code
*/
PRInt32 SkipPast(nsString& aSequence);
/**
* Skip whitespace on scanner input stream
*
* @update gess 3/25/98
* @return error status
*/
PRInt32 SkipWhite(void);
/**
* Determine if the scanner has reached EOF.
* This method can also cause the buffer to be filled
* if it happens to be empty
*
* @update gess 3/25/98
* @return PR_TRUE upon eof condition
*/
PRBool Eof(void);
/**
* Consume characters until you find the terminal char
*
* @update gess 3/25/98
* @param aString receives new data from stream
* @param aTerminal contains terminating char
* @param addTerminal tells us whether to append terminal to aString
* @return error code
*/
PRInt32 ReadUntil(nsString& aString,PRUnichar aTerminal,PRBool addTerminal);
/**
* Consume characters until you find one contained in given
* terminal set.
*
* @update gess 3/25/98
* @param aString receives new data from stream
* @param aTermSet contains set of terminating chars
* @param addTerminal tells us whether to append terminal to aString
* @return error code
*/
PRInt32 ReadUntil(nsString& aString,nsString& aTermSet,PRBool addTerminal);
/**
* Consume characters while they're members of anInputSet
*
* @update gess 3/25/98
* @param aString receives new data from stream
* @param anInputSet contains valid chars
* @param addTerminal tells us whether to append terminal to aString
* @return error code
*/
PRInt32 ReadWhile(nsString& aString,nsString& anInputSet,PRBool addTerminal);
static void SelfTest();
protected:
PRInt32 FillBuffer(PRInt32& anError);
/**
* Internal method used to cause the internal buffer to
* be filled with data.
*
* @update gess4/3/98
* @param anError is the resulting error code (hopefully 0)
* @return number of new bytes read
*/
PRInt32 FillBuffer(void);
#ifdef __INCREMENTAL
fstream* mStream;
#else
nsIInputStream* mStream;
#endif
nsString mBuffer;
PRInt32 mOffset;
PRInt32 mTotalRead;

64
htmlparser/src/nsTokenizer.cpp
Просмотреть файл

@ -72,9 +72,9 @@ nsDeque& CTokenizer::GetDeque(void) {
* @param anError -- ref to error code
* @return new token or null
*/
CToken* CTokenizer::GetToken(PRInt32& anError) {
CToken* nextToken=mDelegate->GetToken(*mScanner,anError);
return nextToken;
PRInt32 CTokenizer::GetToken(CToken*& aToken) {
PRInt32 result=mDelegate->GetToken(*mScanner,aToken);
return result;
}
/**
@ -83,7 +83,7 @@ CToken* CTokenizer::GetToken(PRInt32& anError) {
* @update gess 3/25/98
* @param
* @return int containing element count
*-----------------------------------------------------*/
*/
PRInt32 CTokenizer::GetSize(void) {
return mTokenDeque.GetSize();
}
@ -98,9 +98,9 @@ PRInt32 CTokenizer::GetSize(void) {
* @param
* @return TRUE if it's ok to proceed
*/
PRBool CTokenizer::WillTokenize(){
PRBool CTokenizer::WillTokenize(PRBool aIncremental){
PRBool result=PR_TRUE;
result=mDelegate->WillTokenize();
result=mDelegate->WillTokenize(aIncremental);
return result;
}
@ -110,30 +110,56 @@ PRBool CTokenizer::WillTokenize(){
* of data.
*
* @update gess 3/25/98
* @param
* @return error code
*/
PRInt32 CTokenizer::Tokenize(void) {
CToken* nextToken;
PRInt32 result;
CToken* theToken=0;
PRInt32 result=kNoError;
if(WillTokenize()) {
if(WillTokenize(PR_FALSE)) {
do {
nextToken=GetToken(result);
if(nextToken) {
result=GetToken(theToken);
if(theToken) {
#ifdef VERBOSE_DEBUG
nextToken->DebugDumpToken(cout);
theToken->DebugDumpToken(cout);
#endif
if(mDelegate->WillAddToken(*nextToken)) {
mTokenDeque.Push(nextToken);
if(mDelegate->WillAddToken(*theToken)) {
mTokenDeque.Push(theToken);
}
}
} while(nextToken!=0);
result=DidTokenize();
} while(0!=theToken);
result=DidTokenize(PR_FALSE);
}
return result;
}
/**
* This is the primary control routine. It iteratively
* consumes tokens until an error occurs or you run out
* of data.
*
* @update gess 3/25/98
* @return error code
*/
PRInt32 CTokenizer::TokenizeAvailable(int anIteration) {
CToken* theToken=0;
PRInt32 result=kNoError;
PRBool done=(0==anIteration) ? (!WillTokenize(PR_TRUE)) : PR_FALSE;
PRBool moreData=PR_TRUE;
while((PR_FALSE==done) && (PR_TRUE==moreData)) {
result=GetToken(theToken);
if(theToken) {
if(mDelegate->WillAddToken(*theToken)) {
mTokenDeque.Push(theToken);
}
}
else done=PR_TRUE;
}
if((PR_TRUE==done) && (PR_FALSE==moreData))
DidTokenize(PR_TRUE);
return result;
}
/**
* This is the tail-end of the code sandwich for the
@ -144,8 +170,8 @@ PRInt32 CTokenizer::Tokenize(void) {
* @param
* @return TRUE if all went well
*/
PRBool CTokenizer::DidTokenize() {
PRBool result=mDelegate->DidTokenize();
PRBool CTokenizer::DidTokenize(PRBool aIncremental) {
PRBool result=mDelegate->DidTokenize(aIncremental);
#ifdef VERBOSE_DEBUG
DebugDumpTokens(cout);

102
htmlparser/src/nsTokenizer.h
Просмотреть файл

@ -50,22 +50,102 @@ class nsIURL;
class CTokenizer {
public:
CTokenizer(nsIURL* aURL,ITokenizerDelegate* aDelegate,eParseMode aMode);
~CTokenizer();
CTokenizer(nsIURL* aURL,ITokenizerDelegate* aDelegate,eParseMode aMode);
~CTokenizer();
PRInt32 Tokenize(void);
CToken* GetToken(PRInt32& anErrorCode);
PRInt32 GetSize(void);
nsDeque& GetDeque(void);
/**
* This control routine causes the entire stream to be
* tokenized. You probably want to call TokenizeAvailable()
* instead (for incremental tokenization).
*
* @update gess 3/25/98
* @return TRUE if it's ok to proceed
*/
PRInt32 Tokenize(void);
void DebugDumpSource(ostream& out);
void DebugDumpTokens(ostream& out);
static void SelfTest();
/**
* This method incrementally tokenizes as much content as
* it can get its hands on.
*
* @update gess 3/25/98
* @return TRUE if it's ok to proceed
*/
PRInt32 TokenizeAvailable(int anIteration); //your friendly incremental version
/**
* Cause the tokenizer to consume the next token, and
* return an error result.
*
* @update gess 3/25/98
* @param anError -- ref to error code
* @return new token or null
*/
PRInt32 GetToken(CToken*& aToken);
/**
* Retrieve the number of elements in the deque
*
* @update gess 3/25/98
* @return int containing element count
*/
PRInt32 GetSize(void);
/**
* Retrieve a reference to the internal token deque.
*
* @update gess 4/20/98
* @return deque reference
*/
nsDeque& GetDeque(void);
/**
* This debug routine is used to cause the tokenizer to
* iterate its token list, asking each token to dump its
* contents to the given output stream.
*
* @update gess 3/25/98
* @param
* @return
*/
void DebugDumpSource(ostream& out);
/**
* This debug routine is used to cause the tokenizer to
* iterate its token list, asking each token to dump its
* contents to the given output stream.
*
* @update gess 3/25/98
* @param
* @return
*/
void DebugDumpTokens(ostream& out);
static void SelfTest();
protected:
PRBool WillTokenize();
PRBool DidTokenize();
/**
* This is the front-end of the code sandwich for the
* tokenization process. It gets called once just before
* tokenziation begins.
*
* @update gess 3/25/98
* @param aIncremental tells us if tokenization is incremental
* @return TRUE if all went well
*/
PRBool WillTokenize(PRBool aIncremental);
/**
* This is the tail-end of the code sandwich for the
* tokenization process. It gets called once tokenziation
* has completed.
*
* @update gess 3/25/98
* @param aIncremental tells us if tokenization was incremental
* @return TRUE if all went well
*/
PRBool DidTokenize(PRBool aIncremental);
ITokenizerDelegate* mDelegate;
CScanner* mScanner;

171
parser/htmlparser/src/CNavDelegate.cpp
Просмотреть файл

@ -85,36 +85,36 @@ nsIDTD* CNavDelegate::GetDTD(void) const{
* @param
* @return
*/
CToken* CNavDelegate::ConsumeTag(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode) {
CToken* result=0;
PRInt32 CNavDelegate::ConsumeTag(PRUnichar aChar,CScanner& aScanner,CToken*& aToken) {
nsAutoString empty("");
anErrorCode=anErrorCode=aScanner.GetChar(aChar);
PRInt32 result=aScanner.GetChar(aChar);
switch(aChar) {
case kForwardSlash:
PRUnichar ch;
anErrorCode=aScanner.Peek(ch);
result=aScanner.Peek(ch);
if(nsString::IsAlpha(ch))
result=new CEndToken(empty);
else result=new CCommentToken(empty); //Special case: </ ...> is treated as a comment
aToken=new CEndToken(empty);
else aToken=new CCommentToken(empty); //Special case: </ ...> is treated as a comment
break;
case kExclamation:
result=new CCommentToken(empty);
aToken=new CCommentToken(empty);
break;
default:
if(nsString::IsAlpha(aChar))
return ConsumeStartTag(aChar,aScanner,anErrorCode);
return ConsumeStartTag(aChar,aScanner,aToken);
else if(kEOF!=aChar) {
nsAutoString temp("<");
return ConsumeText(temp,aScanner,anErrorCode);
return ConsumeText(temp,aScanner,aToken);
}
} //switch
if(result!=0) {
anErrorCode= result->Consume(aChar,aScanner); //tell new token to finish consuming text...
if(anErrorCode) {
result=0;
delete result;
if(0!=aToken) {
result= aToken->Consume(aChar,aScanner); //tell new token to finish consuming text...
if(result) {
delete aToken;
aToken=0;
}
}
return result;
@ -129,23 +129,23 @@ CToken* CNavDelegate::ConsumeTag(PRUnichar aChar,CScanner& aScanner,PRInt32& anE
* @param aScanner: see nsScanner.h
* @return
*/
void CNavDelegate::ConsumeAttributes(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode) {
PRInt32 CNavDelegate::ConsumeAttributes(PRUnichar aChar,CScanner& aScanner) {
PRBool done=PR_FALSE;
nsAutoString as("");
anErrorCode=kNoError;
while((!done) && (anErrorCode==kNoError)) {
CToken* result = new CAttributeToken(as);
if(result){
anErrorCode= result->Consume(aChar,aScanner); //tell new token to finish consuming text...
mTokenDeque.Push(result);
PRInt32 result=kNoError;
while((!done) && (result==kNoError)) {
CToken* theToken= new CAttributeToken(as);
if(theToken){
result= theToken->Consume(aChar,aScanner); //tell new token to finish consuming text...
mTokenDeque.Push(theToken);
}
aScanner.Peek(aChar);
if(aChar==kGreaterThan) { //you just ate the '>'
aScanner.GetChar(aChar); //skip the '>'
done=PR_TRUE;
}
if(aChar==kGreaterThan) { //you just ate the '>'
aScanner.GetChar(aChar); //skip the '>'
done=PR_TRUE;
}
}
return;
return result;
}
/**
@ -157,7 +157,7 @@ void CNavDelegate::ConsumeAttributes(PRUnichar aChar,CScanner& aScanner,PRInt32&
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* CNavDelegate::ConsumeContentToEndTag(const nsString& aString,PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode){
PRInt32 CNavDelegate::ConsumeContentToEndTag(const nsString& aString,PRUnichar aChar,CScanner& aScanner,CToken*& aToken){
//In the case that we just read the given tag, we should go and
//consume all the input until we find a matching end tag.
@ -165,9 +165,9 @@ CToken* CNavDelegate::ConsumeContentToEndTag(const nsString& aString,PRUnichar a
nsAutoString endTag("</");
endTag.Append(aString);
endTag.Append(">");
CSkippedContentToken* sc=new CSkippedContentToken(endTag);
anErrorCode= sc->Consume(aChar,aScanner); //tell new token to finish consuming text...
return sc;
aToken=new CSkippedContentToken(endTag);
PRInt32 result= aToken->Consume(aChar,aScanner); //tell new token to finish consuming text...
return result;
}
/**
@ -180,30 +180,32 @@ CToken* CNavDelegate::ConsumeContentToEndTag(const nsString& aString,PRUnichar a
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* CNavDelegate::ConsumeStartTag(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode) {
CStartToken* result=new CStartToken(nsAutoString(""));
if(result) {
anErrorCode= result->Consume(aChar,aScanner); //tell new token to finish consuming text...
if(result->IsAttributed()) {
ConsumeAttributes(aChar,aScanner,anErrorCode);
PRInt32 CNavDelegate::ConsumeStartTag(PRUnichar aChar,CScanner& aScanner,CToken*& aToken) {
aToken=new CStartToken(nsAutoString(""));
PRInt32 result=kNoError;
if(aToken) {
result= aToken->Consume(aChar,aScanner); //tell new token to finish consuming text...
if(((CStartToken*)aToken)->IsAttributed()) {
result=ConsumeAttributes(aChar,aScanner);
}
//now that that's over with, we have one more problem to solve.
//In the case that we just read a <SCRIPT> or <STYLE> tags, we should go and
//consume all the content itself.
nsString& str=result->GetText();
nsString& str=aToken->GetText();
CToken* skippedToken=0;
if(str.EqualsIgnoreCase("SCRIPT") ||
str.EqualsIgnoreCase("STYLE") ||
str.EqualsIgnoreCase("TITLE") ||
str.EqualsIgnoreCase("TEXTAREA")) {
CToken* sc=ConsumeContentToEndTag(str,aChar,aScanner,anErrorCode);
result=ConsumeContentToEndTag(str,aChar,aScanner,skippedToken);
if(sc){
if(skippedToken){
//now we strip the ending sequence from our new SkippedContent token...
PRInt32 slen=str.Length()+3;
nsString& skippedText=sc->GetText();
nsString& skippedText=skippedToken->GetText();
skippedText.Cut(skippedText.Length()-slen,slen);
mTokenDeque.Push(sc);
mTokenDeque.Push(skippedToken);
//In the case that we just read a given tag, we should go and
//consume all the tag content itself (and throw it all away).
@ -226,22 +228,21 @@ CToken* CNavDelegate::ConsumeStartTag(PRUnichar aChar,CScanner& aScanner,PRInt32
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* CNavDelegate::ConsumeEntity(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode) {
CToken* result = 0;
PRInt32 CNavDelegate::ConsumeEntity(PRUnichar aChar,CScanner& aScanner,CToken*& aToken) {
PRUnichar ch;
anErrorCode=aScanner.GetChar(ch);
PRInt32 result=aScanner.GetChar(ch);
if(nsString::IsAlpha(ch)) { //handle common enity references &xxx; or &#000.
result = new CEntityToken(nsAutoString(""));
anErrorCode= result->Consume(ch,aScanner); //tell new token to finish consuming text...
aToken = new CEntityToken(nsAutoString(""));
result = aToken->Consume(ch,aScanner); //tell new token to finish consuming text...
}
else if(kHashsign==ch) {
result = new CEntityToken(nsAutoString(""));
anErrorCode=result->Consume(ch,aScanner);
aToken = new CEntityToken(nsAutoString(""));
result=aToken->Consume(ch,aScanner);
}
else {
//oops, we're actually looking at plain text...
nsAutoString temp("&");
return ConsumeText(temp,aScanner,anErrorCode);
result=ConsumeText(temp,aScanner,aToken);
}
return result;
}
@ -256,9 +257,12 @@ CToken* CNavDelegate::ConsumeEntity(PRUnichar aChar,CScanner& aScanner,PRInt32&
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* CNavDelegate::ConsumeWhitespace(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode) {
CToken* result = new CWhitespaceToken(nsAutoString(""));
anErrorCode=result->Consume(aChar,aScanner);
PRInt32 CNavDelegate::ConsumeWhitespace(PRUnichar aChar,CScanner& aScanner,CToken*& aToken) {
aToken = new CWhitespaceToken(nsAutoString(""));
PRInt32 result=kNoError;
if(aToken) {
result=aToken->Consume(aChar,aScanner);
}
return result;
}
@ -272,9 +276,12 @@ CToken* CNavDelegate::ConsumeWhitespace(PRUnichar aChar,CScanner& aScanner,PRInt
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* CNavDelegate::ConsumeComment(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode){
CToken* result= new CCommentToken(nsAutoString(""));
anErrorCode=result->Consume(aChar,aScanner);
PRInt32 CNavDelegate::ConsumeComment(PRUnichar aChar,CScanner& aScanner,CToken*& aToken){
aToken = new CCommentToken(nsAutoString(""));
PRInt32 result=kNoError;
if(aToken) {
result=aToken->Consume(aChar,aScanner);
}
return result;
}
@ -288,11 +295,12 @@ CToken* CNavDelegate::ConsumeComment(PRUnichar aChar,CScanner& aScanner,PRInt32&
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* CNavDelegate::ConsumeText(const nsString& aString,CScanner& aScanner,PRInt32& anErrorCode){
CToken* result=new CTextToken(aString);
if(result) {
PRInt32 CNavDelegate::ConsumeText(const nsString& aString,CScanner& aScanner,CToken*& aToken){
PRInt32 result=kNoError;
if(aToken=new CTextToken(aString)) {
PRUnichar ch;
anErrorCode=result->Consume(ch,aScanner);
result=aToken->Consume(ch,aScanner);
}
return result;
}
@ -306,10 +314,11 @@ CToken* CNavDelegate::ConsumeText(const nsString& aString,CScanner& aScanner,PRI
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* CNavDelegate::ConsumeNewline(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode){
CToken* result=new CNewlineToken(nsAutoString(""));
if(result) {
anErrorCode=result->Consume(aChar,aScanner);
PRInt32 CNavDelegate::ConsumeNewline(PRUnichar aChar,CScanner& aScanner,CToken*& aToken){
aToken=new CNewlineToken(nsAutoString(""));
PRInt32 result=kNoError;
if(aToken) {
result=aToken->Consume(aChar,aScanner);
}
return result;
}
@ -326,36 +335,36 @@ CToken* CNavDelegate::ConsumeNewline(PRUnichar aChar,CScanner& aScanner,PRInt32&
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* CNavDelegate::GetToken(CScanner& aScanner,PRInt32& anErrorCode){
CToken* result=0;
PRInt32 CNavDelegate::GetToken(CScanner& aScanner,CToken*& aToken){
PRInt32 result=kNoError;
PRUnichar aChar;
if(mTokenDeque.GetSize()>0) {
return (CToken*)mTokenDeque.Pop();
aToken=(CToken*)mTokenDeque.Pop();
return result;
}
while(!aScanner.Eof()) {
anErrorCode=aScanner.GetChar(aChar);
aToken=0;
while(!aScanner.Eof()) {
result=aScanner.GetChar(aChar);
switch(aChar) {
case kAmpersand:
return ConsumeEntity(aChar,aScanner,anErrorCode);
return ConsumeEntity(aChar,aScanner,aToken);
case kLessThan:
return ConsumeTag(aChar,aScanner,anErrorCode);
return ConsumeTag(aChar,aScanner,aToken);
case kCR: case kLF:
return ConsumeNewline(aChar,aScanner,anErrorCode);
return ConsumeNewline(aChar,aScanner,aToken);
case kNotFound:
break;
default:
if(nsString::IsSpace(aChar))
return ConsumeWhitespace(aChar,aScanner,anErrorCode);
else
{
if(!nsString::IsSpace(aChar)) {
nsAutoString temp(aChar);
return ConsumeText(temp,aScanner,anErrorCode);
return ConsumeText(temp,aScanner,aToken);
}
else return ConsumeWhitespace(aChar,aScanner,aToken);
break;
} //switch
if(anErrorCode==kEOF)
anErrorCode=0;
if(result==kEOF)
result=0;
} //while
return result;
}
@ -395,7 +404,7 @@ PRBool CNavDelegate::WillAddToken(CToken& /*aToken*/) {
* @update gess 3/25/98
* @return TRUE if preinitialization completed successfully
*/
PRBool CNavDelegate::WillTokenize() {
PRBool CNavDelegate::WillTokenize(PRBool aIncremental) {
PRBool result=PR_TRUE;
return result;
}
@ -408,7 +417,7 @@ PRBool CNavDelegate::WillTokenize() {
* @update gess 3/25/98
* @return TRUE if preinitialization completed successfully
*/
PRBool CNavDelegate::DidTokenize() {
PRBool CNavDelegate::DidTokenize(PRBool aIncremental) {
PRBool result=PR_TRUE;
return result;
}

24
parser/htmlparser/src/CNavDelegate.h
Просмотреть файл

@ -49,11 +49,11 @@ class CNavDelegate : public ITokenizerDelegate {
CNavDelegate();
CNavDelegate(CNavDelegate& aDelegate);
virtual CToken* GetToken(CScanner& aScanner,PRInt32& anErrorCode);
virtual PRInt32 GetToken(CScanner& aScanner,CToken*& aToken);
virtual PRBool WillAddToken(CToken& aToken);
virtual PRBool WillTokenize();
virtual PRBool DidTokenize();
virtual PRBool WillTokenize(PRBool aIncremental);
virtual PRBool DidTokenize(PRBool aIncremental);
virtual eParseMode GetParseMode(void) const;
virtual nsIDTD* GetDTD(void) const;
@ -63,17 +63,17 @@ class CNavDelegate : public ITokenizerDelegate {
virtual CToken* CreateTokenOfType(eHTMLTokenTypes aType);
CToken* ConsumeTag(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeStartTag(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
void ConsumeAttributes(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeText(const nsString& aString,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeEntity(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeWhitespace(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeComment(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeNewline(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
PRInt32 ConsumeTag(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeStartTag(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeAttributes(PRUnichar aChar,CScanner& aScanner);
PRInt32 ConsumeText(const nsString& aString,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeEntity(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeWhitespace(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeComment(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeNewline(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
//the only special case method...
virtual CToken* ConsumeContentToEndTag(const nsString& aString,PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
virtual PRInt32 ConsumeContentToEndTag(const nsString& aString,PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
nsDeque mTokenDeque;

174
parser/htmlparser/src/COtherDelegate.cpp
Просмотреть файл

@ -20,8 +20,7 @@
#include "COtherDelegate.h"
#include "nsScanner.h"
#include "nsParserTypes.h"
#include "COtherDTD.h"
#include "CNavDTD.h"
// Note: We already handle the following special case conditions:
@ -66,6 +65,7 @@ eParseMode COtherDelegate::GetParseMode(void) const {
return eParseMode_unknown;
}
/**
* Cause delegate to create and return a new DTD.
*
@ -73,7 +73,7 @@ eParseMode COtherDelegate::GetParseMode(void) const {
* @return new DTD or null
*/
nsIDTD* COtherDelegate::GetDTD(void) const{
return new COtherDTD();
return new CNavDTD();
}
/**
@ -85,36 +85,36 @@ nsIDTD* COtherDelegate::GetDTD(void) const{
* @param
* @return
*/
CToken* COtherDelegate::ConsumeTag(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode) {
CToken* result=0;
PRInt32 COtherDelegate::ConsumeTag(PRUnichar aChar,CScanner& aScanner,CToken*& aToken) {
nsAutoString empty("");
anErrorCode=anErrorCode=aScanner.GetChar(aChar);
PRInt32 result=aScanner.GetChar(aChar);
switch(aChar) {
case kForwardSlash:
PRUnichar ch;
anErrorCode=aScanner.Peek(ch);
result=aScanner.Peek(ch);
if(nsString::IsAlpha(ch))
result=new CEndToken(empty);
else result=new CCommentToken(empty); //Special case: </ ...> is treated as a comment
aToken=new CEndToken(empty);
else aToken=new CCommentToken(empty); //Special case: </ ...> is treated as a comment
break;
case kExclamation:
result=new CCommentToken(empty);
aToken=new CCommentToken(empty);
break;
default:
if(nsString::IsAlpha(aChar))
return ConsumeStartTag(aChar,aScanner,anErrorCode);
return ConsumeStartTag(aChar,aScanner,aToken);
else if(kEOF!=aChar) {
nsAutoString temp("<");
return ConsumeText(temp,aScanner,anErrorCode);
return ConsumeText(temp,aScanner,aToken);
}
} //switch
if(result!=0) {
anErrorCode= result->Consume(aChar,aScanner); //tell new token to finish consuming text...
if(anErrorCode) {
result=0;
delete result;
if(0!=aToken) {
result= aToken->Consume(aChar,aScanner); //tell new token to finish consuming text...
if(result) {
delete aToken;
aToken=0;
}
}
return result;
@ -129,23 +129,23 @@ CToken* COtherDelegate::ConsumeTag(PRUnichar aChar,CScanner& aScanner,PRInt32& a
* @param aScanner: see nsScanner.h
* @return
*/
void COtherDelegate::ConsumeAttributes(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode) {
PRInt32 COtherDelegate::ConsumeAttributes(PRUnichar aChar,CScanner& aScanner) {
PRBool done=PR_FALSE;
nsAutoString as("");
anErrorCode=kNoError;
while((!done) && (anErrorCode==kNoError)) {
CToken* result = new CAttributeToken(as);
if(result){
anErrorCode= result->Consume(aChar,aScanner); //tell new token to finish consuming text...
mTokenDeque.Push(result);
PRInt32 result=kNoError;
while((!done) && (result==kNoError)) {
CToken* theToken= new CAttributeToken(as);
if(theToken){
result= theToken->Consume(aChar,aScanner); //tell new token to finish consuming text...
mTokenDeque.Push(theToken);
}
aScanner.Peek(aChar);
if(aChar==kGreaterThan) { //you just ate the '>'
aScanner.GetChar(aChar); //skip the '>'
done=PR_TRUE;
}
if(aChar==kGreaterThan) { //you just ate the '>'
aScanner.GetChar(aChar); //skip the '>'
done=PR_TRUE;
}
}
return;
return result;
}
/**
@ -157,7 +157,7 @@ void COtherDelegate::ConsumeAttributes(PRUnichar aChar,CScanner& aScanner,PRInt3
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* COtherDelegate::ConsumeContentToEndTag(const nsString& aString,PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode){
PRInt32 COtherDelegate::ConsumeContentToEndTag(const nsString& aString,PRUnichar aChar,CScanner& aScanner,CToken*& aToken){
//In the case that we just read the given tag, we should go and
//consume all the input until we find a matching end tag.
@ -165,9 +165,9 @@ CToken* COtherDelegate::ConsumeContentToEndTag(const nsString& aString,PRUnichar
nsAutoString endTag("</");
endTag.Append(aString);
endTag.Append(">");
CSkippedContentToken* sc=new CSkippedContentToken(endTag);
anErrorCode= sc->Consume(aChar,aScanner); //tell new token to finish consuming text...
return sc;
aToken=new CSkippedContentToken(endTag);
PRInt32 result= aToken->Consume(aChar,aScanner); //tell new token to finish consuming text...
return result;
}
/**
@ -180,30 +180,31 @@ CToken* COtherDelegate::ConsumeContentToEndTag(const nsString& aString,PRUnichar
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* COtherDelegate::ConsumeStartTag(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode) {
CStartToken* result=new CStartToken(nsAutoString(""));
if(result) {
anErrorCode= result->Consume(aChar,aScanner); //tell new token to finish consuming text...
if(result->IsAttributed()) {
ConsumeAttributes(aChar,aScanner,anErrorCode);
PRInt32 COtherDelegate::ConsumeStartTag(PRUnichar aChar,CScanner& aScanner,CToken*& aToken) {
aToken=new CStartToken(nsAutoString(""));
PRInt32 result=kNoError;
if(aToken) {
result= aToken->Consume(aChar,aScanner); //tell new token to finish consuming text...
if(((CStartToken*)aToken)->IsAttributed()) {
result=ConsumeAttributes(aChar,aScanner);
}
//now that that's over with, we have one more problem to solve.
//In the case that we just read a <SCRIPT> or <STYLE> tags, we should go and
//consume all the content itself.
nsString& str=result->GetText();
nsString& str=aToken->GetText();
if(str.EqualsIgnoreCase("SCRIPT") ||
str.EqualsIgnoreCase("STYLE") ||
str.EqualsIgnoreCase("TITLE") ||
str.EqualsIgnoreCase("TEXTAREA")) {
CToken* sc=ConsumeContentToEndTag(str,aChar,aScanner,anErrorCode);
result=ConsumeContentToEndTag(str,aChar,aScanner,aToken);
if(sc){
if(aToken){
//now we strip the ending sequence from our new SkippedContent token...
PRInt32 slen=str.Length()+3;
nsString& skippedText=sc->GetText();
nsString& skippedText=aToken->GetText();
skippedText.Cut(skippedText.Length()-slen,slen);
mTokenDeque.Push(sc);
mTokenDeque.Push(aToken);
//In the case that we just read a given tag, we should go and
//consume all the tag content itself (and throw it all away).
@ -226,22 +227,21 @@ CToken* COtherDelegate::ConsumeStartTag(PRUnichar aChar,CScanner& aScanner,PRInt
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* COtherDelegate::ConsumeEntity(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode) {
CToken* result = 0;
PRInt32 COtherDelegate::ConsumeEntity(PRUnichar aChar,CScanner& aScanner,CToken*& aToken) {
PRUnichar ch;
anErrorCode=aScanner.GetChar(ch);
PRInt32 result=aScanner.GetChar(ch);
if(nsString::IsAlpha(ch)) { //handle common enity references &xxx; or &#000.
result = new CEntityToken(nsAutoString(""));
anErrorCode= result->Consume(ch,aScanner); //tell new token to finish consuming text...
aToken = new CEntityToken(nsAutoString(""));
result = aToken->Consume(ch,aScanner); //tell new token to finish consuming text...
}
else if(kHashsign==ch) {
result = new CEntityToken(nsAutoString(""));
anErrorCode=result->Consume(ch,aScanner);
aToken = new CEntityToken(nsAutoString(""));
result=aToken->Consume(ch,aScanner);
}
else {
//oops, we're actually looking at plain text...
nsAutoString temp("&");
return ConsumeText(temp,aScanner,anErrorCode);
result=ConsumeText(temp,aScanner,aToken);
}
return result;
}
@ -256,9 +256,12 @@ CToken* COtherDelegate::ConsumeEntity(PRUnichar aChar,CScanner& aScanner,PRInt32
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* COtherDelegate::ConsumeWhitespace(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode) {
CToken* result = new CWhitespaceToken(nsAutoString(""));
anErrorCode=result->Consume(aChar,aScanner);
PRInt32 COtherDelegate::ConsumeWhitespace(PRUnichar aChar,CScanner& aScanner,CToken*& aToken) {
aToken = new CWhitespaceToken(nsAutoString(""));
PRInt32 result=kNoError;
if(aToken) {
result=aToken->Consume(aChar,aScanner);
}
return result;
}
@ -272,9 +275,12 @@ CToken* COtherDelegate::ConsumeWhitespace(PRUnichar aChar,CScanner& aScanner,PRI
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* COtherDelegate::ConsumeComment(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode){
CToken* result= new CCommentToken(nsAutoString(""));
anErrorCode=result->Consume(aChar,aScanner);
PRInt32 COtherDelegate::ConsumeComment(PRUnichar aChar,CScanner& aScanner,CToken*& aToken){
aToken = new CCommentToken(nsAutoString(""));
PRInt32 result=kNoError;
if(aToken) {
result=aToken->Consume(aChar,aScanner);
}
return result;
}
@ -288,11 +294,12 @@ CToken* COtherDelegate::ConsumeComment(PRUnichar aChar,CScanner& aScanner,PRInt3
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* COtherDelegate::ConsumeText(const nsString& aString,CScanner& aScanner,PRInt32& anErrorCode){
CToken* result=new CTextToken(aString);
if(result) {
PRInt32 COtherDelegate::ConsumeText(const nsString& aString,CScanner& aScanner,CToken*& aToken){
aToken=new CTextToken(aString);
PRInt32 result=kNoError;
if(aToken) {
PRUnichar ch;
anErrorCode=result->Consume(ch,aScanner);
result=aToken->Consume(ch,aScanner);
}
return result;
}
@ -306,10 +313,11 @@ CToken* COtherDelegate::ConsumeText(const nsString& aString,CScanner& aScanner,P
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* COtherDelegate::ConsumeNewline(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode){
CToken* result=new CNewlineToken(nsAutoString(""));
if(result) {
anErrorCode=result->Consume(aChar,aScanner);
PRInt32 COtherDelegate::ConsumeNewline(PRUnichar aChar,CScanner& aScanner,CToken*& aToken){
aToken=new CNewlineToken(nsAutoString(""));
PRInt32 result=kNoError;
if(aToken) {
result=aToken->Consume(aChar,aScanner);
}
return result;
}
@ -326,36 +334,36 @@ CToken* COtherDelegate::ConsumeNewline(PRUnichar aChar,CScanner& aScanner,PRInt3
* @param anErrorCode: arg that will hold error condition
* @return new token or null
*/
CToken* COtherDelegate::GetToken(CScanner& aScanner,PRInt32& anErrorCode){
CToken* result=0;
PRInt32 COtherDelegate::GetToken(CScanner& aScanner,CToken*& aToken){
PRInt32 result=kNoError;
PRUnichar aChar;
if(mTokenDeque.GetSize()>0) {
return (CToken*)mTokenDeque.Pop();
aToken=(CToken*)mTokenDeque.Pop();
return result;
}
while(!aScanner.Eof()) {
anErrorCode=aScanner.GetChar(aChar);
while(!aScanner.Eof()) {
result=aScanner.GetChar(aChar);
switch(aChar) {
case kAmpersand:
return ConsumeEntity(aChar,aScanner,anErrorCode);
return ConsumeEntity(aChar,aScanner,aToken);
case kLessThan:
return ConsumeTag(aChar,aScanner,anErrorCode);
return ConsumeTag(aChar,aScanner,aToken);
case kCR: case kLF:
return ConsumeNewline(aChar,aScanner,anErrorCode);
return ConsumeNewline(aChar,aScanner,aToken);
case kNotFound:
break;
default:
if(nsString::IsSpace(aChar))
return ConsumeWhitespace(aChar,aScanner,anErrorCode);
else
{
if(!nsString::IsSpace(aChar)) {
nsAutoString temp(aChar);
return ConsumeText(temp,aScanner,anErrorCode);
return ConsumeText(temp,aScanner,aToken);
}
else return ConsumeWhitespace(aChar,aScanner,aToken);
break;
} //switch
if(anErrorCode==kEOF)
anErrorCode=0;
if(result==kEOF)
result=0;
} //while
return result;
}
@ -395,7 +403,7 @@ PRBool COtherDelegate::WillAddToken(CToken& /*aToken*/) {
* @update gess 3/25/98
* @return TRUE if preinitialization completed successfully
*/
PRBool COtherDelegate::WillTokenize() {
PRBool COtherDelegate::WillTokenize(PRBool aIncremental) {
PRBool result=PR_TRUE;
return result;
}
@ -408,7 +416,7 @@ PRBool COtherDelegate::WillTokenize() {
* @update gess 3/25/98
* @return TRUE if preinitialization completed successfully
*/
PRBool COtherDelegate::DidTokenize() {
PRBool COtherDelegate::DidTokenize(PRBool aIncremental) {
PRBool result=PR_TRUE;
return result;
}

25
parser/htmlparser/src/COtherDelegate.h
Просмотреть файл

@ -49,11 +49,11 @@ class COtherDelegate : public ITokenizerDelegate {
COtherDelegate();
COtherDelegate(COtherDelegate& aDelegate);
virtual CToken* GetToken(CScanner& aScanner,PRInt32& anErrorCode);
virtual PRInt32 GetToken(CScanner& aScanner,CToken*& aToken);
virtual PRBool WillAddToken(CToken& aToken);
virtual PRBool WillTokenize();
virtual PRBool DidTokenize();
virtual PRBool WillTokenize(PRBool aIncremental);
virtual PRBool DidTokenize(PRBool aIncremental);
virtual eParseMode GetParseMode(void) const;
virtual nsIDTD* GetDTD(void) const;
@ -63,17 +63,17 @@ class COtherDelegate : public ITokenizerDelegate {
virtual CToken* CreateTokenOfType(eHTMLTokenTypes aType);
CToken* ConsumeTag(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeStartTag(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
void ConsumeAttributes(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeText(const nsString& aString,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeEntity(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeWhitespace(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeComment(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
CToken* ConsumeNewline(PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
PRInt32 ConsumeTag(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeStartTag(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeAttributes(PRUnichar aChar,CScanner& aScanner);
PRInt32 ConsumeText(const nsString& aString,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeEntity(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeWhitespace(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeComment(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
PRInt32 ConsumeNewline(PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
//the only special case method...
virtual CToken* ConsumeContentToEndTag(const nsString& aString,PRUnichar aChar,CScanner& aScanner,PRInt32& anErrorCode);
virtual PRInt32 ConsumeContentToEndTag(const nsString& aString,PRUnichar aChar,CScanner& aScanner,CToken*& aToken);
nsDeque mTokenDeque;
@ -82,4 +82,3 @@ class COtherDelegate : public ITokenizerDelegate {
#endif

10
parser/htmlparser/src/nsHTMLParser.cpp
Просмотреть файл

@ -16,6 +16,8 @@
* Reserved.
*/
//#define __INCREMENTAL 1
#include "nsHTMLParser.h"
#include "nsHTMLContentSink.h"
#include "nsTokenizer.h"
@ -513,7 +515,15 @@ PRBool nsHTMLParser::Parse(nsIURL* aURL,eParseMode aMode){
if(mDTD)
mDTD->SetParser(this);
mTokenizer=new CTokenizer(aURL, theDelegate, mParseMode);
#ifdef __INCREMENTAL
int iter=-1;
for(;;){
mTokenizer->TokenizeAvailable(++iter);
}
#else
mTokenizer->Tokenize();
#endif
result=IterateTokens();
}
return result;

2
parser/htmlparser/src/nsHTMLTokens.cpp
Просмотреть файл

@ -560,7 +560,7 @@ CCommentToken::CCommentToken(const nsString& aName) : CHTMLToken(aName) {
PRInt32 CCommentToken::Consume(PRUnichar aChar, CScanner& aScanner) {
PRUnichar ch,ch2;
PRInt32 result;
PRInt32 result=kNoError;
static nsAutoString terminals(">");

6
parser/htmlparser/src/nsITokenizerDelegate.h
Просмотреть файл

@ -48,10 +48,10 @@ class CToken;
class ITokenizerDelegate {
public:
virtual PRBool WillTokenize()=0;
virtual PRBool DidTokenize()=0;
virtual PRBool WillTokenize(PRBool aIncremental)=0;
virtual PRBool DidTokenize(PRBool aIncremental)=0;
virtual CToken* GetToken(CScanner& aScanner,PRInt32& anErrorCode)=0;
virtual PRInt32 GetToken(CScanner& aScanner,CToken*& aToken)=0;
virtual PRBool WillAddToken(CToken& aToken)=0;
virtual eParseMode GetParseMode(void) const=0;

56
parser/htmlparser/src/nsScanner.cpp
Просмотреть файл

@ -16,6 +16,7 @@
* Reserved.
*/
//#define __INCREMENTAL 1
#include "nsScanner.h"
#include "nsIURL.h"
@ -25,6 +26,12 @@ const char* gURLRef;
const char* kBadHTMLText1="<HTML><BODY><H3>Oops...</H3>You just tried to read a non-existent document: <BR>";
const char* kBadHTMLText2="</BODY></HTML>";
#ifdef __INCREMENTAL
const int kBufsize=1;
#else
const int kBufsize=64;
#endif
/**
* default constructor
*
@ -39,9 +46,15 @@ CScanner::CScanner(nsIURL* aURL,eParseMode aMode) : mBuffer("") {
mTotalRead=0;
mParseMode=aMode;
if(aURL) {
PRInt32 error;
gURLRef=aURL->GetSpec();
#ifdef __INCREMENTAL
mStream=new fstream("c:/temp/temp.html",ios::in|ios::binary);
#else
int error;
mStream=aURL->Open(&error);
#endif
}
}
@ -53,22 +66,30 @@ CScanner::CScanner(nsIURL* aURL,eParseMode aMode) : mBuffer("") {
* @return
*/
CScanner::~CScanner() {
#ifdef __INCREMENTAL
mStream->close();
delete mStream;
mStream=0;
#else
if(mStream) {
mStream->Close();
mStream->Release();
mStream=0;
}
#endif
}
/*-------------------------------------------------------
*
/**
* Grab data from underlying stream.
*
* @update gess4/3/98
* @param
* @return
* @return error code
*/
PRInt32 CScanner::FillBuffer(PRInt32& anError) {
mBuffer.Cut(0,mBuffer.Length());
PRInt32 CScanner::FillBuffer(void) {
PRInt32 anError=0;
mBuffer.Truncate();
if(!mStream) {
//This is DEBUG code!!!!!! XXX DEBUG XXX
//If you're here, it means someone tried to load a
@ -82,16 +103,21 @@ PRInt32 CScanner::FillBuffer(PRInt32& anError) {
else return 0;
}
else {
anError=0;
PRInt32 numread=0;
char buf[64];
buf[sizeof(buf)-1]=0;
numread=mStream->Read(&anError,buf,0,sizeof(buf)-1);
char buf[kBufsize+1];
buf[kBufsize]=0;
#ifdef __INCREMENTAL
mStream->read(buf,kBufsize);
numread=mStream->gcount();
#else
numread=mStream->Read(&anError,buf,0,kBufsize);
#endif
if((0<numread) && (0==anError))
mBuffer.Append((const char*)buf,numread);
}
mTotalRead+=mBuffer.Length();
return mBuffer.Length();
return anError;
}
/**
@ -104,16 +130,13 @@ PRInt32 CScanner::FillBuffer(PRInt32& anError) {
PRBool CScanner::Eof() {
PRInt32 theError=0;
if(mOffset>=mBuffer.Length()) {
PRInt32 numread=FillBuffer(theError);
theError=FillBuffer();
mOffset=0;
}
PRBool result=PR_TRUE;
if(0==theError) {
result=PRBool(0==mBuffer.Length());
}
else {
result=PR_TRUE;
}
return result;
}
@ -163,7 +186,6 @@ PRInt32 CScanner::PutBack(PRUnichar aChar) {
}
/**
* Skip whitespace on scanner input stream
*

138
parser/htmlparser/src/nsScanner.h
Просмотреть файл

@ -28,44 +28,150 @@
* readWhile() and SkipWhite().
*/
#ifndef SCANNER
#define SCANNER
#include "nsString.h"
#include "nsParserTypes.h"
#include "prtypes.h"
#include "nsIInputStream.h"
#include <fstream.h>
class nsIURL;
class ifstream;
class CScanner {
public:
CScanner(nsIURL* aURL,eParseMode aMode=eParseMode_navigator);
~CScanner();
CScanner(nsIURL* aURL,eParseMode aMode=eParseMode_navigator);
~CScanner();
PRInt32 GetChar(PRUnichar& ch);
PRInt32 Peek(PRUnichar& ch);
PRInt32 PutBack(PRUnichar ch);
PRInt32 SkipOver(nsString& SkipChars);
PRInt32 SkipOver(PRUnichar aSkipChar);
PRInt32 SkipPast(nsString& aSequence);
PRInt32 SkipWhite(void);
PRBool Eof(void);
/**
* retrieve next char from internal input stream
*
* @update gess 3/25/98
* @param ch is the char to accept new value
* @return error code reflecting read status
*/
PRInt32 GetChar(PRUnichar& ch);
PRInt32 ReadUntil(nsString& aString,PRUnichar aTerminal,PRBool addTerminal);
PRInt32 ReadUntil(nsString& aString,nsString& terminals,PRBool addTerminal);
PRInt32 ReadWhile(nsString& aString,nsString& validChars,PRBool addTerminal);
/**
* peek ahead to consume next char from scanner's internal
* input buffer
*
* @update gess 3/25/98
* @param ch is the char to accept new value
* @return error code reflecting read status
*/
PRInt32 Peek(PRUnichar& ch);
/**
* Push the given char back onto the scanner
*
* @update gess 3/25/98
* @param character to be pushed
* @return error code
*/
PRInt32 PutBack(PRUnichar ch);
/**
* Skip over chars as long as they're in aSkipSet
*
* @update gess 3/25/98
* @param set of chars to be skipped
* @return error code
*/
PRInt32 SkipOver(nsString& SkipChars);
/**
* Skip over chars as long as they equal given char
*
* @update gess 3/25/98
* @param char to be skipped
* @return error code
*/
PRInt32 SkipOver(PRUnichar aSkipChar);
/**
* Skip over chars as long as they're in aSequence
*
* @update gess 3/25/98
* @param contains sequence to be skipped
* @return error code
*/
PRInt32 SkipPast(nsString& aSequence);
/**
* Skip whitespace on scanner input stream
*
* @update gess 3/25/98
* @return error status
*/
PRInt32 SkipWhite(void);
/**
* Determine if the scanner has reached EOF.
* This method can also cause the buffer to be filled
* if it happens to be empty
*
* @update gess 3/25/98
* @return PR_TRUE upon eof condition
*/
PRBool Eof(void);
/**
* Consume characters until you find the terminal char
*
* @update gess 3/25/98
* @param aString receives new data from stream
* @param aTerminal contains terminating char
* @param addTerminal tells us whether to append terminal to aString
* @return error code
*/
PRInt32 ReadUntil(nsString& aString,PRUnichar aTerminal,PRBool addTerminal);
/**
* Consume characters until you find one contained in given
* terminal set.
*
* @update gess 3/25/98
* @param aString receives new data from stream
* @param aTermSet contains set of terminating chars
* @param addTerminal tells us whether to append terminal to aString
* @return error code
*/
PRInt32 ReadUntil(nsString& aString,nsString& aTermSet,PRBool addTerminal);
/**
* Consume characters while they're members of anInputSet
*
* @update gess 3/25/98
* @param aString receives new data from stream
* @param anInputSet contains valid chars
* @param addTerminal tells us whether to append terminal to aString
* @return error code
*/
PRInt32 ReadWhile(nsString& aString,nsString& anInputSet,PRBool addTerminal);
static void SelfTest();
protected:
PRInt32 FillBuffer(PRInt32& anError);
/**
* Internal method used to cause the internal buffer to
* be filled with data.
*
* @update gess4/3/98
* @param anError is the resulting error code (hopefully 0)
* @return number of new bytes read
*/
PRInt32 FillBuffer(void);
#ifdef __INCREMENTAL
fstream* mStream;
#else
nsIInputStream* mStream;
#endif
nsString mBuffer;
PRInt32 mOffset;
PRInt32 mTotalRead;

64
parser/htmlparser/src/nsTokenizer.cpp
Просмотреть файл

@ -72,9 +72,9 @@ nsDeque& CTokenizer::GetDeque(void) {
* @param anError -- ref to error code
* @return new token or null
*/
CToken* CTokenizer::GetToken(PRInt32& anError) {
CToken* nextToken=mDelegate->GetToken(*mScanner,anError);
return nextToken;
PRInt32 CTokenizer::GetToken(CToken*& aToken) {
PRInt32 result=mDelegate->GetToken(*mScanner,aToken);
return result;
}
/**
@ -83,7 +83,7 @@ CToken* CTokenizer::GetToken(PRInt32& anError) {
* @update gess 3/25/98
* @param
* @return int containing element count
*-----------------------------------------------------*/
*/
PRInt32 CTokenizer::GetSize(void) {
return mTokenDeque.GetSize();
}
@ -98,9 +98,9 @@ PRInt32 CTokenizer::GetSize(void) {
* @param
* @return TRUE if it's ok to proceed
*/
PRBool CTokenizer::WillTokenize(){
PRBool CTokenizer::WillTokenize(PRBool aIncremental){
PRBool result=PR_TRUE;
result=mDelegate->WillTokenize();
result=mDelegate->WillTokenize(aIncremental);
return result;
}
@ -110,30 +110,56 @@ PRBool CTokenizer::WillTokenize(){
* of data.
*
* @update gess 3/25/98
* @param
* @return error code
*/
PRInt32 CTokenizer::Tokenize(void) {
CToken* nextToken;
PRInt32 result;
CToken* theToken=0;
PRInt32 result=kNoError;
if(WillTokenize()) {
if(WillTokenize(PR_FALSE)) {
do {
nextToken=GetToken(result);
if(nextToken) {
result=GetToken(theToken);
if(theToken) {
#ifdef VERBOSE_DEBUG
nextToken->DebugDumpToken(cout);
theToken->DebugDumpToken(cout);
#endif
if(mDelegate->WillAddToken(*nextToken)) {
mTokenDeque.Push(nextToken);
if(mDelegate->WillAddToken(*theToken)) {
mTokenDeque.Push(theToken);
}
}
} while(nextToken!=0);
result=DidTokenize();
} while(0!=theToken);
result=DidTokenize(PR_FALSE);
}
return result;
}
/**
* This is the primary control routine. It iteratively
* consumes tokens until an error occurs or you run out
* of data.
*
* @update gess 3/25/98
* @return error code
*/
PRInt32 CTokenizer::TokenizeAvailable(int anIteration) {
CToken* theToken=0;
PRInt32 result=kNoError;
PRBool done=(0==anIteration) ? (!WillTokenize(PR_TRUE)) : PR_FALSE;
PRBool moreData=PR_TRUE;
while((PR_FALSE==done) && (PR_TRUE==moreData)) {
result=GetToken(theToken);
if(theToken) {
if(mDelegate->WillAddToken(*theToken)) {
mTokenDeque.Push(theToken);
}
}
else done=PR_TRUE;
}
if((PR_TRUE==done) && (PR_FALSE==moreData))
DidTokenize(PR_TRUE);
return result;
}
/**
* This is the tail-end of the code sandwich for the
@ -144,8 +170,8 @@ PRInt32 CTokenizer::Tokenize(void) {
* @param
* @return TRUE if all went well
*/
PRBool CTokenizer::DidTokenize() {
PRBool result=mDelegate->DidTokenize();
PRBool CTokenizer::DidTokenize(PRBool aIncremental) {
PRBool result=mDelegate->DidTokenize(aIncremental);
#ifdef VERBOSE_DEBUG
DebugDumpTokens(cout);

102
parser/htmlparser/src/nsTokenizer.h
Просмотреть файл

@ -50,22 +50,102 @@ class nsIURL;
class CTokenizer {
public:
CTokenizer(nsIURL* aURL,ITokenizerDelegate* aDelegate,eParseMode aMode);
~CTokenizer();
CTokenizer(nsIURL* aURL,ITokenizerDelegate* aDelegate,eParseMode aMode);
~CTokenizer();
PRInt32 Tokenize(void);
CToken* GetToken(PRInt32& anErrorCode);
PRInt32 GetSize(void);
nsDeque& GetDeque(void);
/**
* This control routine causes the entire stream to be
* tokenized. You probably want to call TokenizeAvailable()
* instead (for incremental tokenization).
*
* @update gess 3/25/98
* @return TRUE if it's ok to proceed
*/
PRInt32 Tokenize(void);
void DebugDumpSource(ostream& out);
void DebugDumpTokens(ostream& out);
static void SelfTest();
/**
* This method incrementally tokenizes as much content as
* it can get its hands on.
*
* @update gess 3/25/98
* @return TRUE if it's ok to proceed
*/
PRInt32 TokenizeAvailable(int anIteration); //your friendly incremental version
/**
* Cause the tokenizer to consume the next token, and
* return an error result.
*
* @update gess 3/25/98
* @param anError -- ref to error code
* @return new token or null
*/
PRInt32 GetToken(CToken*& aToken);
/**
* Retrieve the number of elements in the deque
*
* @update gess 3/25/98
* @return int containing element count
*/
PRInt32 GetSize(void);
/**
* Retrieve a reference to the internal token deque.
*
* @update gess 4/20/98
* @return deque reference
*/
nsDeque& GetDeque(void);
/**
* This debug routine is used to cause the tokenizer to
* iterate its token list, asking each token to dump its
* contents to the given output stream.
*
* @update gess 3/25/98
* @param
* @return
*/
void DebugDumpSource(ostream& out);
/**
* This debug routine is used to cause the tokenizer to
* iterate its token list, asking each token to dump its
* contents to the given output stream.
*
* @update gess 3/25/98
* @param
* @return
*/
void DebugDumpTokens(ostream& out);
static void SelfTest();
protected:
PRBool WillTokenize();
PRBool DidTokenize();
/**
* This is the front-end of the code sandwich for the
* tokenization process. It gets called once just before
* tokenziation begins.
*
* @update gess 3/25/98
* @param aIncremental tells us if tokenization is incremental
* @return TRUE if all went well
*/
PRBool WillTokenize(PRBool aIncremental);
/**
* This is the tail-end of the code sandwich for the
* tokenization process. It gets called once tokenziation
* has completed.
*
* @update gess 3/25/98
* @param aIncremental tells us if tokenization was incremental
* @return TRUE if all went well
*/
PRBool DidTokenize(PRBool aIncremental);
ITokenizerDelegate* mDelegate;
CScanner* mScanner;