diff --git a/extensions/universalchardet/src/base/nsEscCharsetProber.cpp b/extensions/universalchardet/src/base/nsEscCharsetProber.cpp index 55366095563..464c75346a2 100644 --- a/extensions/universalchardet/src/base/nsEscCharsetProber.cpp +++ b/extensions/universalchardet/src/base/nsEscCharsetProber.cpp @@ -37,13 +37,21 @@ #include "nsEscCharsetProber.h" +#include "nsUniversalDetector.h" -nsEscCharSetProber::nsEscCharSetProber(void) +nsEscCharSetProber::nsEscCharSetProber(PRUint32 aLanguageFilter) { - mCodingSM[0] = new nsCodingStateMachine(&HZSMModel); - mCodingSM[1] = new nsCodingStateMachine(&ISO2022CNSMModel); - mCodingSM[2] = new nsCodingStateMachine(&ISO2022JPSMModel); - mCodingSM[3] = new nsCodingStateMachine(&ISO2022KRSMModel); + for (PRUint32 i = 0; i < NUM_OF_ESC_CHARSETS; i++) + mCodingSM[i] = nsnull; + if (aLanguageFilter & NS_FILTER_CHINESE_SIMPLIFIED) + { + mCodingSM[0] = new nsCodingStateMachine(&HZSMModel); + mCodingSM[1] = new nsCodingStateMachine(&ISO2022CNSMModel); + } + if (aLanguageFilter & NS_FILTER_JAPANESE) + mCodingSM[2] = new nsCodingStateMachine(&ISO2022JPSMModel); + if (aLanguageFilter & NS_FILTER_KOREAN) + mCodingSM[3] = new nsCodingStateMachine(&ISO2022KRSMModel); mActiveSM = NUM_OF_ESC_CHARSETS; mState = eDetecting; mDetectedCharset = nsnull; @@ -59,7 +67,8 @@ void nsEscCharSetProber::Reset(void) { mState = eDetecting; for (PRUint32 i = 0; i < NUM_OF_ESC_CHARSETS; i++) - mCodingSM[i]->Reset(); + if (mCodingSM[i]) + mCodingSM[i]->Reset(); mActiveSM = NUM_OF_ESC_CHARSETS; mDetectedCharset = nsnull; } @@ -74,12 +83,15 @@ nsProbingState nsEscCharSetProber::HandleData(const char* aBuf, PRUint32 aLen) { for (j = mActiveSM-1; j>= 0; j--) { - codingState = mCodingSM[j]->NextState(aBuf[i]); - if (codingState == eItsMe) + if (mCodingSM[j]) { - mState = eFoundIt; - mDetectedCharset = mCodingSM[j]->GetCodingStateMachine(); - return mState; + codingState = mCodingSM[j]->NextState(aBuf[i]); + if (codingState == eItsMe) + { + mState = eFoundIt; + mDetectedCharset = mCodingSM[j]->GetCodingStateMachine(); + return mState; + } } } } diff --git a/extensions/universalchardet/src/base/nsEscCharsetProber.h b/extensions/universalchardet/src/base/nsEscCharsetProber.h index c08f4420e46..4b648e012c8 100644 --- a/extensions/universalchardet/src/base/nsEscCharsetProber.h +++ b/extensions/universalchardet/src/base/nsEscCharsetProber.h @@ -45,7 +45,7 @@ class nsEscCharSetProber: public nsCharSetProber { public: - nsEscCharSetProber(void); + nsEscCharSetProber(PRUint32 aLanguageFilter); virtual ~nsEscCharSetProber(void); nsProbingState HandleData(const char* aBuf, PRUint32 aLen); const char* GetCharSetName() {return mDetectedCharset;} diff --git a/extensions/universalchardet/src/base/nsMBCSGroupProber.cpp b/extensions/universalchardet/src/base/nsMBCSGroupProber.cpp index 0141814e0b7..ffb80cfe5a9 100644 --- a/extensions/universalchardet/src/base/nsMBCSGroupProber.cpp +++ b/extensions/universalchardet/src/base/nsMBCSGroupProber.cpp @@ -39,6 +39,7 @@ #include #include "nsMBCSGroupProber.h" +#include "nsUniversalDetector.h" #if defined(DEBUG_chardet) || defined(DEBUG_jgmyers) const char *ProberName[] = @@ -54,15 +55,26 @@ const char *ProberName[] = #endif -nsMBCSGroupProber::nsMBCSGroupProber() +nsMBCSGroupProber::nsMBCSGroupProber(PRUint32 aLanguageFilter) { + for (PRUint32 i = 0; i < NUM_OF_PROBERS; i++) + mProbers[i] = nsnull; + mProbers[0] = new nsUTF8Prober(); - mProbers[1] = new nsSJISProber(); - mProbers[2] = new nsEUCJPProber(); - mProbers[3] = new nsGB18030Prober(); - mProbers[4] = new nsEUCKRProber(); - mProbers[5] = new nsBig5Prober(); - mProbers[6] = new nsEUCTWProber(); + if (aLanguageFilter & NS_FILTER_JAPANESE) + { + mProbers[1] = new nsSJISProber(); + mProbers[2] = new nsEUCJPProber(); + } + if (aLanguageFilter & NS_FILTER_CHINESE_SIMPLIFIED) + mProbers[3] = new nsGB18030Prober(); + if (aLanguageFilter & NS_FILTER_KOREAN) + mProbers[4] = new nsEUCKRProber(); + if (aLanguageFilter & NS_FILTER_CHINESE_TRADITIONAL) + { + mProbers[5] = new nsBig5Prober(); + mProbers[6] = new nsEUCTWProber(); + } Reset(); } diff --git a/extensions/universalchardet/src/base/nsMBCSGroupProber.h b/extensions/universalchardet/src/base/nsMBCSGroupProber.h index 7c069f2bddc..c4e99649713 100644 --- a/extensions/universalchardet/src/base/nsMBCSGroupProber.h +++ b/extensions/universalchardet/src/base/nsMBCSGroupProber.h @@ -51,7 +51,7 @@ class nsMBCSGroupProber: public nsCharSetProber { public: - nsMBCSGroupProber(); + nsMBCSGroupProber(PRUint32 aLanguageFilter); virtual ~nsMBCSGroupProber(); nsProbingState HandleData(const char* aBuf, PRUint32 aLen); const char* GetCharSetName(); diff --git a/extensions/universalchardet/src/base/nsUniversalDetector.cpp b/extensions/universalchardet/src/base/nsUniversalDetector.cpp index 98fd644e485..5205520bbf9 100644 --- a/extensions/universalchardet/src/base/nsUniversalDetector.cpp +++ b/extensions/universalchardet/src/base/nsUniversalDetector.cpp @@ -45,7 +45,7 @@ #include "nsEscCharsetProber.h" #include "nsLatin1Prober.h" -nsUniversalDetector::nsUniversalDetector() +nsUniversalDetector::nsUniversalDetector(PRUint32 aLanguageFilter) { mDone = PR_FALSE; mBestGuess = -1; //illegal value as signal @@ -57,6 +57,7 @@ nsUniversalDetector::nsUniversalDetector() mGotData = PR_FALSE; mInputState = ePureAscii; mLastChar = '\0'; + mLanguageFilter = aLanguageFilter; PRUint32 i; for (i = 0; i < NUM_OF_CHARSET_PROBERS; i++) @@ -171,16 +172,21 @@ nsresult nsUniversalDetector::HandleData(const char* aBuf, PRUint32 aLen) //start multibyte and singlebyte charset prober if (nsnull == mCharSetProbers[0]) - mCharSetProbers[0] = new nsMBCSGroupProber; - if (nsnull == mCharSetProbers[1]) - mCharSetProbers[1] = new nsSBCSGroupProber; - if (nsnull == mCharSetProbers[2]) - mCharSetProbers[2] = new nsLatin1Prober; - - if ((nsnull == mCharSetProbers[0]) || - (nsnull == mCharSetProbers[1]) || - (nsnull == mCharSetProbers[2])) + { + mCharSetProbers[0] = new nsMBCSGroupProber(mLanguageFilter); + if (nsnull == mCharSetProbers[0]) return NS_ERROR_OUT_OF_MEMORY; + } + if (nsnull == mCharSetProbers[1] && + (mLanguageFilter & NS_FILTER_NON_CJK)) + { + mCharSetProbers[1] = new nsSBCSGroupProber; + if (nsnull == mCharSetProbers[1]) + return NS_ERROR_OUT_OF_MEMORY; + } + mCharSetProbers[2] = new nsLatin1Prober; + if (nsnull == mCharSetProbers[2]) + return NS_ERROR_OUT_OF_MEMORY; } } else @@ -201,7 +207,7 @@ nsresult nsUniversalDetector::HandleData(const char* aBuf, PRUint32 aLen) { case eEscAscii: if (nsnull == mEscCharSetProber) { - mEscCharSetProber = new nsEscCharSetProber; + mEscCharSetProber = new nsEscCharSetProber(mLanguageFilter); if (nsnull == mEscCharSetProber) return NS_ERROR_OUT_OF_MEMORY; } @@ -215,12 +221,15 @@ nsresult nsUniversalDetector::HandleData(const char* aBuf, PRUint32 aLen) case eHighbyte: for (i = 0; i < NUM_OF_CHARSET_PROBERS; i++) { - st = mCharSetProbers[i]->HandleData(aBuf, aLen); - if (st == eFoundIt) + if (mCharSetProbers[i]) { - mDone = PR_TRUE; - mDetectedCharset = mCharSetProbers[i]->GetCharSetName(); - return NS_OK; + st = mCharSetProbers[i]->HandleData(aBuf, aLen); + if (st == eFoundIt) + { + mDone = PR_TRUE; + mDetectedCharset = mCharSetProbers[i]->GetCharSetName(); + return NS_OK; + } } } break; @@ -259,11 +268,14 @@ void nsUniversalDetector::DataEnd() for (PRInt32 i = 0; i < NUM_OF_CHARSET_PROBERS; i++) { - proberConfidence = mCharSetProbers[i]->GetConfidence(); - if (proberConfidence > maxProberConfidence) + if (mCharSetProbers[i]) { - maxProberConfidence = proberConfidence; - maxProber = i; + proberConfidence = mCharSetProbers[i]->GetConfidence(); + if (proberConfidence > maxProberConfidence) + { + maxProberConfidence = proberConfidence; + maxProber = i; + } } } //do not report anything because we are not confident of it, that's in fact a negative answer diff --git a/extensions/universalchardet/src/base/nsUniversalDetector.h b/extensions/universalchardet/src/base/nsUniversalDetector.h index 36f3fa03fb5..525f7220fe5 100644 --- a/extensions/universalchardet/src/base/nsUniversalDetector.h +++ b/extensions/universalchardet/src/base/nsUniversalDetector.h @@ -48,9 +48,22 @@ typedef enum { eHighbyte = 2 } nsInputState; +#define NS_FILTER_CHINESE_SIMPLIFIED 0x01 +#define NS_FILTER_CHINESE_TRADITIONAL 0x02 +#define NS_FILTER_JAPANESE 0x04 +#define NS_FILTER_KOREAN 0x08 +#define NS_FILTER_NON_CJK 0x10 +#define NS_FILTER_ALL 0x1F +#define NS_FILTER_CHINESE (NS_FILTER_CHINESE_SIMPLIFIED | \ + NS_FILTER_CHINESE_TRADITIONAL) +#define NS_FILTER_CJK (NS_FILTER_CHINESE_SIMPLIFIED | \ + NS_FILTER_CHINESE_TRADITIONAL | \ + NS_FILTER_JAPANESE | \ + NS_FILTER_KOREAN) + class nsUniversalDetector { public: - nsUniversalDetector(); + nsUniversalDetector(PRUint32 aLanguageFilter); virtual ~nsUniversalDetector(); virtual nsresult HandleData(const char* aBuf, PRUint32 aLen); virtual void DataEnd(void); @@ -66,6 +79,7 @@ protected: char mLastChar; const char * mDetectedCharset; PRInt32 mBestGuess; + PRUint32 mLanguageFilter; nsCharSetProber *mCharSetProbers[NUM_OF_CHARSET_PROBERS]; nsCharSetProber *mEscCharSetProber; diff --git a/extensions/universalchardet/src/xpcom/nsUdetXPCOMWrapper.cpp b/extensions/universalchardet/src/xpcom/nsUdetXPCOMWrapper.cpp index ef81e47a9ae..838904b050d 100644 --- a/extensions/universalchardet/src/xpcom/nsUdetXPCOMWrapper.cpp +++ b/extensions/universalchardet/src/xpcom/nsUdetXPCOMWrapper.cpp @@ -54,19 +54,20 @@ static NS_DEFINE_CID(kUniversalDetectorCID, NS_UNIVERSAL_DETECTOR_CID); static NS_DEFINE_CID(kUniversalStringDetectorCID, NS_UNIVERSAL_STRING_DETECTOR_CID); //--------------------------------------------------------------------- -nsUniversalXPCOMDetector:: nsUniversalXPCOMDetector() : nsUniversalDetector() +nsXPCOMDetector:: nsXPCOMDetector(PRUint32 aLanguageFilter) + : nsUniversalDetector(aLanguageFilter) { } //--------------------------------------------------------------------- -nsUniversalXPCOMDetector::~nsUniversalXPCOMDetector() +nsXPCOMDetector::~nsXPCOMDetector() { } //--------------------------------------------------------------------- -NS_IMPL_ISUPPORTS1(nsUniversalXPCOMDetector, nsICharsetDetector) +NS_IMPL_ISUPPORTS1(nsXPCOMDetector, nsICharsetDetector) //--------------------------------------------------------------------- -NS_IMETHODIMP nsUniversalXPCOMDetector::Init( +NS_IMETHODIMP nsXPCOMDetector::Init( nsICharsetDetectionObserver* aObserver) { NS_ASSERTION(mObserver == nsnull , "Init twice"); @@ -77,7 +78,7 @@ NS_IMETHODIMP nsUniversalXPCOMDetector::Init( return NS_OK; } //---------------------------------------------------------- -NS_IMETHODIMP nsUniversalXPCOMDetector::DoIt(const char* aBuf, +NS_IMETHODIMP nsXPCOMDetector::DoIt(const char* aBuf, PRUint32 aLen, PRBool* oDontFeedMe) { NS_ASSERTION(mObserver != nsnull , "have not init yet"); @@ -100,7 +101,7 @@ NS_IMETHODIMP nsUniversalXPCOMDetector::DoIt(const char* aBuf, return NS_OK; } //---------------------------------------------------------- -NS_IMETHODIMP nsUniversalXPCOMDetector::Done() +NS_IMETHODIMP nsXPCOMDetector::Done() { NS_ASSERTION(mObserver != nsnull , "have not init yet"); #ifdef DEBUG_chardet @@ -117,7 +118,7 @@ NS_IMETHODIMP nsUniversalXPCOMDetector::Done() return NS_OK; } //---------------------------------------------------------- -void nsUniversalXPCOMDetector::Report(const char* aCharset) +void nsXPCOMDetector::Report(const char* aCharset) { NS_ASSERTION(mObserver != nsnull , "have not init yet"); #ifdef DEBUG_chardet @@ -128,18 +129,18 @@ void nsUniversalXPCOMDetector::Report(const char* aCharset) //--------------------------------------------------------------------- -nsUniversalXPCOMStringDetector:: nsUniversalXPCOMStringDetector() - : nsUniversalDetector() +nsXPCOMStringDetector:: nsXPCOMStringDetector(PRUint32 aLanguageFilter) + : nsUniversalDetector(aLanguageFilter) { } //--------------------------------------------------------------------- -nsUniversalXPCOMStringDetector::~nsUniversalXPCOMStringDetector() +nsXPCOMStringDetector::~nsXPCOMStringDetector() { } //--------------------------------------------------------------------- -NS_IMPL_ISUPPORTS1(nsUniversalXPCOMStringDetector, nsIStringCharsetDetector) +NS_IMPL_ISUPPORTS1(nsXPCOMStringDetector, nsIStringCharsetDetector) //--------------------------------------------------------------------- -void nsUniversalXPCOMStringDetector::Report(const char *aCharset) +void nsXPCOMStringDetector::Report(const char *aCharset) { mResult = aCharset; #ifdef DEBUG_chardet @@ -147,7 +148,7 @@ void nsUniversalXPCOMStringDetector::Report(const char *aCharset) #endif } //--------------------------------------------------------------------- -NS_IMETHODIMP nsUniversalXPCOMStringDetector::DoIt(const char* aBuf, +NS_IMETHODIMP nsXPCOMStringDetector::DoIt(const char* aBuf, PRUint32 aLen, const char** oCharset, nsDetectionConfident &oConf) { diff --git a/extensions/universalchardet/src/xpcom/nsUdetXPCOMWrapper.h b/extensions/universalchardet/src/xpcom/nsUdetXPCOMWrapper.h index 53c60e55a82..82257b2f2fe 100644 --- a/extensions/universalchardet/src/xpcom/nsUdetXPCOMWrapper.h +++ b/extensions/universalchardet/src/xpcom/nsUdetXPCOMWrapper.h @@ -53,16 +53,66 @@ #define NS_UNIVERSAL_STRING_DETECTOR_CID \ { 0x6ee5301a, 0x3981, 0x49bd, { 0x85, 0xf8, 0x1a, 0x2c, 0xc2, 0x28, 0xcf, 0x3e } } +// {12BB8F1B-2389-11d3-B3BF-00805F8A6670} +#define NS_JA_PSMDETECTOR_CID \ +{ 0x12bb8f1b, 0x2389, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } + +// {12BB8F1C-2389-11d3-B3BF-00805F8A6670} +#define NS_JA_STRING_PSMDETECTOR_CID \ +{ 0x12bb8f1c, 0x2389, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } + +// {EA06D4E1-2B3D-11d3-B3BF-00805F8A6670} +#define NS_KO_PSMDETECTOR_CID \ +{ 0xea06d4e1, 0x2b3d, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } + +// {EA06D4E2-2B3D-11d3-B3BF-00805F8A6670} +#define NS_ZHCN_PSMDETECTOR_CID \ +{ 0xea06d4e2, 0x2b3d, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } + +// {EA06D4E3-2B3D-11d3-B3BF-00805F8A6670} +#define NS_ZHTW_PSMDETECTOR_CID \ +{ 0xea06d4e3, 0x2b3d, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } + + +// {EA06D4E4-2B3D-11d3-B3BF-00805F8A6670} +#define NS_KO_STRING_PSMDETECTOR_CID \ +{ 0xea06d4e4, 0x2b3d, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } + +// {EA06D4E5-2B3D-11d3-B3BF-00805F8A6670} +#define NS_ZHCN_STRING_PSMDETECTOR_CID \ +{ 0xea06d4e5, 0x2b3d, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } + +// {EA06D4E6-2B3D-11d3-B3BF-00805F8A6670} +#define NS_ZHTW_STRING_PSMDETECTOR_CID \ +{ 0xea06d4e6, 0x2b3d, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } + + +// {FCACEF21-2B40-11d3-B3BF-00805F8A6670} +#define NS_ZH_STRING_PSMDETECTOR_CID \ +{ 0xfcacef21, 0x2b40, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } + +// {FCACEF22-2B40-11d3-B3BF-00805F8A6670} +#define NS_CJK_STRING_PSMDETECTOR_CID \ +{ 0xfcacef22, 0x2b40, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } + + +// {FCACEF23-2B40-11d3-B3BF-00805F8A6670} +#define NS_ZH_PSMDETECTOR_CID \ +{ 0xfcacef23, 0x2b40, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } + +// {FCACEF24-2B40-11d3-B3BF-00805F8A6670} +#define NS_CJK_PSMDETECTOR_CID \ +{ 0xfcacef24, 0x2b40, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } //===================================================================== -class nsUniversalXPCOMDetector : +class nsXPCOMDetector : public nsUniversalDetector, public nsICharsetDetector { NS_DECL_ISUPPORTS public: - nsUniversalXPCOMDetector(); - virtual ~nsUniversalXPCOMDetector(); + nsXPCOMDetector(PRUint32 aLanguageFilter); + virtual ~nsXPCOMDetector(); NS_IMETHOD Init(nsICharsetDetectionObserver* aObserver); NS_IMETHOD DoIt(const char* aBuf, PRUint32 aLen, PRBool *oDontFeedMe); NS_IMETHOD Done(); @@ -74,14 +124,14 @@ class nsUniversalXPCOMDetector : //===================================================================== -class nsUniversalXPCOMStringDetector : +class nsXPCOMStringDetector : public nsUniversalDetector, public nsIStringCharsetDetector { NS_DECL_ISUPPORTS public: - nsUniversalXPCOMStringDetector(); - virtual ~nsUniversalXPCOMStringDetector(); + nsXPCOMStringDetector(PRUint32 aLanguageFilter); + virtual ~nsXPCOMStringDetector(); NS_IMETHOD DoIt(const char* aBuf, PRUint32 aLen, const char** oCharset, nsDetectionConfident &oConf); protected: @@ -91,4 +141,103 @@ class nsUniversalXPCOMStringDetector : const char* mResult; }; +//===================================================================== +class nsUniversalXPCOMDetector : public nsXPCOMDetector +{ +public: + nsUniversalXPCOMDetector() + : nsXPCOMDetector(NS_FILTER_ALL) {} +}; + +class nsUniversalXPCOMStringDetector : public nsXPCOMStringDetector +{ +public: + nsUniversalXPCOMStringDetector() + : nsXPCOMStringDetector(NS_FILTER_ALL) {} +}; + +class nsJAPSMDetector : public nsXPCOMDetector +{ +public: + nsJAPSMDetector() + : nsXPCOMDetector(NS_FILTER_JAPANESE) {} +}; + +class nsJAStringPSMDetector : public nsXPCOMStringDetector +{ +public: + nsJAStringPSMDetector() + : nsXPCOMStringDetector(NS_FILTER_JAPANESE) {} +}; + +class nsKOPSMDetector : public nsXPCOMDetector +{ +public: + nsKOPSMDetector() + : nsXPCOMDetector(NS_FILTER_KOREAN) {} +}; + +class nsKOStringPSMDetector : public nsXPCOMStringDetector +{ +public: + nsKOStringPSMDetector() + : nsXPCOMStringDetector(NS_FILTER_KOREAN) {} +}; + +class nsZHTWPSMDetector : public nsXPCOMDetector +{ +public: + nsZHTWPSMDetector() + : nsXPCOMDetector(NS_FILTER_CHINESE_TRADITIONAL) {} +}; + +class nsZHTWStringPSMDetector : public nsXPCOMStringDetector +{ +public: + nsZHTWStringPSMDetector() + : nsXPCOMStringDetector(NS_FILTER_CHINESE_TRADITIONAL) {} +}; + +class nsZHCNPSMDetector : public nsXPCOMDetector +{ +public: + nsZHCNPSMDetector() + : nsXPCOMDetector(NS_FILTER_CHINESE_SIMPLIFIED) {} +}; + +class nsZHCNStringPSMDetector : public nsXPCOMStringDetector +{ +public: + nsZHCNStringPSMDetector() + : nsXPCOMStringDetector(NS_FILTER_CHINESE_SIMPLIFIED) {} +}; + +class nsZHPSMDetector : public nsXPCOMDetector +{ +public: + nsZHPSMDetector() + : nsXPCOMDetector(NS_FILTER_CHINESE) {} +}; + +class nsZHStringPSMDetector : public nsXPCOMStringDetector +{ +public: + nsZHStringPSMDetector() + : nsXPCOMStringDetector(NS_FILTER_CHINESE) {} +}; + +class nsCJKPSMDetector : public nsXPCOMDetector +{ +public: + nsCJKPSMDetector() + : nsXPCOMDetector(NS_FILTER_CJK) {} +}; + +class nsCJKStringPSMDetector : public nsXPCOMStringDetector +{ +public: + nsCJKStringPSMDetector() + : nsXPCOMStringDetector(NS_FILTER_CJK) {} +}; + #endif //_nsUdetXPCOMWrapper_h__ diff --git a/extensions/universalchardet/src/xpcom/nsUniversalCharDetModule.cpp b/extensions/universalchardet/src/xpcom/nsUniversalCharDetModule.cpp index 7503997a3a6..1c72bf8dec9 100644 --- a/extensions/universalchardet/src/xpcom/nsUniversalCharDetModule.cpp +++ b/extensions/universalchardet/src/xpcom/nsUniversalCharDetModule.cpp @@ -58,26 +58,118 @@ NS_GENERIC_FACTORY_CONSTRUCTOR(nsUniversalXPCOMDetector) NS_GENERIC_FACTORY_CONSTRUCTOR(nsUniversalXPCOMStringDetector) +NS_GENERIC_FACTORY_CONSTRUCTOR(nsJAPSMDetector) +NS_GENERIC_FACTORY_CONSTRUCTOR(nsJAStringPSMDetector) +NS_GENERIC_FACTORY_CONSTRUCTOR(nsKOPSMDetector) +NS_GENERIC_FACTORY_CONSTRUCTOR(nsKOStringPSMDetector) +NS_GENERIC_FACTORY_CONSTRUCTOR(nsZHTWPSMDetector) +NS_GENERIC_FACTORY_CONSTRUCTOR(nsZHTWStringPSMDetector) +NS_GENERIC_FACTORY_CONSTRUCTOR(nsZHCNPSMDetector) +NS_GENERIC_FACTORY_CONSTRUCTOR(nsZHCNStringPSMDetector) +NS_GENERIC_FACTORY_CONSTRUCTOR(nsZHPSMDetector) +NS_GENERIC_FACTORY_CONSTRUCTOR(nsZHStringPSMDetector) +NS_GENERIC_FACTORY_CONSTRUCTOR(nsCJKPSMDetector) +NS_GENERIC_FACTORY_CONSTRUCTOR(nsCJKStringPSMDetector) //---------------------------------------- -static NS_METHOD nsUniversalCharDetectorRegistrationProc(nsIComponentManager *aCompMgr, - nsIFile *aPath, - const char *registryLocation, - const char *componentType, - const nsModuleComponentInfo *info) +static NS_METHOD +AddCategoryEntry(const char* category, + const char* key, + const char* value) { nsresult rv; nsCOMPtr categoryManager(do_GetService("@mozilla.org/categorymanager;1", &rv)); if (NS_FAILED(rv)) return rv; - return categoryManager->AddCategoryEntry(NS_CHARSET_DETECTOR_CATEGORY, - "universal_charset_detector", - info->mContractID, + return categoryManager->AddCategoryEntry(category, key, value, PR_TRUE, PR_TRUE, nsnull); } +static NS_METHOD nsUniversalCharDetectorRegistrationProc(nsIComponentManager *aCompMgr, + nsIFile *aPath, + const char *registryLocation, + const char *componentType, + const nsModuleComponentInfo *info) +{ + return AddCategoryEntry(NS_CHARSET_DETECTOR_CATEGORY, + "universal_charset_detector", + info->mContractID); +} + +static NS_METHOD +nsJAPSMDetectorRegistrationProc(nsIComponentManager *aCompMgr, + nsIFile *aPath, + const char *registryLocation, + const char *componentType, + const nsModuleComponentInfo *info) +{ + return AddCategoryEntry(NS_CHARSET_DETECTOR_CATEGORY, + "ja_parallel_state_machine", + info->mContractID); +} + +static NS_METHOD +nsKOPSMDetectorRegistrationProc(nsIComponentManager *aCompMgr, + nsIFile *aPath, + const char *registryLocation, + const char *componentType, + const nsModuleComponentInfo *info) +{ + return AddCategoryEntry(NS_CHARSET_DETECTOR_CATEGORY, + "ko_parallel_state_machine", + info->mContractID); +} + +static NS_METHOD +nsZHTWPSMDetectorRegistrationProc(nsIComponentManager *aCompMgr, + nsIFile *aPath, + const char *registryLocation, + const char *componentType, + const nsModuleComponentInfo *info) +{ + return AddCategoryEntry(NS_CHARSET_DETECTOR_CATEGORY, + "zhtw_parallel_state_machine", + info->mContractID); +} + +static NS_METHOD +nsZHCNPSMDetectorRegistrationProc(nsIComponentManager *aCompMgr, + nsIFile *aPath, + const char *registryLocation, + const char *componentType, + const nsModuleComponentInfo *info) +{ + return AddCategoryEntry(NS_CHARSET_DETECTOR_CATEGORY, + "zhcn_parallel_state_machine", + info->mContractID); +} + +static NS_METHOD +nsZHPSMDetectorRegistrationProc(nsIComponentManager *aCompMgr, + nsIFile *aPath, + const char *registryLocation, + const char *componentType, + const nsModuleComponentInfo *info) +{ + return AddCategoryEntry(NS_CHARSET_DETECTOR_CATEGORY, + "zh_parallel_state_machine", + info->mContractID); +} + +static NS_METHOD +nsCJKPSMDetectorRegistrationProc(nsIComponentManager *aCompMgr, + nsIFile *aPath, + const char *registryLocation, + const char *componentType, + const nsModuleComponentInfo *info) +{ + return AddCategoryEntry(NS_CHARSET_DETECTOR_CATEGORY, + "cjk_parallel_state_machine", + info->mContractID); +} + // Component Table static const nsModuleComponentInfo components[] = { @@ -86,7 +178,43 @@ static const nsModuleComponentInfo components[] = nsUniversalCharDetectorRegistrationProc, NULL}, { "Universal String Charset Detector", NS_UNIVERSAL_STRING_DETECTOR_CID, NS_STRCDETECTOR_CONTRACTID_BASE "universal_charset_detector", nsUniversalXPCOMStringDetectorConstructor, - NULL, NULL} + NULL, NULL}, + { "PSM based Japanese Charset Detector", NS_JA_PSMDETECTOR_CID, + NS_CHARSET_DETECTOR_CONTRACTID_BASE "ja_parallel_state_machine", nsJAPSMDetectorConstructor, + nsJAPSMDetectorRegistrationProc, NULL}, + { "PSM based Japanese String Charset Detector", NS_JA_STRING_PSMDETECTOR_CID, + NS_STRCDETECTOR_CONTRACTID_BASE "ja_parallel_state_machine", nsJAStringPSMDetectorConstructor, + NULL, NULL}, + { "PSM based Korean Charset Detector", NS_KO_PSMDETECTOR_CID, + NS_CHARSET_DETECTOR_CONTRACTID_BASE "ko_parallel_state_machine", nsKOPSMDetectorConstructor, + nsKOPSMDetectorRegistrationProc, NULL}, + { "PSM based Korean String Charset Detector", NS_KO_STRING_PSMDETECTOR_CID, + NS_STRCDETECTOR_CONTRACTID_BASE "ko_parallel_state_machine", nsKOStringPSMDetectorConstructor, + NULL, NULL}, + { "PSM based Traditional Chinese Charset Detector", NS_ZHTW_PSMDETECTOR_CID, + NS_CHARSET_DETECTOR_CONTRACTID_BASE "zhtw_parallel_state_machine", nsZHTWPSMDetectorConstructor, + nsZHTWPSMDetectorRegistrationProc, NULL}, + { "PSM based Traditional Chinese String Charset Detector", NS_ZHTW_STRING_PSMDETECTOR_CID, + NS_STRCDETECTOR_CONTRACTID_BASE "zhtw_parallel_state_machine", nsZHTWStringPSMDetectorConstructor, + NULL, NULL}, + { "PSM based Simplified Chinese Charset Detector", NS_ZHCN_PSMDETECTOR_CID, + NS_CHARSET_DETECTOR_CONTRACTID_BASE "zhcn_parallel_state_machine", nsZHCNPSMDetectorConstructor, + nsZHCNPSMDetectorRegistrationProc, NULL}, + { "PSM based Simplified Chinese String Charset Detector", NS_ZHCN_STRING_PSMDETECTOR_CID, + NS_STRCDETECTOR_CONTRACTID_BASE "zhcn_parallel_state_machine", nsZHCNStringPSMDetectorConstructor, + NULL, NULL}, + { "PSM based Chinese Charset Detector", NS_ZH_PSMDETECTOR_CID, + NS_CHARSET_DETECTOR_CONTRACTID_BASE "zh_parallel_state_machine", nsZHPSMDetectorConstructor, + nsZHPSMDetectorRegistrationProc, NULL}, + { "PSM based Chinese String Charset Detector", NS_ZH_STRING_PSMDETECTOR_CID, + NS_STRCDETECTOR_CONTRACTID_BASE "zh_parallel_state_machine", nsZHStringPSMDetectorConstructor, + NULL, NULL}, + { "PSM based CJK Charset Detector", NS_CJK_PSMDETECTOR_CID, + NS_CHARSET_DETECTOR_CONTRACTID_BASE "cjk_parallel_state_machine", nsCJKPSMDetectorConstructor, + nsCJKPSMDetectorRegistrationProc, NULL}, + { "PSM based CJK String Charset Detector", NS_CJK_STRING_PSMDETECTOR_CID, + NS_STRCDETECTOR_CONTRACTID_BASE "cjk_parallel_state_machine", nsCJKStringPSMDetectorConstructor, + NULL, NULL}, }; NS_IMPL_NSGETMODULE(nsUniversalCharDetModule, components) diff --git a/extensions/universalchardet/tests/Makefile.in b/extensions/universalchardet/tests/Makefile.in index 402b141467d..a8be53e2ec0 100644 --- a/extensions/universalchardet/tests/Makefile.in +++ b/extensions/universalchardet/tests/Makefile.in @@ -67,3 +67,17 @@ LIBS += \ include $(topsrcdir)/config/rules.mk +ifdef MOZ_MOCHITEST +relativesrcdir = extensions/universalchardet/tests + +_TEST_FILES = \ + test_bug426271-euc-jp.html \ + bug426271_text-euc-jp.html \ + test_bug426271-utf-8.html \ + bug426271_text-utf-8.html \ + $(NULL) + +libs:: $(_TEST_FILES) + $(INSTALL) $(foreach f,$^,"$f") $(DEPTH)/_tests/testing/mochitest/tests/$(relativesrcdir) + +endif # MOZ_MOCHITEST diff --git a/extensions/universalchardet/tests/UniversalChardetTest.cpp b/extensions/universalchardet/tests/UniversalChardetTest.cpp index 2b1f9e758bd..0475064cb5d 100644 --- a/extensions/universalchardet/tests/UniversalChardetTest.cpp +++ b/extensions/universalchardet/tests/UniversalChardetTest.cpp @@ -59,7 +59,7 @@ void usage() { class nsUniversalChardetTest : public nsUniversalDetector { public: - nsUniversalChardetTest() { }; + nsUniversalChardetTest() : nsUniversalDetector(NS_FILTER_ALL) { }; virtual ~nsUniversalChardetTest() { }; PRBool done() const { return mDone; } diff --git a/extensions/universalchardet/tests/bug426271_text-euc-jp.html b/extensions/universalchardet/tests/bug426271_text-euc-jp.html new file mode 100644 index 00000000000..48719606ba9 --- /dev/null +++ b/extensions/universalchardet/tests/bug426271_text-euc-jp.html @@ -0,0 +1,11 @@ + + + +日本語エンコードテスト + + +これはEUC-JPです昔々、ある所に子供のいない老夫婦が住んでいた。ある日、お婆さんが川で洗濯をしていると、大きな桃が流れて来たので、お爺さんと食べようと持ち帰った。二人で桃を割ると中から男の子が生まれたので、「桃太郎」と名付けて大事に育てた。 + +成長した桃太郎は、鬼ヶ島の鬼が人々を苦しめていることを知り、鬼退治を決意する。両親から黍団子を餞別に貰い、道中にそれを分け与えてイヌ、サル、キジを家来に従える。鬼ヶ島で鬼と戦い、見事に勝利を収め、鬼が方々から奪っていった財宝を持ち帰り、お爺さん・お婆さんの元に返り、幸せに暮らしたという。出典: フリー百科事典『ウィキペディア（Wikipedia）』 + + diff --git a/extensions/universalchardet/tests/bug426271_text-utf-8.html b/extensions/universalchardet/tests/bug426271_text-utf-8.html new file mode 100644 index 00000000000..e4b8440015e --- /dev/null +++ b/extensions/universalchardet/tests/bug426271_text-utf-8.html @@ -0,0 +1,11 @@ + + + +ユ茯潟潟若鴻 + + +UTF-8с絖箴紊紿篏сャ紿綏ф羶紊с罅羌ャс冴蕋鴻≦鍵ｃ篋篋冴ф蚊筝激絖障с罅紊篁紊т蚊 + +激罅紊薔若九涯薔若篋冴ャ薔守羃祉羆堺筝∴Κ藥ｅ蕕ャ莢違筝筝ゃ泣吾絎倶ャ緇薔若九涯ч若荀篋薔若鴻絅ｃｃ莢≦≦鍵冴祉紿菴綛吾阪: 主丞篋吾ｃｃ鐚Wikipedia鐚 + + diff --git a/extensions/universalchardet/tests/test_bug426271-euc-jp.html b/extensions/universalchardet/tests/test_bug426271-euc-jp.html new file mode 100644 index 00000000000..13ccd6ad0bb --- /dev/null +++ b/extensions/universalchardet/tests/test_bug426271-euc-jp.html @@ -0,0 +1,65 @@ + + + + + Test for Bug 426271 + + + + + +Mozilla Bug 426271 +

+ + +
+
+
+ + + diff --git a/extensions/universalchardet/tests/test_bug426271-utf-8.html b/extensions/universalchardet/tests/test_bug426271-utf-8.html new file mode 100644 index 00000000000..a320562af67 --- /dev/null +++ b/extensions/universalchardet/tests/test_bug426271-utf-8.html @@ -0,0 +1,65 @@ + + + + + Test for Bug 426271 + + + + + +Mozilla Bug 426271 +

+ + +
+
+
+ + + diff --git a/intl/chardet/src/Big5Statistics.h b/intl/chardet/src/Big5Statistics.h deleted file mode 100644 index fae15d91187..00000000000 --- a/intl/chardet/src/Big5Statistics.h +++ /dev/null @@ -1,236 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -{ - { - 0.000000f, // FreqH[a1] - 0.000000f, // FreqH[a2] - 0.000000f, // FreqH[a3] - 0.114427f, // FreqH[a4] - 0.061058f, // FreqH[a5] - 0.075598f, // FreqH[a6] - 0.048386f, // FreqH[a7] - 0.063966f, // FreqH[a8] - 0.027094f, // FreqH[a9] - 0.095787f, // FreqH[aa] - 0.029525f, // FreqH[ab] - 0.031331f, // FreqH[ac] - 0.036915f, // FreqH[ad] - 0.021805f, // FreqH[ae] - 0.019349f, // FreqH[af] - 0.037496f, // FreqH[b0] - 0.018068f, // FreqH[b1] - 0.012760f, // FreqH[b2] - 0.030053f, // FreqH[b3] - 0.017339f, // FreqH[b4] - 0.016731f, // FreqH[b5] - 0.019501f, // FreqH[b6] - 0.011240f, // FreqH[b7] - 0.032973f, // FreqH[b8] - 0.016658f, // FreqH[b9] - 0.015872f, // FreqH[ba] - 0.021458f, // FreqH[bb] - 0.012378f, // FreqH[bc] - 0.017003f, // FreqH[bd] - 0.020802f, // FreqH[be] - 0.012454f, // FreqH[bf] - 0.009239f, // FreqH[c0] - 0.012829f, // FreqH[c1] - 0.007922f, // FreqH[c2] - 0.010079f, // FreqH[c3] - 0.009815f, // FreqH[c4] - 0.010104f, // FreqH[c5] - 0.000000f, // FreqH[c6] - 0.000000f, // FreqH[c7] - 0.000000f, // FreqH[c8] - 0.000053f, // FreqH[c9] - 0.000035f, // FreqH[ca] - 0.000105f, // FreqH[cb] - 0.000031f, // FreqH[cc] - 0.000088f, // FreqH[cd] - 0.000027f, // FreqH[ce] - 0.000027f, // FreqH[cf] - 0.000026f, // FreqH[d0] - 0.000035f, // FreqH[d1] - 0.000024f, // FreqH[d2] - 0.000034f, // FreqH[d3] - 0.000375f, // FreqH[d4] - 0.000025f, // FreqH[d5] - 0.000028f, // FreqH[d6] - 0.000020f, // FreqH[d7] - 0.000024f, // FreqH[d8] - 0.000028f, // FreqH[d9] - 0.000031f, // FreqH[da] - 0.000059f, // FreqH[db] - 0.000040f, // FreqH[dc] - 0.000030f, // FreqH[dd] - 0.000079f, // FreqH[de] - 0.000037f, // FreqH[df] - 0.000040f, // FreqH[e0] - 0.000023f, // FreqH[e1] - 0.000030f, // FreqH[e2] - 0.000027f, // FreqH[e3] - 0.000064f, // FreqH[e4] - 0.000020f, // FreqH[e5] - 0.000027f, // FreqH[e6] - 0.000025f, // FreqH[e7] - 0.000074f, // FreqH[e8] - 0.000019f, // FreqH[e9] - 0.000023f, // FreqH[ea] - 0.000021f, // FreqH[eb] - 0.000018f, // FreqH[ec] - 0.000017f, // FreqH[ed] - 0.000035f, // FreqH[ee] - 0.000021f, // FreqH[ef] - 0.000019f, // FreqH[f0] - 0.000025f, // FreqH[f1] - 0.000017f, // FreqH[f2] - 0.000037f, // FreqH[f3] - 0.000018f, // FreqH[f4] - 0.000018f, // FreqH[f5] - 0.000019f, // FreqH[f6] - 0.000022f, // FreqH[f7] - 0.000033f, // FreqH[f8] - 0.000032f, // FreqH[f9] - 0.000000f, // FreqH[fa] - 0.000000f, // FreqH[fb] - 0.000000f, // FreqH[fc] - 0.000000f, // FreqH[fd] - 0.000000f // FreqH[fe] - }, -0.020606f, // Lead Byte StdDev -0.010638f, // Lead Byte Mean -0.675261f, // Lead Byte Weight - { - 0.020256f, // FreqL[a1] - 0.003293f, // FreqL[a2] - 0.045811f, // FreqL[a3] - 0.016650f, // FreqL[a4] - 0.007066f, // FreqL[a5] - 0.004146f, // FreqL[a6] - 0.009229f, // FreqL[a7] - 0.007333f, // FreqL[a8] - 0.003296f, // FreqL[a9] - 0.005239f, // FreqL[aa] - 0.008282f, // FreqL[ab] - 0.003791f, // FreqL[ac] - 0.006116f, // FreqL[ad] - 0.003536f, // FreqL[ae] - 0.004024f, // FreqL[af] - 0.016654f, // FreqL[b0] - 0.009334f, // FreqL[b1] - 0.005429f, // FreqL[b2] - 0.033392f, // FreqL[b3] - 0.006121f, // FreqL[b4] - 0.008983f, // FreqL[b5] - 0.002801f, // FreqL[b6] - 0.004221f, // FreqL[b7] - 0.010357f, // FreqL[b8] - 0.014695f, // FreqL[b9] - 0.077937f, // FreqL[ba] - 0.006314f, // FreqL[bb] - 0.004020f, // FreqL[bc] - 0.007331f, // FreqL[bd] - 0.007150f, // FreqL[be] - 0.005341f, // FreqL[bf] - 0.009195f, // FreqL[c0] - 0.005350f, // FreqL[c1] - 0.005698f, // FreqL[c2] - 0.004472f, // FreqL[c3] - 0.007242f, // FreqL[c4] - 0.004039f, // FreqL[c5] - 0.011154f, // FreqL[c6] - 0.016184f, // FreqL[c7] - 0.004741f, // FreqL[c8] - 0.012814f, // FreqL[c9] - 0.007679f, // FreqL[ca] - 0.008045f, // FreqL[cb] - 0.016631f, // FreqL[cc] - 0.009451f, // FreqL[cd] - 0.016487f, // FreqL[ce] - 0.007287f, // FreqL[cf] - 0.012688f, // FreqL[d0] - 0.017421f, // FreqL[d1] - 0.013205f, // FreqL[d2] - 0.031480f, // FreqL[d3] - 0.003404f, // FreqL[d4] - 0.009149f, // FreqL[d5] - 0.008921f, // FreqL[d6] - 0.007514f, // FreqL[d7] - 0.008683f, // FreqL[d8] - 0.008203f, // FreqL[d9] - 0.031403f, // FreqL[da] - 0.011733f, // FreqL[db] - 0.015617f, // FreqL[dc] - 0.015306f, // FreqL[dd] - 0.004004f, // FreqL[de] - 0.010899f, // FreqL[df] - 0.009961f, // FreqL[e0] - 0.008388f, // FreqL[e1] - 0.010920f, // FreqL[e2] - 0.003925f, // FreqL[e3] - 0.008585f, // FreqL[e4] - 0.009108f, // FreqL[e5] - 0.015546f, // FreqL[e6] - 0.004659f, // FreqL[e7] - 0.006934f, // FreqL[e8] - 0.007023f, // FreqL[e9] - 0.020252f, // FreqL[ea] - 0.005387f, // FreqL[eb] - 0.024704f, // FreqL[ec] - 0.006963f, // FreqL[ed] - 0.002625f, // FreqL[ee] - 0.009512f, // FreqL[ef] - 0.002971f, // FreqL[f0] - 0.008233f, // FreqL[f1] - 0.010000f, // FreqL[f2] - 0.011973f, // FreqL[f3] - 0.010553f, // FreqL[f4] - 0.005945f, // FreqL[f5] - 0.006349f, // FreqL[f6] - 0.009401f, // FreqL[f7] - 0.008577f, // FreqL[f8] - 0.008186f, // FreqL[f9] - 0.008159f, // FreqL[fa] - 0.005033f, // FreqL[fb] - 0.008714f, // FreqL[fc] - 0.010614f, // FreqL[fd] - 0.006554f // FreqL[fe] - }, -0.009909f, // Trail Byte StdDev -0.010638f, // Trail Byte Mean -0.324739f // Trial Byte Weight -}; diff --git a/intl/chardet/src/EUCJPStatistics.h b/intl/chardet/src/EUCJPStatistics.h deleted file mode 100644 index de6f68e1af2..00000000000 --- a/intl/chardet/src/EUCJPStatistics.h +++ /dev/null @@ -1,236 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -{ - { - 0.364808f, // FreqH[a1] - 0.000000f, // FreqH[a2] - 0.000000f, // FreqH[a3] - 0.145325f, // FreqH[a4] - 0.304891f, // FreqH[a5] - 0.000000f, // FreqH[a6] - 0.000000f, // FreqH[a7] - 0.000000f, // FreqH[a8] - 0.000000f, // FreqH[a9] - 0.000000f, // FreqH[aa] - 0.000000f, // FreqH[ab] - 0.000000f, // FreqH[ac] - 0.000000f, // FreqH[ad] - 0.000000f, // FreqH[ae] - 0.000000f, // FreqH[af] - 0.001835f, // FreqH[b0] - 0.010771f, // FreqH[b1] - 0.006462f, // FreqH[b2] - 0.001157f, // FreqH[b3] - 0.002114f, // FreqH[b4] - 0.003231f, // FreqH[b5] - 0.001356f, // FreqH[b6] - 0.007420f, // FreqH[b7] - 0.004189f, // FreqH[b8] - 0.003231f, // FreqH[b9] - 0.003032f, // FreqH[ba] - 0.033190f, // FreqH[bb] - 0.006303f, // FreqH[bc] - 0.006064f, // FreqH[bd] - 0.009973f, // FreqH[be] - 0.002354f, // FreqH[bf] - 0.003670f, // FreqH[c0] - 0.009135f, // FreqH[c1] - 0.001675f, // FreqH[c2] - 0.002792f, // FreqH[c3] - 0.002194f, // FreqH[c4] - 0.014720f, // FreqH[c5] - 0.011928f, // FreqH[c6] - 0.000878f, // FreqH[c7] - 0.013124f, // FreqH[c8] - 0.001077f, // FreqH[c9] - 0.009295f, // FreqH[ca] - 0.003471f, // FreqH[cb] - 0.002872f, // FreqH[cc] - 0.002433f, // FreqH[cd] - 0.000957f, // FreqH[ce] - 0.001636f, // FreqH[cf] - 0.000000f, // FreqH[d0] - 0.000000f, // FreqH[d1] - 0.000000f, // FreqH[d2] - 0.000000f, // FreqH[d3] - 0.000000f, // FreqH[d4] - 0.000000f, // FreqH[d5] - 0.000000f, // FreqH[d6] - 0.000000f, // FreqH[d7] - 0.000000f, // FreqH[d8] - 0.000000f, // FreqH[d9] - 0.000000f, // FreqH[da] - 0.000000f, // FreqH[db] - 0.000000f, // FreqH[dc] - 0.000000f, // FreqH[dd] - 0.000080f, // FreqH[de] - 0.000279f, // FreqH[df] - 0.000000f, // FreqH[e0] - 0.000000f, // FreqH[e1] - 0.000000f, // FreqH[e2] - 0.000000f, // FreqH[e3] - 0.000000f, // FreqH[e4] - 0.000000f, // FreqH[e5] - 0.000000f, // FreqH[e6] - 0.000000f, // FreqH[e7] - 0.000000f, // FreqH[e8] - 0.000000f, // FreqH[e9] - 0.000000f, // FreqH[ea] - 0.000000f, // FreqH[eb] - 0.000000f, // FreqH[ec] - 0.000000f, // FreqH[ed] - 0.000000f, // FreqH[ee] - 0.000000f, // FreqH[ef] - 0.000000f, // FreqH[f0] - 0.000000f, // FreqH[f1] - 0.000000f, // FreqH[f2] - 0.000000f, // FreqH[f3] - 0.000000f, // FreqH[f4] - 0.000000f, // FreqH[f5] - 0.000000f, // FreqH[f6] - 0.000000f, // FreqH[f7] - 0.000000f, // FreqH[f8] - 0.000000f, // FreqH[f9] - 0.000000f, // FreqH[fa] - 0.000000f, // FreqH[fb] - 0.000000f, // FreqH[fc] - 0.000080f, // FreqH[fd] - 0.000000f // FreqH[fe] - }, -0.050407f, // Lead Byte StdDev -0.010638f, // Lead Byte Mean -0.640871f, // Lead Byte Weight - { - 0.002473f, // FreqL[a1] - 0.039134f, // FreqL[a2] - 0.152745f, // FreqL[a3] - 0.009694f, // FreqL[a4] - 0.000359f, // FreqL[a5] - 0.022180f, // FreqL[a6] - 0.000758f, // FreqL[a7] - 0.004308f, // FreqL[a8] - 0.000160f, // FreqL[a9] - 0.002513f, // FreqL[aa] - 0.003072f, // FreqL[ab] - 0.001316f, // FreqL[ac] - 0.003830f, // FreqL[ad] - 0.001037f, // FreqL[ae] - 0.003590f, // FreqL[af] - 0.000957f, // FreqL[b0] - 0.000160f, // FreqL[b1] - 0.000239f, // FreqL[b2] - 0.006462f, // FreqL[b3] - 0.001596f, // FreqL[b4] - 0.031554f, // FreqL[b5] - 0.001316f, // FreqL[b6] - 0.002194f, // FreqL[b7] - 0.016555f, // FreqL[b8] - 0.003271f, // FreqL[b9] - 0.000678f, // FreqL[ba] - 0.000598f, // FreqL[bb] - 0.206438f, // FreqL[bc] - 0.000718f, // FreqL[bd] - 0.001077f, // FreqL[be] - 0.003710f, // FreqL[bf] - 0.001356f, // FreqL[c0] - 0.001356f, // FreqL[c1] - 0.000439f, // FreqL[c2] - 0.004388f, // FreqL[c3] - 0.005704f, // FreqL[c4] - 0.000878f, // FreqL[c5] - 0.010172f, // FreqL[c6] - 0.007061f, // FreqL[c7] - 0.014680f, // FreqL[c8] - 0.000638f, // FreqL[c9] - 0.025730f, // FreqL[ca] - 0.002792f, // FreqL[cb] - 0.000718f, // FreqL[cc] - 0.001795f, // FreqL[cd] - 0.091551f, // FreqL[ce] - 0.000758f, // FreqL[cf] - 0.003909f, // FreqL[d0] - 0.000558f, // FreqL[d1] - 0.031195f, // FreqL[d2] - 0.007061f, // FreqL[d3] - 0.001316f, // FreqL[d4] - 0.022579f, // FreqL[d5] - 0.006981f, // FreqL[d6] - 0.007260f, // FreqL[d7] - 0.001117f, // FreqL[d8] - 0.000239f, // FreqL[d9] - 0.012127f, // FreqL[da] - 0.000878f, // FreqL[db] - 0.003790f, // FreqL[dc] - 0.001077f, // FreqL[dd] - 0.000758f, // FreqL[de] - 0.002114f, // FreqL[df] - 0.002234f, // FreqL[e0] - 0.000678f, // FreqL[e1] - 0.002992f, // FreqL[e2] - 0.003311f, // FreqL[e3] - 0.023416f, // FreqL[e4] - 0.001237f, // FreqL[e5] - 0.002753f, // FreqL[e6] - 0.005146f, // FreqL[e7] - 0.002194f, // FreqL[e8] - 0.007021f, // FreqL[e9] - 0.008497f, // FreqL[ea] - 0.013763f, // FreqL[eb] - 0.011768f, // FreqL[ec] - 0.006303f, // FreqL[ed] - 0.001915f, // FreqL[ee] - 0.000638f, // FreqL[ef] - 0.008776f, // FreqL[f0] - 0.000918f, // FreqL[f1] - 0.003431f, // FreqL[f2] - 0.057603f, // FreqL[f3] - 0.000439f, // FreqL[f4] - 0.000439f, // FreqL[f5] - 0.000758f, // FreqL[f6] - 0.002872f, // FreqL[f7] - 0.001675f, // FreqL[f8] - 0.011050f, // FreqL[f9] - 0.000000f, // FreqL[fa] - 0.000279f, // FreqL[fb] - 0.012127f, // FreqL[fc] - 0.000718f, // FreqL[fd] - 0.007380f // FreqL[fe] - }, -0.028247f, // Trail Byte StdDev -0.010638f, // Trail Byte Mean -0.359129f // Trial Byte Weight -}; diff --git a/intl/chardet/src/EUCKRStatistics.h b/intl/chardet/src/EUCKRStatistics.h deleted file mode 100644 index 5e0817b9470..00000000000 --- a/intl/chardet/src/EUCKRStatistics.h +++ /dev/null @@ -1,236 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -{ - { - 0.000000f, // FreqH[a1] - 0.000000f, // FreqH[a2] - 0.000000f, // FreqH[a3] - 0.000000f, // FreqH[a4] - 0.000000f, // FreqH[a5] - 0.000000f, // FreqH[a6] - 0.000000f, // FreqH[a7] - 0.000412f, // FreqH[a8] - 0.000000f, // FreqH[a9] - 0.000000f, // FreqH[aa] - 0.000000f, // FreqH[ab] - 0.000000f, // FreqH[ac] - 0.000000f, // FreqH[ad] - 0.000000f, // FreqH[ae] - 0.000000f, // FreqH[af] - 0.057502f, // FreqH[b0] - 0.033182f, // FreqH[b1] - 0.002267f, // FreqH[b2] - 0.016076f, // FreqH[b3] - 0.014633f, // FreqH[b4] - 0.032976f, // FreqH[b5] - 0.004122f, // FreqH[b6] - 0.011336f, // FreqH[b7] - 0.058533f, // FreqH[b8] - 0.024526f, // FreqH[b9] - 0.025969f, // FreqH[ba] - 0.054411f, // FreqH[bb] - 0.019580f, // FreqH[bc] - 0.063273f, // FreqH[bd] - 0.113974f, // FreqH[be] - 0.029885f, // FreqH[bf] - 0.150041f, // FreqH[c0] - 0.059151f, // FreqH[c1] - 0.002679f, // FreqH[c2] - 0.009893f, // FreqH[c3] - 0.014839f, // FreqH[c4] - 0.026381f, // FreqH[c5] - 0.015045f, // FreqH[c6] - 0.069456f, // FreqH[c7] - 0.089860f, // FreqH[c8] - 0.000000f, // FreqH[c9] - 0.000000f, // FreqH[ca] - 0.000000f, // FreqH[cb] - 0.000000f, // FreqH[cc] - 0.000000f, // FreqH[cd] - 0.000000f, // FreqH[ce] - 0.000000f, // FreqH[cf] - 0.000000f, // FreqH[d0] - 0.000000f, // FreqH[d1] - 0.000000f, // FreqH[d2] - 0.000000f, // FreqH[d3] - 0.000000f, // FreqH[d4] - 0.000000f, // FreqH[d5] - 0.000000f, // FreqH[d6] - 0.000000f, // FreqH[d7] - 0.000000f, // FreqH[d8] - 0.000000f, // FreqH[d9] - 0.000000f, // FreqH[da] - 0.000000f, // FreqH[db] - 0.000000f, // FreqH[dc] - 0.000000f, // FreqH[dd] - 0.000000f, // FreqH[de] - 0.000000f, // FreqH[df] - 0.000000f, // FreqH[e0] - 0.000000f, // FreqH[e1] - 0.000000f, // FreqH[e2] - 0.000000f, // FreqH[e3] - 0.000000f, // FreqH[e4] - 0.000000f, // FreqH[e5] - 0.000000f, // FreqH[e6] - 0.000000f, // FreqH[e7] - 0.000000f, // FreqH[e8] - 0.000000f, // FreqH[e9] - 0.000000f, // FreqH[ea] - 0.000000f, // FreqH[eb] - 0.000000f, // FreqH[ec] - 0.000000f, // FreqH[ed] - 0.000000f, // FreqH[ee] - 0.000000f, // FreqH[ef] - 0.000000f, // FreqH[f0] - 0.000000f, // FreqH[f1] - 0.000000f, // FreqH[f2] - 0.000000f, // FreqH[f3] - 0.000000f, // FreqH[f4] - 0.000000f, // FreqH[f5] - 0.000000f, // FreqH[f6] - 0.000000f, // FreqH[f7] - 0.000000f, // FreqH[f8] - 0.000000f, // FreqH[f9] - 0.000000f, // FreqH[fa] - 0.000000f, // FreqH[fb] - 0.000000f, // FreqH[fc] - 0.000000f, // FreqH[fd] - 0.000000f // FreqH[fe] - }, -0.025593f, // Lead Byte StdDev -0.010638f, // Lead Byte Mean -0.647437f, // Lead Byte Weight - { - 0.016694f, // FreqL[a1] - 0.000000f, // FreqL[a2] - 0.012778f, // FreqL[a3] - 0.030091f, // FreqL[a4] - 0.002679f, // FreqL[a5] - 0.006595f, // FreqL[a6] - 0.001855f, // FreqL[a7] - 0.000824f, // FreqL[a8] - 0.005977f, // FreqL[a9] - 0.004740f, // FreqL[aa] - 0.003092f, // FreqL[ab] - 0.000824f, // FreqL[ac] - 0.019580f, // FreqL[ad] - 0.037304f, // FreqL[ae] - 0.008244f, // FreqL[af] - 0.014633f, // FreqL[b0] - 0.001031f, // FreqL[b1] - 0.000000f, // FreqL[b2] - 0.003298f, // FreqL[b3] - 0.002061f, // FreqL[b4] - 0.006183f, // FreqL[b5] - 0.005977f, // FreqL[b6] - 0.000824f, // FreqL[b7] - 0.021847f, // FreqL[b8] - 0.014839f, // FreqL[b9] - 0.052968f, // FreqL[ba] - 0.017312f, // FreqL[bb] - 0.007626f, // FreqL[bc] - 0.000412f, // FreqL[bd] - 0.000824f, // FreqL[be] - 0.011129f, // FreqL[bf] - 0.000000f, // FreqL[c0] - 0.000412f, // FreqL[c1] - 0.001649f, // FreqL[c2] - 0.005977f, // FreqL[c3] - 0.065746f, // FreqL[c4] - 0.020198f, // FreqL[c5] - 0.021434f, // FreqL[c6] - 0.014633f, // FreqL[c7] - 0.004122f, // FreqL[c8] - 0.001649f, // FreqL[c9] - 0.000824f, // FreqL[ca] - 0.000824f, // FreqL[cb] - 0.051937f, // FreqL[cc] - 0.019580f, // FreqL[cd] - 0.023289f, // FreqL[ce] - 0.026381f, // FreqL[cf] - 0.040396f, // FreqL[d0] - 0.009068f, // FreqL[d1] - 0.001443f, // FreqL[d2] - 0.003710f, // FreqL[d3] - 0.007420f, // FreqL[d4] - 0.001443f, // FreqL[d5] - 0.013190f, // FreqL[d6] - 0.002885f, // FreqL[d7] - 0.000412f, // FreqL[d8] - 0.003298f, // FreqL[d9] - 0.025969f, // FreqL[da] - 0.000412f, // FreqL[db] - 0.000412f, // FreqL[dc] - 0.006183f, // FreqL[dd] - 0.003298f, // FreqL[de] - 0.066983f, // FreqL[df] - 0.002679f, // FreqL[e0] - 0.002267f, // FreqL[e1] - 0.011129f, // FreqL[e2] - 0.000412f, // FreqL[e3] - 0.010099f, // FreqL[e4] - 0.015251f, // FreqL[e5] - 0.007626f, // FreqL[e6] - 0.043899f, // FreqL[e7] - 0.003710f, // FreqL[e8] - 0.002679f, // FreqL[e9] - 0.001443f, // FreqL[ea] - 0.010923f, // FreqL[eb] - 0.002885f, // FreqL[ec] - 0.009068f, // FreqL[ed] - 0.019992f, // FreqL[ee] - 0.000412f, // FreqL[ef] - 0.008450f, // FreqL[f0] - 0.005153f, // FreqL[f1] - 0.000000f, // FreqL[f2] - 0.010099f, // FreqL[f3] - 0.000000f, // FreqL[f4] - 0.001649f, // FreqL[f5] - 0.012160f, // FreqL[f6] - 0.011542f, // FreqL[f7] - 0.006595f, // FreqL[f8] - 0.001855f, // FreqL[f9] - 0.010923f, // FreqL[fa] - 0.000412f, // FreqL[fb] - 0.023702f, // FreqL[fc] - 0.003710f, // FreqL[fd] - 0.001855f // FreqL[fe] - }, -0.013937f, // Trail Byte StdDev -0.010638f, // Trail Byte Mean -0.352563f // Trial Byte Weight -}; diff --git a/intl/chardet/src/EUCTWStatistics.h b/intl/chardet/src/EUCTWStatistics.h deleted file mode 100644 index bec749a7aa5..00000000000 --- a/intl/chardet/src/EUCTWStatistics.h +++ /dev/null @@ -1,236 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -{ - { - 0.000000f, // FreqH[a1] - 0.000000f, // FreqH[a2] - 0.000000f, // FreqH[a3] - 0.000000f, // FreqH[a4] - 0.000000f, // FreqH[a5] - 0.000000f, // FreqH[a6] - 0.000000f, // FreqH[a7] - 0.000000f, // FreqH[a8] - 0.000000f, // FreqH[a9] - 0.000000f, // FreqH[aa] - 0.000000f, // FreqH[ab] - 0.000000f, // FreqH[ac] - 0.000000f, // FreqH[ad] - 0.000000f, // FreqH[ae] - 0.000000f, // FreqH[af] - 0.000000f, // FreqH[b0] - 0.000000f, // FreqH[b1] - 0.000000f, // FreqH[b2] - 0.000000f, // FreqH[b3] - 0.000000f, // FreqH[b4] - 0.000000f, // FreqH[b5] - 0.000000f, // FreqH[b6] - 0.000000f, // FreqH[b7] - 0.000000f, // FreqH[b8] - 0.000000f, // FreqH[b9] - 0.000000f, // FreqH[ba] - 0.000000f, // FreqH[bb] - 0.000000f, // FreqH[bc] - 0.000000f, // FreqH[bd] - 0.000000f, // FreqH[be] - 0.000000f, // FreqH[bf] - 0.000000f, // FreqH[c0] - 0.000000f, // FreqH[c1] - 0.000000f, // FreqH[c2] - 0.000000f, // FreqH[c3] - 0.119286f, // FreqH[c4] - 0.052233f, // FreqH[c5] - 0.044126f, // FreqH[c6] - 0.052494f, // FreqH[c7] - 0.045906f, // FreqH[c8] - 0.019038f, // FreqH[c9] - 0.032465f, // FreqH[ca] - 0.026252f, // FreqH[cb] - 0.025502f, // FreqH[cc] - 0.015963f, // FreqH[cd] - 0.052493f, // FreqH[ce] - 0.019256f, // FreqH[cf] - 0.015137f, // FreqH[d0] - 0.031782f, // FreqH[d1] - 0.017370f, // FreqH[d2] - 0.018494f, // FreqH[d3] - 0.015575f, // FreqH[d4] - 0.016621f, // FreqH[d5] - 0.007444f, // FreqH[d6] - 0.011642f, // FreqH[d7] - 0.013916f, // FreqH[d8] - 0.019159f, // FreqH[d9] - 0.016445f, // FreqH[da] - 0.007851f, // FreqH[db] - 0.011079f, // FreqH[dc] - 0.022842f, // FreqH[dd] - 0.015513f, // FreqH[de] - 0.010033f, // FreqH[df] - 0.009950f, // FreqH[e0] - 0.010347f, // FreqH[e1] - 0.013103f, // FreqH[e2] - 0.015371f, // FreqH[e3] - 0.012502f, // FreqH[e4] - 0.007436f, // FreqH[e5] - 0.018253f, // FreqH[e6] - 0.014134f, // FreqH[e7] - 0.008907f, // FreqH[e8] - 0.005411f, // FreqH[e9] - 0.009570f, // FreqH[ea] - 0.013598f, // FreqH[eb] - 0.006092f, // FreqH[ec] - 0.007409f, // FreqH[ed] - 0.008432f, // FreqH[ee] - 0.005816f, // FreqH[ef] - 0.009349f, // FreqH[f0] - 0.005472f, // FreqH[f1] - 0.007170f, // FreqH[f2] - 0.007420f, // FreqH[f3] - 0.003681f, // FreqH[f4] - 0.007523f, // FreqH[f5] - 0.004610f, // FreqH[f6] - 0.006154f, // FreqH[f7] - 0.003348f, // FreqH[f8] - 0.005074f, // FreqH[f9] - 0.005922f, // FreqH[fa] - 0.005254f, // FreqH[fb] - 0.004682f, // FreqH[fc] - 0.002093f, // FreqH[fd] - 0.000000f // FreqH[fe] - }, -0.016681f, // Lead Byte StdDev -0.010638f, // Lead Byte Mean -0.715599f, // Lead Byte Weight - { - 0.028933f, // FreqL[a1] - 0.011371f, // FreqL[a2] - 0.011053f, // FreqL[a3] - 0.007232f, // FreqL[a4] - 0.010192f, // FreqL[a5] - 0.004093f, // FreqL[a6] - 0.015043f, // FreqL[a7] - 0.011752f, // FreqL[a8] - 0.022387f, // FreqL[a9] - 0.008410f, // FreqL[aa] - 0.012448f, // FreqL[ab] - 0.007473f, // FreqL[ac] - 0.003594f, // FreqL[ad] - 0.007139f, // FreqL[ae] - 0.018912f, // FreqL[af] - 0.006083f, // FreqL[b0] - 0.003302f, // FreqL[b1] - 0.010215f, // FreqL[b2] - 0.008791f, // FreqL[b3] - 0.024236f, // FreqL[b4] - 0.014107f, // FreqL[b5] - 0.014108f, // FreqL[b6] - 0.010303f, // FreqL[b7] - 0.009728f, // FreqL[b8] - 0.007877f, // FreqL[b9] - 0.009719f, // FreqL[ba] - 0.007952f, // FreqL[bb] - 0.021028f, // FreqL[bc] - 0.005764f, // FreqL[bd] - 0.009341f, // FreqL[be] - 0.006591f, // FreqL[bf] - 0.012517f, // FreqL[c0] - 0.005921f, // FreqL[c1] - 0.008982f, // FreqL[c2] - 0.008771f, // FreqL[c3] - 0.012802f, // FreqL[c4] - 0.005926f, // FreqL[c5] - 0.008342f, // FreqL[c6] - 0.003086f, // FreqL[c7] - 0.006843f, // FreqL[c8] - 0.007576f, // FreqL[c9] - 0.004734f, // FreqL[ca] - 0.016404f, // FreqL[cb] - 0.008803f, // FreqL[cc] - 0.008071f, // FreqL[cd] - 0.005349f, // FreqL[ce] - 0.008566f, // FreqL[cf] - 0.010840f, // FreqL[d0] - 0.015401f, // FreqL[d1] - 0.031904f, // FreqL[d2] - 0.008670f, // FreqL[d3] - 0.011479f, // FreqL[d4] - 0.010936f, // FreqL[d5] - 0.007617f, // FreqL[d6] - 0.008995f, // FreqL[d7] - 0.008114f, // FreqL[d8] - 0.008658f, // FreqL[d9] - 0.005934f, // FreqL[da] - 0.010452f, // FreqL[db] - 0.009142f, // FreqL[dc] - 0.004519f, // FreqL[dd] - 0.008339f, // FreqL[de] - 0.007476f, // FreqL[df] - 0.007027f, // FreqL[e0] - 0.006025f, // FreqL[e1] - 0.021804f, // FreqL[e2] - 0.024248f, // FreqL[e3] - 0.015895f, // FreqL[e4] - 0.003768f, // FreqL[e5] - 0.010171f, // FreqL[e6] - 0.010007f, // FreqL[e7] - 0.010178f, // FreqL[e8] - 0.008316f, // FreqL[e9] - 0.006832f, // FreqL[ea] - 0.006364f, // FreqL[eb] - 0.009141f, // FreqL[ec] - 0.009148f, // FreqL[ed] - 0.012081f, // FreqL[ee] - 0.011914f, // FreqL[ef] - 0.004464f, // FreqL[f0] - 0.014257f, // FreqL[f1] - 0.006907f, // FreqL[f2] - 0.011292f, // FreqL[f3] - 0.018622f, // FreqL[f4] - 0.008149f, // FreqL[f5] - 0.004636f, // FreqL[f6] - 0.006612f, // FreqL[f7] - 0.013478f, // FreqL[f8] - 0.012614f, // FreqL[f9] - 0.005186f, // FreqL[fa] - 0.048285f, // FreqL[fb] - 0.006816f, // FreqL[fc] - 0.006743f, // FreqL[fd] - 0.008671f // FreqL[fe] - }, -0.006630f, // Trail Byte StdDev -0.010638f, // Trail Byte Mean -0.284401f // Trial Byte Weight -}; diff --git a/intl/chardet/src/GB2312Statistics.h b/intl/chardet/src/GB2312Statistics.h deleted file mode 100644 index 9411025224a..00000000000 --- a/intl/chardet/src/GB2312Statistics.h +++ /dev/null @@ -1,236 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -{ - { - 0.011628f, // FreqH[a1] - 0.000000f, // FreqH[a2] - 0.000000f, // FreqH[a3] - 0.000000f, // FreqH[a4] - 0.000000f, // FreqH[a5] - 0.000000f, // FreqH[a6] - 0.000000f, // FreqH[a7] - 0.000000f, // FreqH[a8] - 0.000000f, // FreqH[a9] - 0.000000f, // FreqH[aa] - 0.000000f, // FreqH[ab] - 0.000000f, // FreqH[ac] - 0.000000f, // FreqH[ad] - 0.000000f, // FreqH[ae] - 0.000000f, // FreqH[af] - 0.011628f, // FreqH[b0] - 0.012403f, // FreqH[b1] - 0.009302f, // FreqH[b2] - 0.003876f, // FreqH[b3] - 0.017829f, // FreqH[b4] - 0.037209f, // FreqH[b5] - 0.008527f, // FreqH[b6] - 0.010078f, // FreqH[b7] - 0.019380f, // FreqH[b8] - 0.054264f, // FreqH[b9] - 0.010078f, // FreqH[ba] - 0.041085f, // FreqH[bb] - 0.020930f, // FreqH[bc] - 0.018605f, // FreqH[bd] - 0.010078f, // FreqH[be] - 0.013178f, // FreqH[bf] - 0.016279f, // FreqH[c0] - 0.006202f, // FreqH[c1] - 0.009302f, // FreqH[c2] - 0.017054f, // FreqH[c3] - 0.011628f, // FreqH[c4] - 0.008527f, // FreqH[c5] - 0.004651f, // FreqH[c6] - 0.006202f, // FreqH[c7] - 0.017829f, // FreqH[c8] - 0.024806f, // FreqH[c9] - 0.020155f, // FreqH[ca] - 0.013953f, // FreqH[cb] - 0.032558f, // FreqH[cc] - 0.035659f, // FreqH[cd] - 0.068217f, // FreqH[ce] - 0.010853f, // FreqH[cf] - 0.036434f, // FreqH[d0] - 0.117054f, // FreqH[d1] - 0.027907f, // FreqH[d2] - 0.100775f, // FreqH[d3] - 0.010078f, // FreqH[d4] - 0.017829f, // FreqH[d5] - 0.062016f, // FreqH[d6] - 0.012403f, // FreqH[d7] - 0.000000f, // FreqH[d8] - 0.000000f, // FreqH[d9] - 0.000000f, // FreqH[da] - 0.000000f, // FreqH[db] - 0.000000f, // FreqH[dc] - 0.000000f, // FreqH[dd] - 0.000000f, // FreqH[de] - 0.000000f, // FreqH[df] - 0.000000f, // FreqH[e0] - 0.000000f, // FreqH[e1] - 0.000000f, // FreqH[e2] - 0.000000f, // FreqH[e3] - 0.000000f, // FreqH[e4] - 0.000000f, // FreqH[e5] - 0.000000f, // FreqH[e6] - 0.000000f, // FreqH[e7] - 0.000000f, // FreqH[e8] - 0.000000f, // FreqH[e9] - 0.001550f, // FreqH[ea] - 0.000000f, // FreqH[eb] - 0.000000f, // FreqH[ec] - 0.000000f, // FreqH[ed] - 0.000000f, // FreqH[ee] - 0.000000f, // FreqH[ef] - 0.000000f, // FreqH[f0] - 0.000000f, // FreqH[f1] - 0.000000f, // FreqH[f2] - 0.000000f, // FreqH[f3] - 0.000000f, // FreqH[f4] - 0.000000f, // FreqH[f5] - 0.000000f, // FreqH[f6] - 0.000000f, // FreqH[f7] - 0.000000f, // FreqH[f8] - 0.000000f, // FreqH[f9] - 0.000000f, // FreqH[fa] - 0.000000f, // FreqH[fb] - 0.000000f, // FreqH[fc] - 0.000000f, // FreqH[fd] - 0.000000f // FreqH[fe] - }, -0.020081f, // Lead Byte StdDev -0.010638f, // Lead Byte Mean -0.586533f, // Lead Byte Weight - { - 0.006202f, // FreqL[a1] - 0.031008f, // FreqL[a2] - 0.005426f, // FreqL[a3] - 0.003101f, // FreqL[a4] - 0.001550f, // FreqL[a5] - 0.003101f, // FreqL[a6] - 0.082171f, // FreqL[a7] - 0.014729f, // FreqL[a8] - 0.006977f, // FreqL[a9] - 0.001550f, // FreqL[aa] - 0.013953f, // FreqL[ab] - 0.000000f, // FreqL[ac] - 0.013953f, // FreqL[ad] - 0.010078f, // FreqL[ae] - 0.008527f, // FreqL[af] - 0.006977f, // FreqL[b0] - 0.004651f, // FreqL[b1] - 0.003101f, // FreqL[b2] - 0.003101f, // FreqL[b3] - 0.003101f, // FreqL[b4] - 0.008527f, // FreqL[b5] - 0.003101f, // FreqL[b6] - 0.005426f, // FreqL[b7] - 0.005426f, // FreqL[b8] - 0.005426f, // FreqL[b9] - 0.003101f, // FreqL[ba] - 0.001550f, // FreqL[bb] - 0.006202f, // FreqL[bc] - 0.014729f, // FreqL[bd] - 0.010853f, // FreqL[be] - 0.000000f, // FreqL[bf] - 0.011628f, // FreqL[c0] - 0.000000f, // FreqL[c1] - 0.031783f, // FreqL[c2] - 0.013953f, // FreqL[c3] - 0.030233f, // FreqL[c4] - 0.039535f, // FreqL[c5] - 0.008527f, // FreqL[c6] - 0.015504f, // FreqL[c7] - 0.000000f, // FreqL[c8] - 0.003101f, // FreqL[c9] - 0.008527f, // FreqL[ca] - 0.016279f, // FreqL[cb] - 0.005426f, // FreqL[cc] - 0.001550f, // FreqL[cd] - 0.013953f, // FreqL[ce] - 0.013953f, // FreqL[cf] - 0.044961f, // FreqL[d0] - 0.003101f, // FreqL[d1] - 0.004651f, // FreqL[d2] - 0.006977f, // FreqL[d3] - 0.001550f, // FreqL[d4] - 0.005426f, // FreqL[d5] - 0.012403f, // FreqL[d6] - 0.001550f, // FreqL[d7] - 0.015504f, // FreqL[d8] - 0.000000f, // FreqL[d9] - 0.006202f, // FreqL[da] - 0.001550f, // FreqL[db] - 0.000000f, // FreqL[dc] - 0.007752f, // FreqL[dd] - 0.006977f, // FreqL[de] - 0.001550f, // FreqL[df] - 0.009302f, // FreqL[e0] - 0.011628f, // FreqL[e1] - 0.004651f, // FreqL[e2] - 0.010853f, // FreqL[e3] - 0.012403f, // FreqL[e4] - 0.017829f, // FreqL[e5] - 0.005426f, // FreqL[e6] - 0.024806f, // FreqL[e7] - 0.000000f, // FreqL[e8] - 0.006202f, // FreqL[e9] - 0.000000f, // FreqL[ea] - 0.082171f, // FreqL[eb] - 0.015504f, // FreqL[ec] - 0.004651f, // FreqL[ed] - 0.000000f, // FreqL[ee] - 0.006977f, // FreqL[ef] - 0.004651f, // FreqL[f0] - 0.000000f, // FreqL[f1] - 0.008527f, // FreqL[f2] - 0.012403f, // FreqL[f3] - 0.004651f, // FreqL[f4] - 0.003876f, // FreqL[f5] - 0.003101f, // FreqL[f6] - 0.022481f, // FreqL[f7] - 0.024031f, // FreqL[f8] - 0.001550f, // FreqL[f9] - 0.047287f, // FreqL[fa] - 0.009302f, // FreqL[fb] - 0.001550f, // FreqL[fc] - 0.005426f, // FreqL[fd] - 0.017054f // FreqL[fe] - }, -0.014156f, // Trail Byte StdDev -0.010638f, // Trail Byte Mean -0.413467f // Trial Byte Weight -}; diff --git a/intl/chardet/src/Makefile.in b/intl/chardet/src/Makefile.in index f8cd8d91b6f..b084cfdb334 100644 --- a/intl/chardet/src/Makefile.in +++ b/intl/chardet/src/Makefile.in @@ -69,7 +69,6 @@ CPPSRCS = \ nsMetaCharsetObserver.cpp \ nsDetectionAdaptor.cpp \ nsDebugDetector.cpp \ - nsPSMDetectors.cpp \ nsCyrillicDetector.cpp \ nsDocumentCharsetInfo.cpp \ nsChardetModule.cpp \ diff --git a/intl/chardet/src/nsBIG5Verifier.h b/intl/chardet/src/nsBIG5Verifier.h deleted file mode 100644 index 175466b07da..00000000000 --- a/intl/chardet/src/nsBIG5Verifier.h +++ /dev/null @@ -1,105 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -/* - * DO NOT EDIT THIS DOCUMENT MANUALLY !!! - * THIS FILE IS AUTOMATICALLY GENERATED BY THE TOOLS UNDER - * mozilla/intl/chardet/tools/ - * Please contact ftang@netscape.com or mozilla-i18n@mozilla.org - * if you have any question. Thanks - */ -#include "nsVerifier.h" -static const PRUint32 BIG5_cls [ 256 / 8 ] = { -PCK4BITS(1,1,1,1,1,1,1,1), // 00 - 07 -PCK4BITS(1,1,1,1,1,1,0,0), // 08 - 0f -PCK4BITS(1,1,1,1,1,1,1,1), // 10 - 17 -PCK4BITS(1,1,1,0,1,1,1,1), // 18 - 1f -PCK4BITS(1,1,1,1,1,1,1,1), // 20 - 27 -PCK4BITS(1,1,1,1,1,1,1,1), // 28 - 2f -PCK4BITS(1,1,1,1,1,1,1,1), // 30 - 37 -PCK4BITS(1,1,1,1,1,1,1,1), // 38 - 3f -PCK4BITS(2,2,2,2,2,2,2,2), // 40 - 47 -PCK4BITS(2,2,2,2,2,2,2,2), // 48 - 4f -PCK4BITS(2,2,2,2,2,2,2,2), // 50 - 57 -PCK4BITS(2,2,2,2,2,2,2,2), // 58 - 5f -PCK4BITS(2,2,2,2,2,2,2,2), // 60 - 67 -PCK4BITS(2,2,2,2,2,2,2,2), // 68 - 6f -PCK4BITS(2,2,2,2,2,2,2,2), // 70 - 77 -PCK4BITS(2,2,2,2,2,2,2,1), // 78 - 7f -PCK4BITS(4,4,4,4,4,4,4,4), // 80 - 87 -PCK4BITS(4,4,4,4,4,4,4,4), // 88 - 8f -PCK4BITS(4,4,4,4,4,4,4,4), // 90 - 97 -PCK4BITS(4,4,4,4,4,4,4,4), // 98 - 9f -PCK4BITS(4,3,3,3,3,3,3,3), // a0 - a7 -PCK4BITS(3,3,3,3,3,3,3,3), // a8 - af -PCK4BITS(3,3,3,3,3,3,3,3), // b0 - b7 -PCK4BITS(3,3,3,3,3,3,3,3), // b8 - bf -PCK4BITS(3,3,3,3,3,3,3,3), // c0 - c7 -PCK4BITS(3,3,3,3,3,3,3,3), // c8 - cf -PCK4BITS(3,3,3,3,3,3,3,3), // d0 - d7 -PCK4BITS(3,3,3,3,3,3,3,3), // d8 - df -PCK4BITS(3,3,3,3,3,3,3,3), // e0 - e7 -PCK4BITS(3,3,3,3,3,3,3,3), // e8 - ef -PCK4BITS(3,3,3,3,3,3,3,3), // f0 - f7 -PCK4BITS(3,3,3,3,3,3,3,0) // f8 - ff -}; - - -static const PRUint32 BIG5_st [ 3] = { -PCK4BITS(eError,eStart,eStart, 3,eError,eError,eError,eError),//00-07 -PCK4BITS(eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError),//08-0f -PCK4BITS(eError,eStart,eStart,eStart,eStart,eStart,eStart,eStart) //10-17 -}; - - -static nsVerifier nsBIG5Verifier = { - "Big5", - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - BIG5_cls - }, - 5, - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - BIG5_st - } -}; diff --git a/intl/chardet/src/nsCP1252Verifier.h b/intl/chardet/src/nsCP1252Verifier.h deleted file mode 100644 index 9059565eb51..00000000000 --- a/intl/chardet/src/nsCP1252Verifier.h +++ /dev/null @@ -1,105 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -/* - * DO NOT EDIT THIS DOCUMENT MANUALLY !!! - * THIS FILE IS AUTOMATICALLY GENERATED BY THE TOOLS UNDER - * mozilla/intl/chardet/tools/ - * Please contact ftang@netscape.com or mozilla-i18n@mozilla.org - * if you have any question. Thanks - */ -#include "nsVerifier.h" -static const PRUint32 CP1252_cls [ 256 / 8 ] = { -PCK4BITS(2,2,2,2,2,2,2,2), // 00 - 07 -PCK4BITS(2,2,2,2,2,2,0,0), // 08 - 0f -PCK4BITS(2,2,2,2,2,2,2,2), // 10 - 17 -PCK4BITS(2,2,2,0,2,2,2,2), // 18 - 1f -PCK4BITS(2,2,2,2,2,2,2,2), // 20 - 27 -PCK4BITS(2,2,2,2,2,2,2,2), // 28 - 2f -PCK4BITS(2,2,2,2,2,2,2,2), // 30 - 37 -PCK4BITS(2,2,2,2,2,2,2,2), // 38 - 3f -PCK4BITS(2,2,2,2,2,2,2,2), // 40 - 47 -PCK4BITS(2,2,2,2,2,2,2,2), // 48 - 4f -PCK4BITS(2,2,2,2,2,2,2,2), // 50 - 57 -PCK4BITS(2,2,2,2,2,2,2,2), // 58 - 5f -PCK4BITS(2,2,2,2,2,2,2,2), // 60 - 67 -PCK4BITS(2,2,2,2,2,2,2,2), // 68 - 6f -PCK4BITS(2,2,2,2,2,2,2,2), // 70 - 77 -PCK4BITS(2,2,2,2,2,2,2,2), // 78 - 7f -PCK4BITS(2,0,2,2,2,2,2,2), // 80 - 87 -PCK4BITS(2,2,1,2,1,0,1,0), // 88 - 8f -PCK4BITS(0,2,2,2,2,2,2,2), // 90 - 97 -PCK4BITS(2,2,1,2,1,0,1,1), // 98 - 9f -PCK4BITS(2,2,2,2,2,2,2,2), // a0 - a7 -PCK4BITS(2,2,2,2,2,2,2,2), // a8 - af -PCK4BITS(2,2,2,2,2,2,2,2), // b0 - b7 -PCK4BITS(2,2,2,2,2,2,2,2), // b8 - bf -PCK4BITS(1,1,1,1,1,1,1,1), // c0 - c7 -PCK4BITS(1,1,1,1,1,1,1,1), // c8 - cf -PCK4BITS(1,1,1,1,1,1,1,2), // d0 - d7 -PCK4BITS(1,1,1,1,1,1,1,1), // d8 - df -PCK4BITS(1,1,1,1,1,1,1,1), // e0 - e7 -PCK4BITS(1,1,1,1,1,1,1,1), // e8 - ef -PCK4BITS(1,1,1,1,1,1,1,2), // f0 - f7 -PCK4BITS(1,1,1,1,1,1,1,1) // f8 - ff -}; - - -static const PRUint32 CP1252_st [ 3] = { -PCK4BITS(eError, 3,eStart,eError,eError,eError,eItsMe,eItsMe),//00-07 -PCK4BITS(eItsMe,eError, 4,eStart,eError, 5, 4,eError),//08-0f -PCK4BITS(eError, 4,eStart,eStart,eStart,eStart,eStart,eStart) //10-17 -}; - - -static nsVerifier nsCP1252Verifier = { - "windows-1252", - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - CP1252_cls - }, - 3, - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - CP1252_st - } -}; diff --git a/intl/chardet/src/nsCharDetConstructors.h b/intl/chardet/src/nsCharDetConstructors.h index 948002ef4ad..35129970054 100644 --- a/intl/chardet/src/nsCharDetConstructors.h +++ b/intl/chardet/src/nsCharDetConstructors.h @@ -56,7 +56,6 @@ #include "nsICharsetDetectionObserver.h" #include "nsDetectionAdaptor.h" #include "nsIStringCharsetDetector.h" -#include "nsPSMDetectors.h" #include "nsCyrillicDetector.h" #include "nsDocumentCharsetInfoCID.h" #include "nsXMLEncodingCID.h" @@ -66,18 +65,6 @@ NS_GENERIC_FACTORY_CONSTRUCTOR(nsMetaCharsetObserver) NS_GENERIC_FACTORY_CONSTRUCTOR(nsDocumentCharsetInfo) NS_GENERIC_FACTORY_CONSTRUCTOR(nsXMLEncodingObserver) NS_GENERIC_FACTORY_CONSTRUCTOR(nsDetectionAdaptor) -NS_GENERIC_FACTORY_CONSTRUCTOR(nsJAPSMDetector) -NS_GENERIC_FACTORY_CONSTRUCTOR(nsJAStringPSMDetector) -NS_GENERIC_FACTORY_CONSTRUCTOR(nsKOPSMDetector) -NS_GENERIC_FACTORY_CONSTRUCTOR(nsKOStringPSMDetector) -NS_GENERIC_FACTORY_CONSTRUCTOR(nsZHTWPSMDetector) -NS_GENERIC_FACTORY_CONSTRUCTOR(nsZHTWStringPSMDetector) -NS_GENERIC_FACTORY_CONSTRUCTOR(nsZHCNPSMDetector) -NS_GENERIC_FACTORY_CONSTRUCTOR(nsZHCNStringPSMDetector) -NS_GENERIC_FACTORY_CONSTRUCTOR(nsZHPSMDetector) -NS_GENERIC_FACTORY_CONSTRUCTOR(nsZHStringPSMDetector) -NS_GENERIC_FACTORY_CONSTRUCTOR(nsCJKPSMDetector) -NS_GENERIC_FACTORY_CONSTRUCTOR(nsCJKStringPSMDetector) NS_GENERIC_FACTORY_CONSTRUCTOR(nsRUProbDetector) NS_GENERIC_FACTORY_CONSTRUCTOR(nsUKProbDetector) NS_GENERIC_FACTORY_CONSTRUCTOR(nsRUStringProbDetector) diff --git a/intl/chardet/src/nsChardetModule.cpp b/intl/chardet/src/nsChardetModule.cpp index 8a0e00d6a06..a91d5248ec9 100644 --- a/intl/chardet/src/nsChardetModule.cpp +++ b/intl/chardet/src/nsChardetModule.cpp @@ -102,78 +102,6 @@ nsDetectionAdaptorRegistrationProc(nsIComponentManager *aCompMgr, return AddCategoryEntry(NS_CHARSET_DETECTOR_CATEGORY, "off", "off"); } -static NS_METHOD -nsJAPSMDetectorRegistrationProc(nsIComponentManager *aCompMgr, - nsIFile *aPath, - const char *registryLocation, - const char *componentType, - const nsModuleComponentInfo *info) -{ - return AddCategoryEntry(NS_CHARSET_DETECTOR_CATEGORY, - "ja_parallel_state_machine", - info->mContractID); -} - -static NS_METHOD -nsKOPSMDetectorRegistrationProc(nsIComponentManager *aCompMgr, - nsIFile *aPath, - const char *registryLocation, - const char *componentType, - const nsModuleComponentInfo *info) -{ - return AddCategoryEntry(NS_CHARSET_DETECTOR_CATEGORY, - "ko_parallel_state_machine", - info->mContractID); -} - -static NS_METHOD -nsZHTWPSMDetectorRegistrationProc(nsIComponentManager *aCompMgr, - nsIFile *aPath, - const char *registryLocation, - const char *componentType, - const nsModuleComponentInfo *info) -{ - return AddCategoryEntry(NS_CHARSET_DETECTOR_CATEGORY, - "zhtw_parallel_state_machine", - info->mContractID); -} - -static NS_METHOD -nsZHCNPSMDetectorRegistrationProc(nsIComponentManager *aCompMgr, - nsIFile *aPath, - const char *registryLocation, - const char *componentType, - const nsModuleComponentInfo *info) -{ - return AddCategoryEntry(NS_CHARSET_DETECTOR_CATEGORY, - "zhcn_parallel_state_machine", - info->mContractID); -} - -static NS_METHOD -nsZHPSMDetectorRegistrationProc(nsIComponentManager *aCompMgr, - nsIFile *aPath, - const char *registryLocation, - const char *componentType, - const nsModuleComponentInfo *info) -{ - return AddCategoryEntry(NS_CHARSET_DETECTOR_CATEGORY, - "zh_parallel_state_machine", - info->mContractID); -} - -static NS_METHOD -nsCJKPSMDetectorRegistrationProc(nsIComponentManager *aCompMgr, - nsIFile *aPath, - const char *registryLocation, - const char *componentType, - const nsModuleComponentInfo *info) -{ - return AddCategoryEntry(NS_CHARSET_DETECTOR_CATEGORY, - "cjk_parallel_state_machine", - info->mContractID); -} - static NS_METHOD nsRUProbDetectorRegistrationProc(nsIComponentManager *aCompMgr, nsIFile *aPath, @@ -214,42 +142,6 @@ static nsModuleComponentInfo components[] = { "Charset Detection Adaptor", NS_CHARSET_DETECTION_ADAPTOR_CID, NS_CHARSET_DETECTION_ADAPTOR_CONTRACTID, nsDetectionAdaptorConstructor, nsDetectionAdaptorRegistrationProc, NULL}, - { "PSM based Japanese Charset Detector", NS_JA_PSMDETECTOR_CID, - NS_CHARSET_DETECTOR_CONTRACTID_BASE "ja_parallel_state_machine", nsJAPSMDetectorConstructor, - nsJAPSMDetectorRegistrationProc, NULL}, - { "PSM based Japanese String Charset Detector", NS_JA_STRING_PSMDETECTOR_CID, - NS_STRCDETECTOR_CONTRACTID_BASE "ja_parallel_state_machine", nsJAStringPSMDetectorConstructor, - NULL, NULL}, - { "PSM based Korean Charset Detector", NS_KO_PSMDETECTOR_CID, - NS_CHARSET_DETECTOR_CONTRACTID_BASE "ko_parallel_state_machine", nsKOPSMDetectorConstructor, - nsKOPSMDetectorRegistrationProc, NULL}, - { "PSM based Korean String Charset Detector", NS_KO_STRING_PSMDETECTOR_CID, - NS_STRCDETECTOR_CONTRACTID_BASE "ko_parallel_state_machine", nsKOStringPSMDetectorConstructor, - NULL, NULL}, - { "PSM based Traditional Chinese Charset Detector", NS_ZHTW_PSMDETECTOR_CID, - NS_CHARSET_DETECTOR_CONTRACTID_BASE "zhtw_parallel_state_machine", nsZHTWPSMDetectorConstructor, - nsZHTWPSMDetectorRegistrationProc, NULL}, - { "PSM based Traditional Chinese String Charset Detector", NS_ZHTW_STRING_PSMDETECTOR_CID, - NS_STRCDETECTOR_CONTRACTID_BASE "zhtw_parallel_state_machine", nsZHTWStringPSMDetectorConstructor, - NULL, NULL}, - { "PSM based Simplified Chinese Charset Detector", NS_ZHCN_PSMDETECTOR_CID, - NS_CHARSET_DETECTOR_CONTRACTID_BASE "zhcn_parallel_state_machine", nsZHCNPSMDetectorConstructor, - nsZHCNPSMDetectorRegistrationProc, NULL}, - { "PSM based Simplified Chinese String Charset Detector", NS_ZHCN_STRING_PSMDETECTOR_CID, - NS_STRCDETECTOR_CONTRACTID_BASE "zhcn_parallel_state_machine", nsZHCNStringPSMDetectorConstructor, - NULL, NULL}, - { "PSM based Chinese Charset Detector", NS_ZH_PSMDETECTOR_CID, - NS_CHARSET_DETECTOR_CONTRACTID_BASE "zh_parallel_state_machine", nsZHPSMDetectorConstructor, - nsZHPSMDetectorRegistrationProc, NULL}, - { "PSM based Chinese String Charset Detector", NS_ZH_STRING_PSMDETECTOR_CID, - NS_STRCDETECTOR_CONTRACTID_BASE "zh_parallel_state_machine", nsZHStringPSMDetectorConstructor, - NULL, NULL}, - { "PSM based CJK Charset Detector", NS_CJK_PSMDETECTOR_CID, - NS_CHARSET_DETECTOR_CONTRACTID_BASE "cjk_parallel_state_machine", nsCJKPSMDetectorConstructor, - nsCJKPSMDetectorRegistrationProc, NULL}, - { "PSM based CJK String Charset Detector", NS_CJK_STRING_PSMDETECTOR_CID, - NS_STRCDETECTOR_CONTRACTID_BASE "cjk_parallel_state_machine", nsCJKStringPSMDetectorConstructor, - NULL, NULL}, { "Probability based Russian Charset Detector", NS_RU_PROBDETECTOR_CID, NS_CHARSET_DETECTOR_CONTRACTID_BASE "ruprob", nsRUProbDetectorConstructor, nsRUProbDetectorRegistrationProc, NULL}, diff --git a/intl/chardet/src/nsEUCJPVerifier.h b/intl/chardet/src/nsEUCJPVerifier.h deleted file mode 100644 index 79974e3eab9..00000000000 --- a/intl/chardet/src/nsEUCJPVerifier.h +++ /dev/null @@ -1,107 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -/* - * DO NOT EDIT THIS DOCUMENT MANUALLY !!! - * THIS FILE IS AUTOMATICALLY GENERATED BY THE TOOLS UNDER - * mozilla/intl/chardet/tools/ - * Please contact ftang@netscape.com or mozilla-i18n@mozilla.org - * if you have any question. Thanks - */ -#include "nsVerifier.h" -static const PRUint32 EUCJP_cls [ 256 / 8 ] = { -PCK4BITS(4,4,4,4,4,4,4,4), // 00 - 07 -PCK4BITS(4,4,4,4,4,4,5,5), // 08 - 0f -PCK4BITS(4,4,4,4,4,4,4,4), // 10 - 17 -PCK4BITS(4,4,4,5,4,4,4,4), // 18 - 1f -PCK4BITS(4,4,4,4,4,4,4,4), // 20 - 27 -PCK4BITS(4,4,4,4,4,4,4,4), // 28 - 2f -PCK4BITS(4,4,4,4,4,4,4,4), // 30 - 37 -PCK4BITS(4,4,4,4,4,4,4,4), // 38 - 3f -PCK4BITS(4,4,4,4,4,4,4,4), // 40 - 47 -PCK4BITS(4,4,4,4,4,4,4,4), // 48 - 4f -PCK4BITS(4,4,4,4,4,4,4,4), // 50 - 57 -PCK4BITS(4,4,4,4,4,4,4,4), // 58 - 5f -PCK4BITS(4,4,4,4,4,4,4,4), // 60 - 67 -PCK4BITS(4,4,4,4,4,4,4,4), // 68 - 6f -PCK4BITS(4,4,4,4,4,4,4,4), // 70 - 77 -PCK4BITS(4,4,4,4,4,4,4,4), // 78 - 7f -PCK4BITS(5,5,5,5,5,5,5,5), // 80 - 87 -PCK4BITS(5,5,5,5,5,5,1,3), // 88 - 8f -PCK4BITS(5,5,5,5,5,5,5,5), // 90 - 97 -PCK4BITS(5,5,5,5,5,5,5,5), // 98 - 9f -PCK4BITS(5,2,2,2,2,2,2,2), // a0 - a7 -PCK4BITS(2,2,2,2,2,2,2,2), // a8 - af -PCK4BITS(2,2,2,2,2,2,2,2), // b0 - b7 -PCK4BITS(2,2,2,2,2,2,2,2), // b8 - bf -PCK4BITS(2,2,2,2,2,2,2,2), // c0 - c7 -PCK4BITS(2,2,2,2,2,2,2,2), // c8 - cf -PCK4BITS(2,2,2,2,2,2,2,2), // d0 - d7 -PCK4BITS(2,2,2,2,2,2,2,2), // d8 - df -PCK4BITS(0,0,0,0,0,0,0,0), // e0 - e7 -PCK4BITS(0,0,0,0,0,0,0,0), // e8 - ef -PCK4BITS(0,0,0,0,0,0,0,0), // f0 - f7 -PCK4BITS(0,0,0,0,0,0,0,5) // f8 - ff -}; - - -static const PRUint32 EUCJP_st [ 5] = { -PCK4BITS( 3, 4, 3, 5,eStart,eError,eError,eError),//00-07 -PCK4BITS(eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe),//08-0f -PCK4BITS(eItsMe,eItsMe,eStart,eError,eStart,eError,eError,eError),//10-17 -PCK4BITS(eError,eError,eStart,eError,eError,eError, 3,eError),//18-1f -PCK4BITS( 3,eError,eError,eError,eStart,eStart,eStart,eStart) //20-27 -}; - - -static nsVerifier nsEUCJPVerifier = { - "EUC-JP", - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - EUCJP_cls - }, - 6, - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - EUCJP_st - } -}; diff --git a/intl/chardet/src/nsEUCKRVerifier.h b/intl/chardet/src/nsEUCKRVerifier.h deleted file mode 100644 index c4f83b40e86..00000000000 --- a/intl/chardet/src/nsEUCKRVerifier.h +++ /dev/null @@ -1,104 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -/* - * DO NOT EDIT THIS DOCUMENT MANUALLY !!! - * THIS FILE IS AUTOMATICALLY GENERATED BY THE TOOLS UNDER - * mozilla/intl/chardet/tools/ - * Please contact ftang@netscape.com or mozilla-i18n@mozilla.org - * if you have any question. Thanks - */ -#include "nsVerifier.h" -static const PRUint32 EUCKR_cls [ 256 / 8 ] = { -PCK4BITS(1,1,1,1,1,1,1,1), // 00 - 07 -PCK4BITS(1,1,1,1,1,1,0,0), // 08 - 0f -PCK4BITS(1,1,1,1,1,1,1,1), // 10 - 17 -PCK4BITS(1,1,1,0,1,1,1,1), // 18 - 1f -PCK4BITS(1,1,1,1,1,1,1,1), // 20 - 27 -PCK4BITS(1,1,1,1,1,1,1,1), // 28 - 2f -PCK4BITS(1,1,1,1,1,1,1,1), // 30 - 37 -PCK4BITS(1,1,1,1,1,1,1,1), // 38 - 3f -PCK4BITS(1,1,1,1,1,1,1,1), // 40 - 47 -PCK4BITS(1,1,1,1,1,1,1,1), // 48 - 4f -PCK4BITS(1,1,1,1,1,1,1,1), // 50 - 57 -PCK4BITS(1,1,1,1,1,1,1,1), // 58 - 5f -PCK4BITS(1,1,1,1,1,1,1,1), // 60 - 67 -PCK4BITS(1,1,1,1,1,1,1,1), // 68 - 6f -PCK4BITS(1,1,1,1,1,1,1,1), // 70 - 77 -PCK4BITS(1,1,1,1,1,1,1,1), // 78 - 7f -PCK4BITS(0,0,0,0,0,0,0,0), // 80 - 87 -PCK4BITS(0,0,0,0,0,0,0,0), // 88 - 8f -PCK4BITS(0,0,0,0,0,0,0,0), // 90 - 97 -PCK4BITS(0,0,0,0,0,0,0,0), // 98 - 9f -PCK4BITS(0,2,2,2,2,2,2,2), // a0 - a7 -PCK4BITS(2,2,2,2,2,3,3,3), // a8 - af -PCK4BITS(2,2,2,2,2,2,2,2), // b0 - b7 -PCK4BITS(2,2,2,2,2,2,2,2), // b8 - bf -PCK4BITS(2,2,2,2,2,2,2,2), // c0 - c7 -PCK4BITS(2,3,2,2,2,2,2,2), // c8 - cf -PCK4BITS(2,2,2,2,2,2,2,2), // d0 - d7 -PCK4BITS(2,2,2,2,2,2,2,2), // d8 - df -PCK4BITS(2,2,2,2,2,2,2,2), // e0 - e7 -PCK4BITS(2,2,2,2,2,2,2,2), // e8 - ef -PCK4BITS(2,2,2,2,2,2,2,2), // f0 - f7 -PCK4BITS(2,2,2,2,2,2,2,0) // f8 - ff -}; - - -static const PRUint32 EUCKR_st [ 2] = { -PCK4BITS(eError,eStart, 3,eError,eError,eError,eError,eError),//00-07 -PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,eStart,eStart) //08-0f -}; - - -static nsVerifier nsEUCKRVerifier = { - "EUC-KR", - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - EUCKR_cls - }, - 4, - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - EUCKR_st - } -}; diff --git a/intl/chardet/src/nsEUCTWVerifier.h b/intl/chardet/src/nsEUCTWVerifier.h deleted file mode 100644 index 6fee6e604e4..00000000000 --- a/intl/chardet/src/nsEUCTWVerifier.h +++ /dev/null @@ -1,108 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -/* - * DO NOT EDIT THIS DOCUMENT MANUALLY !!! - * THIS FILE IS AUTOMATICALLY GENERATED BY THE TOOLS UNDER - * mozilla/intl/chardet/tools/ - * Please contact ftang@netscape.com or mozilla-i18n@mozilla.org - * if you have any question. Thanks - */ -#include "nsVerifier.h" -static const PRUint32 EUCTW_cls [ 256 / 8 ] = { -PCK4BITS(2,2,2,2,2,2,2,2), // 00 - 07 -PCK4BITS(2,2,2,2,2,2,0,0), // 08 - 0f -PCK4BITS(2,2,2,2,2,2,2,2), // 10 - 17 -PCK4BITS(2,2,2,0,2,2,2,2), // 18 - 1f -PCK4BITS(2,2,2,2,2,2,2,2), // 20 - 27 -PCK4BITS(2,2,2,2,2,2,2,2), // 28 - 2f -PCK4BITS(2,2,2,2,2,2,2,2), // 30 - 37 -PCK4BITS(2,2,2,2,2,2,2,2), // 38 - 3f -PCK4BITS(2,2,2,2,2,2,2,2), // 40 - 47 -PCK4BITS(2,2,2,2,2,2,2,2), // 48 - 4f -PCK4BITS(2,2,2,2,2,2,2,2), // 50 - 57 -PCK4BITS(2,2,2,2,2,2,2,2), // 58 - 5f -PCK4BITS(2,2,2,2,2,2,2,2), // 60 - 67 -PCK4BITS(2,2,2,2,2,2,2,2), // 68 - 6f -PCK4BITS(2,2,2,2,2,2,2,2), // 70 - 77 -PCK4BITS(2,2,2,2,2,2,2,2), // 78 - 7f -PCK4BITS(0,0,0,0,0,0,0,0), // 80 - 87 -PCK4BITS(0,0,0,0,0,0,6,0), // 88 - 8f -PCK4BITS(0,0,0,0,0,0,0,0), // 90 - 97 -PCK4BITS(0,0,0,0,0,0,0,0), // 98 - 9f -PCK4BITS(0,3,4,4,4,4,4,4), // a0 - a7 -PCK4BITS(5,5,1,1,1,1,1,1), // a8 - af -PCK4BITS(1,1,1,1,1,1,1,1), // b0 - b7 -PCK4BITS(1,1,1,1,1,1,1,1), // b8 - bf -PCK4BITS(1,1,3,1,3,3,3,3), // c0 - c7 -PCK4BITS(3,3,3,3,3,3,3,3), // c8 - cf -PCK4BITS(3,3,3,3,3,3,3,3), // d0 - d7 -PCK4BITS(3,3,3,3,3,3,3,3), // d8 - df -PCK4BITS(3,3,3,3,3,3,3,3), // e0 - e7 -PCK4BITS(3,3,3,3,3,3,3,3), // e8 - ef -PCK4BITS(3,3,3,3,3,3,3,3), // f0 - f7 -PCK4BITS(3,3,3,3,3,3,3,0) // f8 - ff -}; - - -static const PRUint32 EUCTW_st [ 6] = { -PCK4BITS(eError,eError,eStart, 3, 3, 3, 4,eError),//00-07 -PCK4BITS(eError,eError,eError,eError,eError,eError,eItsMe,eItsMe),//08-0f -PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError,eStart,eError),//10-17 -PCK4BITS(eStart,eStart,eStart,eError,eError,eError,eError,eError),//18-1f -PCK4BITS( 5,eError,eError,eError,eStart,eError,eStart,eStart),//20-27 -PCK4BITS(eStart,eError,eStart,eStart,eStart,eStart,eStart,eStart) //28-2f -}; - - -static nsVerifier nsEUCTWVerifier = { - "x-euc-tw", - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - EUCTW_cls - }, - 7, - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - EUCTW_st - } -}; diff --git a/intl/chardet/src/nsGB18030Verifier.h b/intl/chardet/src/nsGB18030Verifier.h deleted file mode 100644 index b88314a51c7..00000000000 --- a/intl/chardet/src/nsGB18030Verifier.h +++ /dev/null @@ -1,108 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -/* - * DO NOT EDIT THIS DOCUMENT MANUALLY !!! - * THIS FILE IS AUTOMATICALLY GENERATED BY THE TOOLS UNDER - * mozilla/intl/chardet/tools/ - * Please contact ftang@netscape.com or mozilla-i18n@mozilla.org - * if you have any question. Thanks - */ -#include "nsVerifier.h" -static const PRUint32 gb18030_cls [ 256 / 8 ] = { -PCK4BITS(1,1,1,1,1,1,1,1), // 00 - 07 -PCK4BITS(1,1,1,1,1,1,0,0), // 08 - 0f -PCK4BITS(1,1,1,1,1,1,1,1), // 10 - 17 -PCK4BITS(1,1,1,0,1,1,1,1), // 18 - 1f -PCK4BITS(1,1,1,1,1,1,1,1), // 20 - 27 -PCK4BITS(1,1,1,1,1,1,1,1), // 28 - 2f -PCK4BITS(3,3,3,3,3,3,3,3), // 30 - 37 -PCK4BITS(3,3,1,1,1,1,1,1), // 38 - 3f -PCK4BITS(2,2,2,2,2,2,2,2), // 40 - 47 -PCK4BITS(2,2,2,2,2,2,2,2), // 48 - 4f -PCK4BITS(2,2,2,2,2,2,2,2), // 50 - 57 -PCK4BITS(2,2,2,2,2,2,2,2), // 58 - 5f -PCK4BITS(2,2,2,2,2,2,2,2), // 60 - 67 -PCK4BITS(2,2,2,2,2,2,2,2), // 68 - 6f -PCK4BITS(2,2,2,2,2,2,2,2), // 70 - 77 -PCK4BITS(2,2,2,2,2,2,2,4), // 78 - 7f -PCK4BITS(5,6,6,6,6,6,6,6), // 80 - 87 -PCK4BITS(6,6,6,6,6,6,6,6), // 88 - 8f -PCK4BITS(6,6,6,6,6,6,6,6), // 90 - 97 -PCK4BITS(6,6,6,6,6,6,6,6), // 98 - 9f -PCK4BITS(6,6,6,6,6,6,6,6), // a0 - a7 -PCK4BITS(6,6,6,6,6,6,6,6), // a8 - af -PCK4BITS(6,6,6,6,6,6,6,6), // b0 - b7 -PCK4BITS(6,6,6,6,6,6,6,6), // b8 - bf -PCK4BITS(6,6,6,6,6,6,6,6), // c0 - c7 -PCK4BITS(6,6,6,6,6,6,6,6), // c8 - cf -PCK4BITS(6,6,6,6,6,6,6,6), // d0 - d7 -PCK4BITS(6,6,6,6,6,6,6,6), // d8 - df -PCK4BITS(6,6,6,6,6,6,6,6), // e0 - e7 -PCK4BITS(6,6,6,6,6,6,6,6), // e8 - ef -PCK4BITS(6,6,6,6,6,6,6,6), // f0 - f7 -PCK4BITS(6,6,6,6,6,6,6,0) // f8 - ff -}; - - -static const PRUint32 gb18030_st [ 6] = { -PCK4BITS(eError,eStart,eStart,eStart,eStart,eStart, 3,eError),//00-07 -PCK4BITS(eError,eError,eError,eError,eError,eError,eItsMe,eItsMe),//08-0f -PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,eStart),//10-17 -PCK4BITS( 4,eError,eStart,eStart,eError,eError,eError,eError),//18-1f -PCK4BITS(eError,eError, 5,eError,eError,eError,eItsMe,eError),//20-27 -PCK4BITS(eError,eError,eStart,eStart,eStart,eStart,eStart,eStart) //28-2f -}; - - -static nsVerifier nsGB18030Verifier = { - "gb18030", - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - gb18030_cls - }, - 7, - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - gb18030_st - } -}; diff --git a/intl/chardet/src/nsGB2312Verifier.h b/intl/chardet/src/nsGB2312Verifier.h deleted file mode 100644 index 221440196c7..00000000000 --- a/intl/chardet/src/nsGB2312Verifier.h +++ /dev/null @@ -1,104 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -/* - * DO NOT EDIT THIS DOCUMENT MANUALLY !!! - * THIS FILE IS AUTOMATICALLY GENERATED BY THE TOOLS UNDER - * mozilla/intl/chardet/tools/ - * Please contact ftang@netscape.com or mozilla-i18n@mozilla.org - * if you have any question. Thanks - */ -#include "nsVerifier.h" -static const PRUint32 GB2312_cls [ 256 / 8 ] = { -PCK4BITS(1,1,1,1,1,1,1,1), // 00 - 07 -PCK4BITS(1,1,1,1,1,1,0,0), // 08 - 0f -PCK4BITS(1,1,1,1,1,1,1,1), // 10 - 17 -PCK4BITS(1,1,1,0,1,1,1,1), // 18 - 1f -PCK4BITS(1,1,1,1,1,1,1,1), // 20 - 27 -PCK4BITS(1,1,1,1,1,1,1,1), // 28 - 2f -PCK4BITS(1,1,1,1,1,1,1,1), // 30 - 37 -PCK4BITS(1,1,1,1,1,1,1,1), // 38 - 3f -PCK4BITS(1,1,1,1,1,1,1,1), // 40 - 47 -PCK4BITS(1,1,1,1,1,1,1,1), // 48 - 4f -PCK4BITS(1,1,1,1,1,1,1,1), // 50 - 57 -PCK4BITS(1,1,1,1,1,1,1,1), // 58 - 5f -PCK4BITS(1,1,1,1,1,1,1,1), // 60 - 67 -PCK4BITS(1,1,1,1,1,1,1,1), // 68 - 6f -PCK4BITS(1,1,1,1,1,1,1,1), // 70 - 77 -PCK4BITS(1,1,1,1,1,1,1,1), // 78 - 7f -PCK4BITS(0,0,0,0,0,0,0,0), // 80 - 87 -PCK4BITS(0,0,0,0,0,0,0,0), // 88 - 8f -PCK4BITS(0,0,0,0,0,0,0,0), // 90 - 97 -PCK4BITS(0,0,0,0,0,0,0,0), // 98 - 9f -PCK4BITS(0,2,2,2,2,2,2,2), // a0 - a7 -PCK4BITS(2,2,3,3,3,3,3,3), // a8 - af -PCK4BITS(2,2,2,2,2,2,2,2), // b0 - b7 -PCK4BITS(2,2,2,2,2,2,2,2), // b8 - bf -PCK4BITS(2,2,2,2,2,2,2,2), // c0 - c7 -PCK4BITS(2,2,2,2,2,2,2,2), // c8 - cf -PCK4BITS(2,2,2,2,2,2,2,2), // d0 - d7 -PCK4BITS(2,2,2,2,2,2,2,2), // d8 - df -PCK4BITS(2,2,2,2,2,2,2,2), // e0 - e7 -PCK4BITS(2,2,2,2,2,2,2,2), // e8 - ef -PCK4BITS(2,2,2,2,2,2,2,2), // f0 - f7 -PCK4BITS(2,2,2,2,2,2,2,0) // f8 - ff -}; - - -static const PRUint32 GB2312_st [ 2] = { -PCK4BITS(eError,eStart, 3,eError,eError,eError,eError,eError),//00-07 -PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,eStart,eStart) //08-0f -}; - - -static nsVerifier nsGB2312Verifier = { - "GB2312", - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - GB2312_cls - }, - 4, - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - GB2312_st - } -}; diff --git a/intl/chardet/src/nsHZVerifier.h b/intl/chardet/src/nsHZVerifier.h deleted file mode 100644 index eb8e85f9892..00000000000 --- a/intl/chardet/src/nsHZVerifier.h +++ /dev/null @@ -1,108 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -/* - * DO NOT EDIT THIS DOCUMENT MANUALLY !!! - * THIS FILE IS AUTOMATICALLY GENERATED BY THE TOOLS UNDER - * mozilla/intl/chardet/tools/ - * Please contact ftang@netscape.com or mozilla-i18n@mozilla.org - * if you have any question. Thanks - */ -#include "nsVerifier.h" -static const PRUint32 HZ_cls [ 256 / 8 ] = { -PCK4BITS(1,0,0,0,0,0,0,0), // 00 - 07 -PCK4BITS(0,0,0,0,0,0,0,0), // 08 - 0f -PCK4BITS(0,0,0,0,0,0,0,0), // 10 - 17 -PCK4BITS(0,0,0,1,0,0,0,0), // 18 - 1f -PCK4BITS(0,0,0,0,0,0,0,0), // 20 - 27 -PCK4BITS(0,0,0,0,0,0,0,0), // 28 - 2f -PCK4BITS(0,0,0,0,0,0,0,0), // 30 - 37 -PCK4BITS(0,0,0,0,0,0,0,0), // 38 - 3f -PCK4BITS(0,0,0,0,0,0,0,0), // 40 - 47 -PCK4BITS(0,0,0,0,0,0,0,0), // 48 - 4f -PCK4BITS(0,0,0,0,0,0,0,0), // 50 - 57 -PCK4BITS(0,0,0,0,0,0,0,0), // 58 - 5f -PCK4BITS(0,0,0,0,0,0,0,0), // 60 - 67 -PCK4BITS(0,0,0,0,0,0,0,0), // 68 - 6f -PCK4BITS(0,0,0,0,0,0,0,0), // 70 - 77 -PCK4BITS(0,0,0,4,0,5,2,0), // 78 - 7f -PCK4BITS(1,1,1,1,1,1,1,1), // 80 - 87 -PCK4BITS(1,1,1,1,1,1,1,1), // 88 - 8f -PCK4BITS(1,1,1,1,1,1,1,1), // 90 - 97 -PCK4BITS(1,1,1,1,1,1,1,1), // 98 - 9f -PCK4BITS(1,1,1,1,1,1,1,1), // a0 - a7 -PCK4BITS(1,1,1,1,1,1,1,1), // a8 - af -PCK4BITS(1,1,1,1,1,1,1,1), // b0 - b7 -PCK4BITS(1,1,1,1,1,1,1,1), // b8 - bf -PCK4BITS(1,1,1,1,1,1,1,1), // c0 - c7 -PCK4BITS(1,1,1,1,1,1,1,1), // c8 - cf -PCK4BITS(1,1,1,1,1,1,1,1), // d0 - d7 -PCK4BITS(1,1,1,1,1,1,1,1), // d8 - df -PCK4BITS(1,1,1,1,1,1,1,1), // e0 - e7 -PCK4BITS(1,1,1,1,1,1,1,1), // e8 - ef -PCK4BITS(1,1,1,1,1,1,1,1), // f0 - f7 -PCK4BITS(1,1,1,1,1,1,1,1) // f8 - ff -}; - - -static const PRUint32 HZ_st [ 6] = { -PCK4BITS(eStart,eError, 3,eStart,eStart,eStart,eError,eError),//00-07 -PCK4BITS(eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe),//08-0f -PCK4BITS(eItsMe,eItsMe,eError,eError,eStart,eStart, 4,eError),//10-17 -PCK4BITS( 5,eError, 6,eError, 5, 5, 4,eError),//18-1f -PCK4BITS( 4,eError, 4, 4, 4,eError, 4,eError),//20-27 -PCK4BITS( 4,eItsMe,eStart,eStart,eStart,eStart,eStart,eStart) //28-2f -}; - - -static nsVerifier nsHZVerifier = { - "HZ-GB-2312", - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - HZ_cls - }, - 6, - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - HZ_st - } -}; diff --git a/intl/chardet/src/nsISO2022CNVerifier.h b/intl/chardet/src/nsISO2022CNVerifier.h deleted file mode 100644 index 437cfecb1e7..00000000000 --- a/intl/chardet/src/nsISO2022CNVerifier.h +++ /dev/null @@ -1,110 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -/* - * DO NOT EDIT THIS DOCUMENT MANUALLY !!! - * THIS FILE IS AUTOMATICALLY GENERATED BY THE TOOLS UNDER - * mozilla/intl/chardet/tools/ - * Please contact ftang@netscape.com or mozilla-i18n@mozilla.org - * if you have any question. Thanks - */ -#include "nsVerifier.h" -static const PRUint32 ISO2022CN_cls [ 256 / 8 ] = { -PCK4BITS(2,0,0,0,0,0,0,0), // 00 - 07 -PCK4BITS(0,0,0,0,0,0,0,0), // 08 - 0f -PCK4BITS(0,0,0,0,0,0,0,0), // 10 - 17 -PCK4BITS(0,0,0,1,0,0,0,0), // 18 - 1f -PCK4BITS(0,0,0,0,0,0,0,0), // 20 - 27 -PCK4BITS(0,3,0,0,0,0,0,0), // 28 - 2f -PCK4BITS(0,0,0,0,0,0,0,0), // 30 - 37 -PCK4BITS(0,0,0,0,0,0,0,0), // 38 - 3f -PCK4BITS(0,0,0,4,0,0,0,0), // 40 - 47 -PCK4BITS(0,0,0,0,0,0,0,0), // 48 - 4f -PCK4BITS(0,0,0,0,0,0,0,0), // 50 - 57 -PCK4BITS(0,0,0,0,0,0,0,0), // 58 - 5f -PCK4BITS(0,0,0,0,0,0,0,0), // 60 - 67 -PCK4BITS(0,0,0,0,0,0,0,0), // 68 - 6f -PCK4BITS(0,0,0,0,0,0,0,0), // 70 - 77 -PCK4BITS(0,0,0,0,0,0,0,0), // 78 - 7f -PCK4BITS(2,2,2,2,2,2,2,2), // 80 - 87 -PCK4BITS(2,2,2,2,2,2,2,2), // 88 - 8f -PCK4BITS(2,2,2,2,2,2,2,2), // 90 - 97 -PCK4BITS(2,2,2,2,2,2,2,2), // 98 - 9f -PCK4BITS(2,2,2,2,2,2,2,2), // a0 - a7 -PCK4BITS(2,2,2,2,2,2,2,2), // a8 - af -PCK4BITS(2,2,2,2,2,2,2,2), // b0 - b7 -PCK4BITS(2,2,2,2,2,2,2,2), // b8 - bf -PCK4BITS(2,2,2,2,2,2,2,2), // c0 - c7 -PCK4BITS(2,2,2,2,2,2,2,2), // c8 - cf -PCK4BITS(2,2,2,2,2,2,2,2), // d0 - d7 -PCK4BITS(2,2,2,2,2,2,2,2), // d8 - df -PCK4BITS(2,2,2,2,2,2,2,2), // e0 - e7 -PCK4BITS(2,2,2,2,2,2,2,2), // e8 - ef -PCK4BITS(2,2,2,2,2,2,2,2), // f0 - f7 -PCK4BITS(2,2,2,2,2,2,2,2) // f8 - ff -}; - - -static const PRUint32 ISO2022CN_st [ 8] = { -PCK4BITS(eStart, 3,eError,eStart,eStart,eStart,eStart,eStart),//00-07 -PCK4BITS(eStart,eError,eError,eError,eError,eError,eError,eError),//08-0f -PCK4BITS(eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe),//10-17 -PCK4BITS(eItsMe,eItsMe,eItsMe,eError,eError,eError, 4,eError),//18-1f -PCK4BITS(eError,eError,eError,eItsMe,eError,eError,eError,eError),//20-27 -PCK4BITS( 5, 6,eError,eError,eError,eError,eError,eError),//28-2f -PCK4BITS(eError,eError,eError,eItsMe,eError,eError,eError,eError),//30-37 -PCK4BITS(eError,eError,eError,eError,eError,eItsMe,eError,eStart) //38-3f -}; - - -static nsVerifier nsISO2022CNVerifier = { - "ISO-2022-CN", - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - ISO2022CN_cls - }, - 9, - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - ISO2022CN_st - } -}; diff --git a/intl/chardet/src/nsISO2022JPVerifier.h b/intl/chardet/src/nsISO2022JPVerifier.h deleted file mode 100644 index 59970111993..00000000000 --- a/intl/chardet/src/nsISO2022JPVerifier.h +++ /dev/null @@ -1,111 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -/* - * DO NOT EDIT THIS DOCUMENT MANUALLY !!! - * THIS FILE IS AUTOMATICALLY GENERATED BY THE TOOLS UNDER - * mozilla/intl/chardet/tools/ - * Please contact ftang@netscape.com or mozilla-i18n@mozilla.org - * if you have any question. Thanks - */ -#include "nsVerifier.h" -static const PRUint32 ISO2022JP_cls [ 256 / 8 ] = { -PCK4BITS(2,0,0,0,0,0,0,0), // 00 - 07 -PCK4BITS(0,0,0,0,0,0,2,2), // 08 - 0f -PCK4BITS(0,0,0,0,0,0,0,0), // 10 - 17 -PCK4BITS(0,0,0,1,0,0,0,0), // 18 - 1f -PCK4BITS(0,0,0,0,7,0,0,0), // 20 - 27 -PCK4BITS(3,0,0,0,0,0,0,0), // 28 - 2f -PCK4BITS(0,0,0,0,0,0,0,0), // 30 - 37 -PCK4BITS(0,0,0,0,0,0,0,0), // 38 - 3f -PCK4BITS(6,0,4,0,8,0,0,0), // 40 - 47 -PCK4BITS(0,9,5,0,0,0,0,0), // 48 - 4f -PCK4BITS(0,0,0,0,0,0,0,0), // 50 - 57 -PCK4BITS(0,0,0,0,0,0,0,0), // 58 - 5f -PCK4BITS(0,0,0,0,0,0,0,0), // 60 - 67 -PCK4BITS(0,0,0,0,0,0,0,0), // 68 - 6f -PCK4BITS(0,0,0,0,0,0,0,0), // 70 - 77 -PCK4BITS(0,0,0,0,0,0,0,0), // 78 - 7f -PCK4BITS(2,2,2,2,2,2,2,2), // 80 - 87 -PCK4BITS(2,2,2,2,2,2,2,2), // 88 - 8f -PCK4BITS(2,2,2,2,2,2,2,2), // 90 - 97 -PCK4BITS(2,2,2,2,2,2,2,2), // 98 - 9f -PCK4BITS(2,2,2,2,2,2,2,2), // a0 - a7 -PCK4BITS(2,2,2,2,2,2,2,2), // a8 - af -PCK4BITS(2,2,2,2,2,2,2,2), // b0 - b7 -PCK4BITS(2,2,2,2,2,2,2,2), // b8 - bf -PCK4BITS(2,2,2,2,2,2,2,2), // c0 - c7 -PCK4BITS(2,2,2,2,2,2,2,2), // c8 - cf -PCK4BITS(2,2,2,2,2,2,2,2), // d0 - d7 -PCK4BITS(2,2,2,2,2,2,2,2), // d8 - df -PCK4BITS(2,2,2,2,2,2,2,2), // e0 - e7 -PCK4BITS(2,2,2,2,2,2,2,2), // e8 - ef -PCK4BITS(2,2,2,2,2,2,2,2), // f0 - f7 -PCK4BITS(2,2,2,2,2,2,2,2) // f8 - ff -}; - - -static const PRUint32 ISO2022JP_st [ 9] = { -PCK4BITS(eStart, 3,eError,eStart,eStart,eStart,eStart,eStart),//00-07 -PCK4BITS(eStart,eStart,eError,eError,eError,eError,eError,eError),//08-0f -PCK4BITS(eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe),//10-17 -PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError,eError),//18-1f -PCK4BITS(eError, 5,eError,eError,eError, 4,eError,eError),//20-27 -PCK4BITS(eError,eError,eError, 6,eItsMe,eError,eItsMe,eError),//28-2f -PCK4BITS(eError,eError,eError,eError,eError,eError,eItsMe,eItsMe),//30-37 -PCK4BITS(eError,eError,eError,eItsMe,eError,eError,eError,eError),//38-3f -PCK4BITS(eError,eError,eError,eError,eItsMe,eError,eStart,eStart) //40-47 -}; - - -static nsVerifier nsISO2022JPVerifier = { - "ISO-2022-JP", - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - ISO2022JP_cls - }, - 10, - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - ISO2022JP_st - } -}; diff --git a/intl/chardet/src/nsISO2022KRVerifier.h b/intl/chardet/src/nsISO2022KRVerifier.h deleted file mode 100644 index 4433b47dfe5..00000000000 --- a/intl/chardet/src/nsISO2022KRVerifier.h +++ /dev/null @@ -1,107 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -/* - * DO NOT EDIT THIS DOCUMENT MANUALLY !!! - * THIS FILE IS AUTOMATICALLY GENERATED BY THE TOOLS UNDER - * mozilla/intl/chardet/tools/ - * Please contact ftang@netscape.com or mozilla-i18n@mozilla.org - * if you have any question. Thanks - */ -#include "nsVerifier.h" -static const PRUint32 ISO2022KR_cls [ 256 / 8 ] = { -PCK4BITS(2,0,0,0,0,0,0,0), // 00 - 07 -PCK4BITS(0,0,0,0,0,0,0,0), // 08 - 0f -PCK4BITS(0,0,0,0,0,0,0,0), // 10 - 17 -PCK4BITS(0,0,0,1,0,0,0,0), // 18 - 1f -PCK4BITS(0,0,0,0,3,0,0,0), // 20 - 27 -PCK4BITS(0,4,0,0,0,0,0,0), // 28 - 2f -PCK4BITS(0,0,0,0,0,0,0,0), // 30 - 37 -PCK4BITS(0,0,0,0,0,0,0,0), // 38 - 3f -PCK4BITS(0,0,0,5,0,0,0,0), // 40 - 47 -PCK4BITS(0,0,0,0,0,0,0,0), // 48 - 4f -PCK4BITS(0,0,0,0,0,0,0,0), // 50 - 57 -PCK4BITS(0,0,0,0,0,0,0,0), // 58 - 5f -PCK4BITS(0,0,0,0,0,0,0,0), // 60 - 67 -PCK4BITS(0,0,0,0,0,0,0,0), // 68 - 6f -PCK4BITS(0,0,0,0,0,0,0,0), // 70 - 77 -PCK4BITS(0,0,0,0,0,0,0,0), // 78 - 7f -PCK4BITS(2,2,2,2,2,2,2,2), // 80 - 87 -PCK4BITS(2,2,2,2,2,2,2,2), // 88 - 8f -PCK4BITS(2,2,2,2,2,2,2,2), // 90 - 97 -PCK4BITS(2,2,2,2,2,2,2,2), // 98 - 9f -PCK4BITS(2,2,2,2,2,2,2,2), // a0 - a7 -PCK4BITS(2,2,2,2,2,2,2,2), // a8 - af -PCK4BITS(2,2,2,2,2,2,2,2), // b0 - b7 -PCK4BITS(2,2,2,2,2,2,2,2), // b8 - bf -PCK4BITS(2,2,2,2,2,2,2,2), // c0 - c7 -PCK4BITS(2,2,2,2,2,2,2,2), // c8 - cf -PCK4BITS(2,2,2,2,2,2,2,2), // d0 - d7 -PCK4BITS(2,2,2,2,2,2,2,2), // d8 - df -PCK4BITS(2,2,2,2,2,2,2,2), // e0 - e7 -PCK4BITS(2,2,2,2,2,2,2,2), // e8 - ef -PCK4BITS(2,2,2,2,2,2,2,2), // f0 - f7 -PCK4BITS(2,2,2,2,2,2,2,2) // f8 - ff -}; - - -static const PRUint32 ISO2022KR_st [ 5] = { -PCK4BITS(eStart, 3,eError,eStart,eStart,eStart,eError,eError),//00-07 -PCK4BITS(eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe),//08-0f -PCK4BITS(eItsMe,eItsMe,eError,eError,eError, 4,eError,eError),//10-17 -PCK4BITS(eError,eError,eError,eError, 5,eError,eError,eError),//18-1f -PCK4BITS(eError,eError,eError,eItsMe,eStart,eStart,eStart,eStart) //20-27 -}; - - -static nsVerifier nsISO2022KRVerifier = { - "ISO-2022-KR", - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - ISO2022KR_cls - }, - 6, - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - ISO2022KR_st - } -}; diff --git a/intl/chardet/src/nsPSMDetectors.cpp b/intl/chardet/src/nsPSMDetectors.cpp deleted file mode 100644 index b784f1c340c..00000000000 --- a/intl/chardet/src/nsPSMDetectors.cpp +++ /dev/null @@ -1,564 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Pierre Phaneuf - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - - -#include -#include -//---- for XPCOM -#include "nsIFactory.h" -#include "nsIGenericFactory.h" -#include "nsISupports.h" -#include "nsCharDetDll.h" -#include "pratom.h" -#include "nsPSMDetectors.h" - -nsEUCStatistics gBig5Statistics = -#include "Big5Statistics.h" -// end of UECTWStatistics.h include - -nsEUCStatistics gEUCTWStatistics = -#include "EUCTWStatistics.h" -// end of UECTWStatistics.h include - -nsEUCStatistics gGB2312Statistics = -#include "GB2312Statistics.h" -// end of GB2312Statistics.h include - -nsEUCStatistics gEUCJPStatistics = -#include "EUCJPStatistics.h" -// end of EUCJPStatistics.h include - -nsEUCStatistics gEUCKRStatistics = -#include "EUCKRStatistics.h" -// end of EUCKRStatistics.h include - -//========================================================== -/* - This class won't detect x-euc-tw for now. It can only - tell a Big5 document is not x-euc-tw , but cannot tell - a x-euc-tw docuement is not Big5 unless we hit characters - defined in CNS 11643 plane 2. - - May need improvement .... - */ - -nsVerifier* const gZhTwVerifierSet[ZHTW_DETECTOR_NUM_VERIFIERS] = { - &nsUTF8Verifier, - &nsBIG5Verifier, - &nsISO2022CNVerifier, - &nsEUCTWVerifier, - &nsCP1252Verifier, - &nsUCS2BEVerifier, - &nsUCS2LEVerifier -}; - -nsEUCStatistics* const gZhTwStatisticsSet[ZHTW_DETECTOR_NUM_VERIFIERS] = { - nsnull, - &gBig5Statistics, - nsnull, - &gEUCTWStatistics, - nsnull, - nsnull, - nsnull -}; - -//========================================================== - -nsVerifier* const gKoVerifierSet[KO_DETECTOR_NUM_VERIFIERS] = { - &nsUTF8Verifier, - &nsEUCKRVerifier, - &nsISO2022KRVerifier, - &nsCP1252Verifier, - &nsUCS2BEVerifier, - &nsUCS2LEVerifier -}; - -//========================================================== - -nsVerifier* const gZhCnVerifierSet[ZHCN_DETECTOR_NUM_VERIFIERS] = { - &nsUTF8Verifier, - &nsGB2312Verifier, - &nsGB18030Verifier, - &nsISO2022CNVerifier, - &nsHZVerifier, - &nsCP1252Verifier, - &nsUCS2BEVerifier, - &nsUCS2LEVerifier -}; - -//========================================================== - -nsVerifier* const gJaVerifierSet[JA_DETECTOR_NUM_VERIFIERS] = { - &nsUTF8Verifier, - &nsSJISVerifier, - &nsEUCJPVerifier, - &nsISO2022JPVerifier, - &nsCP1252Verifier, - &nsUCS2BEVerifier, - &nsUCS2LEVerifier -}; - -//========================================================== - -nsVerifier* const gZhVerifierSet[ZH_DETECTOR_NUM_VERIFIERS] = { - &nsUTF8Verifier, - &nsGB2312Verifier, - &nsGB18030Verifier, - &nsBIG5Verifier, - &nsISO2022CNVerifier, - &nsHZVerifier, - &nsEUCTWVerifier, - &nsCP1252Verifier, - &nsUCS2BEVerifier, - &nsUCS2LEVerifier -}; - -nsEUCStatistics* const gZhStatisticsSet[ZH_DETECTOR_NUM_VERIFIERS] = { - nsnull, - &gGB2312Statistics, - &gBig5Statistics, - nsnull, - nsnull, - &gEUCTWStatistics, - nsnull, - nsnull, - nsnull -}; - -//========================================================== - -nsVerifier* const gCJKVerifierSet[CJK_DETECTOR_NUM_VERIFIERS] = { - &nsUTF8Verifier, - &nsSJISVerifier, - &nsEUCJPVerifier, - &nsISO2022JPVerifier, - &nsEUCKRVerifier, - &nsISO2022KRVerifier, - &nsBIG5Verifier, - &nsEUCTWVerifier, - &nsGB2312Verifier, - &nsGB18030Verifier, - &nsISO2022CNVerifier, - &nsHZVerifier, - &nsCP1252Verifier, - &nsUCS2BEVerifier, - &nsUCS2LEVerifier -}; - -nsEUCStatistics* const gCJKStatisticsSet[CJK_DETECTOR_NUM_VERIFIERS] = { - nsnull, - nsnull, - &gEUCJPStatistics, - nsnull, - &gEUCKRStatistics, - nsnull, - &gBig5Statistics, - &gEUCTWStatistics, - &gGB2312Statistics, - nsnull, - nsnull, - nsnull, - nsnull, - nsnull -}; - -PRBool nsEUCSampler::Sample(const char* aIn, PRUint32 aLen) -{ - if(mState == 1) - return PR_FALSE; - const unsigned char* p = (const unsigned char*) aIn; - if(aLen + mTotal > 0x80000000) - aLen = 0x80000000 - mTotal; - - PRUint32 i; - for(i=0; (i *p)) { - mState = 1; - } else { - mTotal++; - mFirstByteCnt[*p - 0x00a1]++; - mState = 2; - } - } - break; - case 1: - break; - case 2: - if( *p & 0x0080) - { - if((0x00ff == *p) || ( 0x00a1 > *p)) { - mState = 1; - } else { - mTotal++; - mSecondByteCnt[*p - 0x00a1]++; - mState = 0; - } - } else { - mState = 1; - } - break; - default: - mState = 1; - } - } - return (1 != mState ); -} -float nsEUCSampler::GetScore(const float* aFirstByteFreq, float aFirstByteWeight, - const float* aSecondByteFreq, float aSecondByteWeight) -{ - return aFirstByteWeight * GetScore(aFirstByteFreq, mFirstByteFreq) + - aSecondByteWeight * GetScore(aSecondByteFreq, mSecondByteFreq); -} - -float nsEUCSampler::GetScore(const float* array1, const float* array2) -{ - float s; - float sum=0.0; - PRUint16 i; - for(i=0;i<94;i++) { - s = array1[i] - array2[i]; - sum += s * s; - } - return (float)sqrt((double)sum) / 94.0f; -} - -void nsEUCSampler::CalFreq() -{ - PRUint32 i; - for(i = 0 ; i < 94; i++) { - mFirstByteFreq[i] = (float)mFirstByteCnt[i] / (float)mTotal; - mSecondByteFreq[i] = (float)mSecondByteCnt[i] / (float)mTotal; - } -} - -//---------------------------------------------------------- -NS_IMPL_ISUPPORTS1(nsXPCOMDetector, nsICharsetDetector) -NS_IMPL_ISUPPORTS1(nsXPCOMStringDetector, nsIStringCharsetDetector) -nsPSMDetector::nsPSMDetector(PRUint8 aItems, nsVerifier* const * aVerifierSet, nsEUCStatistics* const * aStatisticsSet) -{ - mClassRunSampler = (nsnull != aStatisticsSet); - mStatisticsData = aStatisticsSet; - mVerifier = aVerifierSet; - - mClassItems = aItems; - Reset(); -} -void nsPSMDetector::Reset() -{ - mRunSampler = mClassRunSampler; - mDone= PR_FALSE; - mItems = mClassItems; - NS_ASSERTION(MAX_VERIFIERS >= mItems , "MAX_VERIFIERS is too small!"); - for(PRUint8 i = 0; i < mItems ; i++) - { - mState[i] = 0; - mItemIdx[i] = i; - } -#ifdef DETECTOR_DEBUG - mDbgLen = mDbgTest = 0; -#endif -} -//---------------------------------------------------------- -void nsPSMDetector::DataEnd() -{ - // since gb18030 covers almost all code points in big5, sjis, euc-xx, - // it effectively make other verifiers unusable. Gb18030 is not - // very popular, and it could reach Itsme state. We need to eliminate - // gb18030 when there are only 2 candidates left. - if (mItems == 2) { - if ((&nsGB18030Verifier) == mVerifier[mItemIdx[0]]) { - Report( mVerifier[mItemIdx[1]]->charset); - mDone = PR_TRUE; - } else if ((&nsGB18030Verifier) == mVerifier[mItemIdx[1]]) { - Report( mVerifier[mItemIdx[0]]->charset); - mDone = PR_TRUE; - } - } - if(mRunSampler) - Sample(nsnull, 0, PR_TRUE); -} -//---------------------------------------------------------- - -// #define ftang_TRACE_STATE -// #define TRACE_VERIFIER nsCP1252Verifier - -PRBool nsPSMDetector::HandleData(const char* aBuf, PRUint32 aLen) -{ - PRUint32 i,j; - PRUint32 st; - for(i=0; i < aLen; i++) - { - char b = aBuf[i]; - for(j = 0; j < mItems; ) - { -#ifdef ftang_TRACE_STATE - if( mVerifier[mItemIdx[j]] == & TRACE_VERIFIER ) - { - printf("%d = %d\n", i + mDbgLen, mState[j]); - } -#endif -#ifdef DETECTOR_DEBUG - mDbgTest++; -#endif - st = GETNEXTSTATE( mVerifier[mItemIdx[j]], b, mState[j] ); - if(eItsMe == st) - { -#ifdef DETECTOR_DEBUG - printf("It's %s- byte %d(%x) test %d\n", - mVerifier[mItemIdx[j]]->charset, - i+mDbgLen, - i+mDbgLen, - mDbgTest - ); -#endif - Report( mVerifier[mItemIdx[j]]->charset); - mDone = PR_TRUE; - return mDone; - } else { - mState[j++] = st; - } - } - if( mItems <= 1) - { - if( 1 == mItems) { -#ifdef DETECTOR_DEBUG - printf("It's %s- byte %d (%x) Test %d. The only left\n", - mVerifier[mItemIdx[0]]->charset, - i+mDbgLen, - i+mDbgLen, - mDbgTest); -#endif - Report( mVerifier[mItemIdx[0]]->charset); - } - mDone = PR_TRUE; - return mDone; - } else { - // If the only charset left is UCS2LE/UCS2BE and another, report the other - PRInt32 nonUCS2Num=0; - PRInt32 nonUCS2Idx=0; - for(j = 0; j < mItems; j++) { - if(((&nsUCS2BEVerifier) != mVerifier[mItemIdx[j]]) && - ((&nsUCS2LEVerifier) != mVerifier[mItemIdx[j]])) { - nonUCS2Num++; - nonUCS2Idx = j; - } - } - if(1 == nonUCS2Num) { -#ifdef DETECTOR_DEBUG - printf("It's %s- byte %d (%x) Test %d. The only left except UCS2LE/BE\n", - mVerifier[mItemIdx[nonUCS2Idx]]->charset, - i+mDbgLen, - i+mDbgLen, - mDbgTest); -#endif - Report( mVerifier[mItemIdx[nonUCS2Idx]]->charset); - mDone = PR_TRUE; - return mDone; - } - } - } - if(mRunSampler) - Sample(aBuf, aLen); - -#ifdef DETECTOR_DEBUG - mDbgLen += aLen; -#endif - return PR_FALSE; -} - -void nsPSMDetector::Sample(const char* aBuf, PRUint32 aLen, PRBool aLastChance) -{ - PRInt32 possibleCandidateNum=0; - PRInt32 j; - PRInt32 eucNum=0; - for(j = 0; j < mItems; j++) { - if(nsnull != mStatisticsData[mItemIdx[j]]) - eucNum++; - if(((&nsUCS2BEVerifier) != mVerifier[mItemIdx[j]]) && - ((&nsUCS2LEVerifier) != mVerifier[mItemIdx[j]]) && - ((&nsGB18030Verifier) != mVerifier[mItemIdx[j]]) ) { - possibleCandidateNum++; - } - } - mRunSampler = (eucNum > 1); - if(mRunSampler) { - mRunSampler = mSampler.Sample(aBuf, aLen); - if(((aLastChance && mSampler.GetSomeData()) || - mSampler.EnoughData()) - && (eucNum == possibleCandidateNum)) { - mSampler.CalFreq(); -#ifdef DETECTOR_DEBUG - printf("We cannot figure out charset from the encoding, " - "All EUC based charset share the same encoding structure.\n" - "Detect based on statistics"); - if(aLastChance) { - printf(" after we receive all the data.\n"); - } else { - printf(" after we receive enough data.\n"); - } -#endif - PRInt32 bestIdx = -1; - PRInt32 eucCnt=0; - float bestScore = 0.0f; - for(j = 0; j < mItems; j++) { - if((nsnull != mStatisticsData[mItemIdx[j]]) && - (&gBig5Statistics != mStatisticsData[mItemIdx[j]])) - { - float score = mSampler.GetScore( - mStatisticsData[mItemIdx[j]]->mFirstByteFreq, - mStatisticsData[mItemIdx[j]]->mFirstByteWeight, - mStatisticsData[mItemIdx[j]]->mSecoundByteFreq, - mStatisticsData[mItemIdx[j]]->mSecoundByteWeight ); -#ifdef DETECTOR_DEBUG - printf("Differences between %s and this data is %2.8f\n", - mVerifier[mItemIdx[j]]->charset, - score); -#endif - if(( 0 == eucCnt++) || (bestScore > score )) { - bestScore = score; - bestIdx = j; - } // if(( 0 == eucCnt++) || (bestScore > score )) - } // if(nsnull != ...) - } // for - if (bestIdx >= 0) - { -#ifdef DETECTOR_DEBUG - printf("Based on the statistic, we decide it is %s", - mVerifier[mItemIdx[bestIdx]]->charset); -#endif - Report( mVerifier[mItemIdx[bestIdx]]->charset); - mDone = PR_TRUE; - } - } // if (eucNum == possibleCandidateNum) - } // if(mRunSampler) -} -//---------------------------------------------------------- -nsXPCOMDetector::nsXPCOMDetector(PRUint8 aItems, nsVerifier * const *aVer, nsEUCStatistics* const * aStatisticsSet) - : nsPSMDetector( aItems, aVer, aStatisticsSet) -{ - mObserver = nsnull; -} -//---------------------------------------------------------- -nsXPCOMDetector::~nsXPCOMDetector() -{ -} -//---------------------------------------------------------- -NS_IMETHODIMP nsXPCOMDetector::Init( - nsICharsetDetectionObserver* aObserver) -{ - NS_ASSERTION(mObserver == nsnull , "Init twice"); - if(nsnull == aObserver) - return NS_ERROR_ILLEGAL_VALUE; - - mObserver = aObserver; - return NS_OK; -} -//---------------------------------------------------------- -NS_IMETHODIMP nsXPCOMDetector::DoIt( - const char* aBuf, PRUint32 aLen, PRBool* oDontFeedMe) -{ - NS_ASSERTION(mObserver != nsnull , "have not init yet"); - - if((nsnull == aBuf) || (nsnull == oDontFeedMe)) - return NS_ERROR_ILLEGAL_VALUE; - - this->HandleData(aBuf, aLen); - *oDontFeedMe = mDone; - return NS_OK; -} -//---------------------------------------------------------- -NS_IMETHODIMP nsXPCOMDetector::Done() -{ - NS_ASSERTION(mObserver != nsnull , "have not init yet"); - this->DataEnd(); - return NS_OK; -} -//---------------------------------------------------------- -void nsXPCOMDetector::Report(const char* charset) -{ - mObserver->Notify(charset, eSureAnswer); -} -//---------------------------------------------------------- -nsXPCOMStringDetector::nsXPCOMStringDetector(PRUint8 aItems, nsVerifier* const * aVer, nsEUCStatistics* const * aStatisticsSet) - : nsPSMDetector( aItems, aVer, aStatisticsSet) -{ -} -//---------------------------------------------------------- -nsXPCOMStringDetector::~nsXPCOMStringDetector() -{ -} -//---------------------------------------------------------- -void nsXPCOMStringDetector::Report(const char* charset) -{ - mResult = charset; -} -//---------------------------------------------------------- -NS_IMETHODIMP nsXPCOMStringDetector::DoIt(const char* aBuf, PRUint32 aLen, - const char** oCharset, - nsDetectionConfident &oConfident) -{ - mResult = nsnull; - this->HandleData(aBuf, aLen); - - if( nsnull == mResult) { - // If we have no result and detector is done - answer no match - if(mDone) - { - *oCharset = nsnull; - oConfident = eNoAnswerMatch; - } else { - // if we have no answer force the Done method and find the answer - // if we find one, return it as eBestAnswer - this->DataEnd(); - *oCharset = mResult; - oConfident = (mResult) ? eBestAnswer : eNoAnswerMatch ; - } - } else { - // If we have answer, return as eSureAnswer - *oCharset = mResult; - oConfident = eSureAnswer; - } - this->Reset(); - return NS_OK; -} - diff --git a/intl/chardet/src/nsPSMDetectors.h b/intl/chardet/src/nsPSMDetectors.h deleted file mode 100644 index 3d04226f5ca..00000000000 --- a/intl/chardet/src/nsPSMDetectors.h +++ /dev/null @@ -1,375 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -#ifndef nsPSMDetectors_h__ -#define nsPSMDetectors_h__ - -#include "nsCOMPtr.h" -#include "nsIFactory.h" -#include "nsVerifier.h" -//---- for verifiers -#include "nsSJISVerifier.h" -#include "nsEUCJPVerifier.h" -#include "nsCP1252Verifier.h" -#include "nsUTF8Verifier.h" -#include "nsISO2022JPVerifier.h" -#include "nsISO2022KRVerifier.h" -#include "nsISO2022CNVerifier.h" -#include "nsHZVerifier.h" -#include "nsUCS2BEVerifier.h" -#include "nsUCS2LEVerifier.h" -#include "nsBIG5Verifier.h" -#include "nsGB2312Verifier.h" -#include "nsGB18030Verifier.h" -#include "nsEUCTWVerifier.h" -#include "nsEUCKRVerifier.h" -//---- end verifiers - -//#define DETECTOR_DEBUG - -#define MAX_VERIFIERS 16 - -// {12BB8F1B-2389-11d3-B3BF-00805F8A6670} -#define NS_JA_PSMDETECTOR_CID \ -{ 0x12bb8f1b, 0x2389, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } - -// {12BB8F1C-2389-11d3-B3BF-00805F8A6670} -#define NS_JA_STRING_PSMDETECTOR_CID \ -{ 0x12bb8f1c, 0x2389, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } - -// {EA06D4E1-2B3D-11d3-B3BF-00805F8A6670} -#define NS_KO_PSMDETECTOR_CID \ -{ 0xea06d4e1, 0x2b3d, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } - -// {EA06D4E2-2B3D-11d3-B3BF-00805F8A6670} -#define NS_ZHCN_PSMDETECTOR_CID \ -{ 0xea06d4e2, 0x2b3d, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } - -// {EA06D4E3-2B3D-11d3-B3BF-00805F8A6670} -#define NS_ZHTW_PSMDETECTOR_CID \ -{ 0xea06d4e3, 0x2b3d, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } - - -// {EA06D4E4-2B3D-11d3-B3BF-00805F8A6670} -#define NS_KO_STRING_PSMDETECTOR_CID \ -{ 0xea06d4e4, 0x2b3d, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } - -// {EA06D4E5-2B3D-11d3-B3BF-00805F8A6670} -#define NS_ZHCN_STRING_PSMDETECTOR_CID \ -{ 0xea06d4e5, 0x2b3d, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } - -// {EA06D4E6-2B3D-11d3-B3BF-00805F8A6670} -#define NS_ZHTW_STRING_PSMDETECTOR_CID \ -{ 0xea06d4e6, 0x2b3d, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } - - -// {FCACEF21-2B40-11d3-B3BF-00805F8A6670} -#define NS_ZH_STRING_PSMDETECTOR_CID \ -{ 0xfcacef21, 0x2b40, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } - -// {FCACEF22-2B40-11d3-B3BF-00805F8A6670} -#define NS_CJK_STRING_PSMDETECTOR_CID \ -{ 0xfcacef22, 0x2b40, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } - - -// {FCACEF23-2B40-11d3-B3BF-00805F8A6670} -#define NS_ZH_PSMDETECTOR_CID \ -{ 0xfcacef23, 0x2b40, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } - -// {FCACEF24-2B40-11d3-B3BF-00805F8A6670} -#define NS_CJK_PSMDETECTOR_CID \ -{ 0xfcacef24, 0x2b40, 0x11d3, { 0xb3, 0xbf, 0x0, 0x80, 0x5f, 0x8a, 0x66, 0x70 } } - -typedef struct { - float mFirstByteFreq[94]; - float mFirstByteStdDev; - float mFirstByteMean; - float mFirstByteWeight; - float mSecoundByteFreq[94]; - float mSecoundByteStdDev; - float mSecoundByteMean; - float mSecoundByteWeight; -} nsEUCStatisticsMutable; - -typedef const nsEUCStatisticsMutable nsEUCStatistics; - -/* -extern nsEUCStatistics gBig5Statistics = -#include "Big5Statistics.h" -// end of UECTWStatistics.h include - -static nsEUCStatistics gEUCTWStatistics = -#include "EUCTWStatistics.h" -// end of UECTWStatistics.h include - -static nsEUCStatistics gGB2312Statistics = -#include "GB2312Statistics.h" -// end of GB2312Statistics.h include - -static nsEUCStatistics gEUCJPStatistics = -#include "EUCJPStatistics.h" -// end of EUCJPStatistics.h include - -static nsEUCStatistics gEUCKRStatistics = -#include "EUCKRStatistics.h" -// end of EUCKRStatistics.h include -*/ - -//========================================================== -/* - This class won't detect x-euc-tw for now. It can only - tell a Big5 document is not x-euc-tw , but cannot tell - a x-euc-tw docuement is not Big5 unless we hit characters - defined in CNS 11643 plane 2. - - May need improvement .... - */ -#define ZHTW_DETECTOR_NUM_VERIFIERS 7 -extern nsVerifier* const gZhTwVerifierSet[]; -extern nsEUCStatistics* const gZhTwStatisticsSet[]; - -#define KO_DETECTOR_NUM_VERIFIERS 6 -extern nsVerifier* const gKoVerifierSet[]; - -#define ZHCN_DETECTOR_NUM_VERIFIERS 8 -extern nsVerifier* const gZhCnVerifierSet[]; - -#define JA_DETECTOR_NUM_VERIFIERS 7 -extern nsVerifier* const gJaVerifierSet[]; - -#define ZH_DETECTOR_NUM_VERIFIERS 10 -extern nsVerifier* const gZhVerifierSet[]; -extern nsEUCStatistics* const gZhStatisticsSet[]; - -#define CJK_DETECTOR_NUM_VERIFIERS 15 -extern nsVerifier* const gCJKVerifierSet[]; -extern nsEUCStatistics* const gCJKStatisticsSet[]; - -class nsEUCSampler { - public: - nsEUCSampler() { - mTotal =0; - mThreshold = 200; - mState = 0; - PRInt32 i; - for(i=0;i<94;i++) - mFirstByteCnt[i] = mSecondByteCnt[i]=0; - } - PRBool EnoughData() { return mTotal > mThreshold; } - PRBool GetSomeData() { return mTotal > 1; } - PRBool Sample(const char* aIn, PRUint32 aLen); - void CalFreq(); - float GetScore(const float* aFirstByteFreq, float aFirstByteWeight, - const float* aSecondByteFreq, float aSecondByteWeight); - float GetScore(const float* array1, const float* array2); - private: - PRUint32 mTotal; - PRUint32 mThreshold; - PRInt8 mState; - PRUint32 mFirstByteCnt[94]; - PRUint32 mSecondByteCnt[94]; - float mFirstByteFreq[94]; - float mSecondByteFreq[94]; - -}; - -/* - In the current design, we know the following combination of verifiers - are not good- - - 1. Two or more of the following verifier in one detector: - nsEUCJPVerifer - nsGB2312Verifier - nsEUCKRVerifer - nsEUCTWVerifier - nsBIG5Verifer - - */ -//---------------------------------------------------------- -class nsPSMDetector { -public : - nsPSMDetector(PRUint8 aItems, nsVerifier* const * aVerifierSet, nsEUCStatistics* const * aStatisticsSet); - virtual ~nsPSMDetector() {} - - virtual PRBool HandleData(const char* aBuf, PRUint32 aLen); - virtual void DataEnd(); - -protected: - virtual void Report(const char* charset) = 0; - - PRUint8 mItems; - PRUint8 mClassItems; - PRUint8 mState[MAX_VERIFIERS]; - PRUint8 mItemIdx[MAX_VERIFIERS]; - nsVerifier* const * mVerifier; - nsEUCStatistics* const * mStatisticsData; - PRBool mDone; - - PRBool mRunSampler; - PRBool mClassRunSampler; -protected: - void Reset(); - void Sample(const char* aBuf, PRUint32 aLen, PRBool aLastChance=PR_FALSE); -private: -#ifdef DETECTOR_DEBUG - PRUint32 mDbgTest; - PRUint32 mDbgLen; -#endif - nsEUCSampler mSampler; - -}; - -class nsXPCOMDetector : - private nsPSMDetector, - public nsICharsetDetector // Implement the interface -{ - NS_DECL_ISUPPORTS -public: - nsXPCOMDetector(PRUint8 aItems, nsVerifier* const * aVer, nsEUCStatistics* const * aStatisticsSet); - virtual ~nsXPCOMDetector(); - NS_IMETHOD Init(nsICharsetDetectionObserver* aObserver); - NS_IMETHOD DoIt(const char* aBuf, PRUint32 aLen, PRBool* oDontFeedMe); - NS_IMETHOD Done(); - -protected: - virtual void Report(const char* charset); - -private: - nsCOMPtr mObserver; -}; - -class nsXPCOMStringDetector : - private nsPSMDetector, - public nsIStringCharsetDetector // Implement the interface -{ - NS_DECL_ISUPPORTS -public: - nsXPCOMStringDetector(PRUint8 aItems, nsVerifier* const * aVer, nsEUCStatistics* const * aStatisticsSet); - virtual ~nsXPCOMStringDetector(); - NS_IMETHOD DoIt(const char* aBuf, PRUint32 aLen, - const char** oCharset, - nsDetectionConfident &oConfident); -protected: - virtual void Report(const char* charset); -private: - const char* mResult; -}; - -class nsJAPSMDetector : public nsXPCOMDetector -{ -public: - nsJAPSMDetector() - : nsXPCOMDetector(JA_DETECTOR_NUM_VERIFIERS, gJaVerifierSet, nsnull) {} -}; - -class nsJAStringPSMDetector : public nsXPCOMStringDetector -{ -public: - nsJAStringPSMDetector() - : nsXPCOMStringDetector(JA_DETECTOR_NUM_VERIFIERS - 3, gJaVerifierSet, nsnull) {} -}; - -class nsKOPSMDetector : public nsXPCOMDetector -{ -public: - nsKOPSMDetector() - : nsXPCOMDetector(KO_DETECTOR_NUM_VERIFIERS, gKoVerifierSet, nsnull){} -}; - -class nsKOStringPSMDetector : public nsXPCOMStringDetector -{ -public: - nsKOStringPSMDetector() - : nsXPCOMStringDetector(KO_DETECTOR_NUM_VERIFIERS - 3, gKoVerifierSet, nsnull) {} -}; - -class nsZHTWPSMDetector : public nsXPCOMDetector -{ -public: - nsZHTWPSMDetector() - : nsXPCOMDetector(ZHTW_DETECTOR_NUM_VERIFIERS, gZhTwVerifierSet, gZhTwStatisticsSet) {} -}; - -class nsZHTWStringPSMDetector : public nsXPCOMStringDetector -{ -public: - nsZHTWStringPSMDetector() - : nsXPCOMStringDetector(ZHTW_DETECTOR_NUM_VERIFIERS - 3, gZhTwVerifierSet, gZhTwStatisticsSet) {} -}; - -class nsZHCNPSMDetector : public nsXPCOMDetector -{ -public: - nsZHCNPSMDetector() - : nsXPCOMDetector(ZHCN_DETECTOR_NUM_VERIFIERS, gZhCnVerifierSet, nsnull) {} -}; - -class nsZHCNStringPSMDetector : public nsXPCOMStringDetector -{ -public: - nsZHCNStringPSMDetector() - : nsXPCOMStringDetector(ZHCN_DETECTOR_NUM_VERIFIERS - 3, gZhCnVerifierSet, nsnull) {} -}; - -class nsZHPSMDetector : public nsXPCOMDetector -{ -public: - nsZHPSMDetector() - : nsXPCOMDetector(ZH_DETECTOR_NUM_VERIFIERS, gZhVerifierSet, gZhStatisticsSet) {} -}; - -class nsZHStringPSMDetector : public nsXPCOMStringDetector -{ -public: - nsZHStringPSMDetector() - : nsXPCOMStringDetector(ZH_DETECTOR_NUM_VERIFIERS - 3, gZhVerifierSet, gZhStatisticsSet) {} -}; - -class nsCJKPSMDetector : public nsXPCOMDetector -{ -public: - nsCJKPSMDetector() - : nsXPCOMDetector(CJK_DETECTOR_NUM_VERIFIERS, gCJKVerifierSet, gCJKStatisticsSet) {} -}; - -class nsCJKStringPSMDetector : public nsXPCOMStringDetector -{ -public: - nsCJKStringPSMDetector() - : nsXPCOMStringDetector(CJK_DETECTOR_NUM_VERIFIERS - 3, gCJKVerifierSet, gCJKStatisticsSet) {} -}; - -#endif // nsPSMDetectors_h__ diff --git a/intl/chardet/src/nsPkgInt.h b/intl/chardet/src/nsPkgInt.h deleted file mode 100644 index 1cd2da07f5f..00000000000 --- a/intl/chardet/src/nsPkgInt.h +++ /dev/null @@ -1,88 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#ifndef nsPkgInt_h__ -#define nsPkgInt_h__ -#include "nscore.h" - -typedef enum { - eIdxSft4bits = 3, - eIdxSft8bits = 2, - eIdxSft16bits = 1 -} nsIdxSft; - -typedef enum { - eSftMsk4bits = 7, - eSftMsk8bits = 3, - eSftMsk16bits = 1 -} nsSftMsk; - -typedef enum { - eBitSft4bits = 2, - eBitSft8bits = 3, - eBitSft16bits = 4 -} nsBitSft; - -typedef enum { - eUnitMsk4bits = 0x0000000FL, - eUnitMsk8bits = 0x000000FFL, - eUnitMsk16bits = 0x0000FFFFL -} nsUnitMsk; - -typedef struct nsPkgInt { - nsIdxSft idxsft; - nsSftMsk sftmsk; - nsBitSft bitsft; - nsUnitMsk unitmsk; - const PRUint32 *data; -} nsPkgInt; - - -#define PCK16BITS(a,b) ((PRUint32)(((b) << 16) | (a))) - -#define PCK8BITS(a,b,c,d) PCK16BITS( ((PRUint32)(((b) << 8) | (a))), \ - ((PRUint32)(((d) << 8) | (c)))) - -#define PCK4BITS(a,b,c,d,e,f,g,h) PCK8BITS( ((PRUint32)(((b) << 4) | (a))), \ - ((PRUint32)(((d) << 4) | (c))), \ - ((PRUint32)(((f) << 4) | (e))), \ - ((PRUint32)(((h) << 4) | (g))) ) - -#define GETFROMPCK(i, c) \ - (((((c).data)[(i)>>(c).idxsft])>>(((i)&(c).sftmsk)<<(c).bitsft))&(c).unitmsk) - -#endif /* nsPkgInt_h__ */ diff --git a/intl/chardet/src/nsSJISVerifier.h b/intl/chardet/src/nsSJISVerifier.h deleted file mode 100644 index 9e218308729..00000000000 --- a/intl/chardet/src/nsSJISVerifier.h +++ /dev/null @@ -1,105 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -/* - * DO NOT EDIT THIS DOCUMENT MANUALLY !!! - * THIS FILE IS AUTOMATICALLY GENERATED BY THE TOOLS UNDER - * mozilla/intl/chardet/tools/ - * Please contact ftang@netscape.com or mozilla-i18n@mozilla.org - * if you have any question. Thanks - */ -#include "nsVerifier.h" -static const PRUint32 SJIS_cls [ 256 / 8 ] = { -PCK4BITS(1,1,1,1,1,1,1,1), // 00 - 07 -PCK4BITS(1,1,1,1,1,1,0,0), // 08 - 0f -PCK4BITS(1,1,1,1,1,1,1,1), // 10 - 17 -PCK4BITS(1,1,1,0,1,1,1,1), // 18 - 1f -PCK4BITS(1,1,1,1,1,1,1,1), // 20 - 27 -PCK4BITS(1,1,1,1,1,1,1,1), // 28 - 2f -PCK4BITS(1,1,1,1,1,1,1,1), // 30 - 37 -PCK4BITS(1,1,1,1,1,1,1,1), // 38 - 3f -PCK4BITS(2,2,2,2,2,2,2,2), // 40 - 47 -PCK4BITS(2,2,2,2,2,2,2,2), // 48 - 4f -PCK4BITS(2,2,2,2,2,2,2,2), // 50 - 57 -PCK4BITS(2,2,2,2,2,2,2,2), // 58 - 5f -PCK4BITS(2,2,2,2,2,2,2,2), // 60 - 67 -PCK4BITS(2,2,2,2,2,2,2,2), // 68 - 6f -PCK4BITS(2,2,2,2,2,2,2,2), // 70 - 77 -PCK4BITS(2,2,2,2,2,2,2,1), // 78 - 7f -PCK4BITS(3,3,3,3,3,3,3,3), // 80 - 87 -PCK4BITS(3,3,3,3,3,3,3,3), // 88 - 8f -PCK4BITS(3,3,3,3,3,3,3,3), // 90 - 97 -PCK4BITS(3,3,3,3,3,3,3,3), // 98 - 9f -PCK4BITS(4,2,2,2,2,2,2,2), // a0 - a7 -PCK4BITS(2,2,2,2,2,2,2,2), // a8 - af -PCK4BITS(2,2,2,2,2,2,2,2), // b0 - b7 -PCK4BITS(2,2,2,2,2,2,2,2), // b8 - bf -PCK4BITS(2,2,2,2,2,2,2,2), // c0 - c7 -PCK4BITS(2,2,2,2,2,2,2,2), // c8 - cf -PCK4BITS(2,2,2,2,2,2,2,2), // d0 - d7 -PCK4BITS(2,2,2,2,2,2,2,2), // d8 - df -PCK4BITS(3,3,3,3,3,3,3,3), // e0 - e7 -PCK4BITS(3,3,3,3,3,4,4,4), // e8 - ef -PCK4BITS(4,4,4,4,4,4,4,4), // f0 - f7 -PCK4BITS(4,4,4,4,4,0,0,0) // f8 - ff -}; - - -static PRUint32 const SJIS_st [ 3] = { -PCK4BITS(eError,eStart,eStart, 3,eError,eError,eError,eError),//00-07 -PCK4BITS(eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe),//08-0f -PCK4BITS(eItsMe,eItsMe,eError,eError,eStart,eStart,eStart,eStart) //10-17 -}; - - -static nsVerifier nsSJISVerifier = { - "Shift_JIS", - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - SJIS_cls - }, - 6, - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - SJIS_st - } -}; diff --git a/intl/chardet/src/nsUCS2BEVerifier.h b/intl/chardet/src/nsUCS2BEVerifier.h deleted file mode 100644 index 27d73e6688a..00000000000 --- a/intl/chardet/src/nsUCS2BEVerifier.h +++ /dev/null @@ -1,109 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -/* - * DO NOT EDIT THIS DOCUMENT MANUALLY !!! - * THIS FILE IS AUTOMATICALLY GENERATED BY THE TOOLS UNDER - * mozilla/intl/chardet/tools/ - * Please contact ftang@netscape.com or mozilla-i18n@mozilla.org - * if you have any question. Thanks - */ -#include "nsVerifier.h" -static const PRUint32 UCS2BE_cls [ 256 / 8 ] = { -PCK4BITS(0,0,0,0,0,0,0,0), // 00 - 07 -PCK4BITS(0,0,1,0,0,2,0,0), // 08 - 0f -PCK4BITS(0,0,0,0,0,0,0,0), // 10 - 17 -PCK4BITS(0,0,0,3,0,0,0,0), // 18 - 1f -PCK4BITS(0,0,0,0,0,0,0,0), // 20 - 27 -PCK4BITS(0,3,3,3,3,3,0,0), // 28 - 2f -PCK4BITS(0,0,0,0,0,0,0,0), // 30 - 37 -PCK4BITS(0,0,0,0,0,0,0,0), // 38 - 3f -PCK4BITS(0,0,0,0,0,0,0,0), // 40 - 47 -PCK4BITS(0,0,0,0,0,0,0,0), // 48 - 4f -PCK4BITS(0,0,0,0,0,0,0,0), // 50 - 57 -PCK4BITS(0,0,0,0,0,0,0,0), // 58 - 5f -PCK4BITS(0,0,0,0,0,0,0,0), // 60 - 67 -PCK4BITS(0,0,0,0,0,0,0,0), // 68 - 6f -PCK4BITS(0,0,0,0,0,0,0,0), // 70 - 77 -PCK4BITS(0,0,0,0,0,0,0,0), // 78 - 7f -PCK4BITS(0,0,0,0,0,0,0,0), // 80 - 87 -PCK4BITS(0,0,0,0,0,0,0,0), // 88 - 8f -PCK4BITS(0,0,0,0,0,0,0,0), // 90 - 97 -PCK4BITS(0,0,0,0,0,0,0,0), // 98 - 9f -PCK4BITS(0,0,0,0,0,0,0,0), // a0 - a7 -PCK4BITS(0,0,0,0,0,0,0,0), // a8 - af -PCK4BITS(0,0,0,0,0,0,0,0), // b0 - b7 -PCK4BITS(0,0,0,0,0,0,0,0), // b8 - bf -PCK4BITS(0,0,0,0,0,0,0,0), // c0 - c7 -PCK4BITS(0,0,0,0,0,0,0,0), // c8 - cf -PCK4BITS(0,0,0,0,0,0,0,0), // d0 - d7 -PCK4BITS(0,0,0,0,0,0,0,0), // d8 - df -PCK4BITS(0,0,0,0,0,0,0,0), // e0 - e7 -PCK4BITS(0,0,0,0,0,0,0,0), // e8 - ef -PCK4BITS(0,0,0,0,0,0,0,0), // f0 - f7 -PCK4BITS(0,0,0,0,0,0,4,5) // f8 - ff -}; - - -static const PRUint32 UCS2BE_st [ 7] = { -PCK4BITS( 5, 7, 7,eError, 4, 3,eError,eError),//00-07 -PCK4BITS(eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe),//08-0f -PCK4BITS(eItsMe,eItsMe, 6, 6, 6, 6,eError,eError),//10-17 -PCK4BITS( 6, 6, 6, 6, 6,eItsMe, 6, 6),//18-1f -PCK4BITS( 6, 6, 6, 6, 5, 7, 7,eError),//20-27 -PCK4BITS( 5, 8, 6, 6,eError, 6, 6, 6),//28-2f -PCK4BITS( 6, 6, 6, 6,eError,eError,eStart,eStart) //30-37 -}; - - -static nsVerifier nsUCS2BEVerifier = { - "UTF-16BE", - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - UCS2BE_cls - }, - 6, - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - UCS2BE_st - } -}; diff --git a/intl/chardet/src/nsUCS2LEVerifier.h b/intl/chardet/src/nsUCS2LEVerifier.h deleted file mode 100644 index f76b8b07801..00000000000 --- a/intl/chardet/src/nsUCS2LEVerifier.h +++ /dev/null @@ -1,109 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -/* - * DO NOT EDIT THIS DOCUMENT MANUALLY !!! - * THIS FILE IS AUTOMATICALLY GENERATED BY THE TOOLS UNDER - * mozilla/intl/chardet/tools/ - * Please contact ftang@netscape.com or mozilla-i18n@mozilla.org - * if you have any question. Thanks - */ -#include "nsVerifier.h" -static const PRUint32 UCS2LE_cls [ 256 / 8 ] = { -PCK4BITS(0,0,0,0,0,0,0,0), // 00 - 07 -PCK4BITS(0,0,1,0,0,2,0,0), // 08 - 0f -PCK4BITS(0,0,0,0,0,0,0,0), // 10 - 17 -PCK4BITS(0,0,0,3,0,0,0,0), // 18 - 1f -PCK4BITS(0,0,0,0,0,0,0,0), // 20 - 27 -PCK4BITS(0,3,3,3,3,3,0,0), // 28 - 2f -PCK4BITS(0,0,0,0,0,0,0,0), // 30 - 37 -PCK4BITS(0,0,0,0,0,0,0,0), // 38 - 3f -PCK4BITS(0,0,0,0,0,0,0,0), // 40 - 47 -PCK4BITS(0,0,0,0,0,0,0,0), // 48 - 4f -PCK4BITS(0,0,0,0,0,0,0,0), // 50 - 57 -PCK4BITS(0,0,0,0,0,0,0,0), // 58 - 5f -PCK4BITS(0,0,0,0,0,0,0,0), // 60 - 67 -PCK4BITS(0,0,0,0,0,0,0,0), // 68 - 6f -PCK4BITS(0,0,0,0,0,0,0,0), // 70 - 77 -PCK4BITS(0,0,0,0,0,0,0,0), // 78 - 7f -PCK4BITS(0,0,0,0,0,0,0,0), // 80 - 87 -PCK4BITS(0,0,0,0,0,0,0,0), // 88 - 8f -PCK4BITS(0,0,0,0,0,0,0,0), // 90 - 97 -PCK4BITS(0,0,0,0,0,0,0,0), // 98 - 9f -PCK4BITS(0,0,0,0,0,0,0,0), // a0 - a7 -PCK4BITS(0,0,0,0,0,0,0,0), // a8 - af -PCK4BITS(0,0,0,0,0,0,0,0), // b0 - b7 -PCK4BITS(0,0,0,0,0,0,0,0), // b8 - bf -PCK4BITS(0,0,0,0,0,0,0,0), // c0 - c7 -PCK4BITS(0,0,0,0,0,0,0,0), // c8 - cf -PCK4BITS(0,0,0,0,0,0,0,0), // d0 - d7 -PCK4BITS(0,0,0,0,0,0,0,0), // d8 - df -PCK4BITS(0,0,0,0,0,0,0,0), // e0 - e7 -PCK4BITS(0,0,0,0,0,0,0,0), // e8 - ef -PCK4BITS(0,0,0,0,0,0,0,0), // f0 - f7 -PCK4BITS(0,0,0,0,0,0,4,5) // f8 - ff -}; - - -static const PRUint32 UCS2LE_st [ 7] = { -PCK4BITS( 6, 6, 7, 6, 4, 3,eError,eError),//00-07 -PCK4BITS(eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe),//08-0f -PCK4BITS(eItsMe,eItsMe, 5, 5, 5,eError,eItsMe,eError),//10-17 -PCK4BITS( 5, 5, 5,eError, 5,eError, 6, 6),//18-1f -PCK4BITS( 7, 6, 8, 8, 5, 5, 5,eError),//20-27 -PCK4BITS( 5, 5, 5,eError,eError,eError, 5, 5),//28-2f -PCK4BITS( 5, 5, 5,eError, 5,eError,eStart,eStart) //30-37 -}; - - -static nsVerifier nsUCS2LEVerifier = { - "UTF-16LE", - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - UCS2LE_cls - }, - 6, - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - UCS2LE_st - } -}; diff --git a/intl/chardet/src/nsUTF8Verifier.h b/intl/chardet/src/nsUTF8Verifier.h deleted file mode 100644 index 59d7ddfd2c8..00000000000 --- a/intl/chardet/src/nsUTF8Verifier.h +++ /dev/null @@ -1,128 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -/* - * DO NOT EDIT THIS DOCUMENT MANUALLY !!! - * THIS FILE IS AUTOMATICALLY GENERATED BY THE TOOLS UNDER - * mozilla/intl/chardet/tools/ - * Please contact ftang@netscape.com or mozilla-i18n@mozilla.org - * if you have any question. Thanks - */ -#include "nsVerifier.h" -static const PRUint32 UTF8_cls [ 256 / 8 ] = { -PCK4BITS(1,1,1,1,1,1,1,1), // 00 - 07 -PCK4BITS(1,1,1,1,1,1,0,0), // 08 - 0f -PCK4BITS(1,1,1,1,1,1,1,1), // 10 - 17 -PCK4BITS(1,1,1,0,1,1,1,1), // 18 - 1f -PCK4BITS(1,1,1,1,1,1,1,1), // 20 - 27 -PCK4BITS(1,1,1,1,1,1,1,1), // 28 - 2f -PCK4BITS(1,1,1,1,1,1,1,1), // 30 - 37 -PCK4BITS(1,1,1,1,1,1,1,1), // 38 - 3f -PCK4BITS(1,1,1,1,1,1,1,1), // 40 - 47 -PCK4BITS(1,1,1,1,1,1,1,1), // 48 - 4f -PCK4BITS(1,1,1,1,1,1,1,1), // 50 - 57 -PCK4BITS(1,1,1,1,1,1,1,1), // 58 - 5f -PCK4BITS(1,1,1,1,1,1,1,1), // 60 - 67 -PCK4BITS(1,1,1,1,1,1,1,1), // 68 - 6f -PCK4BITS(1,1,1,1,1,1,1,1), // 70 - 77 -PCK4BITS(1,1,1,1,1,1,1,1), // 78 - 7f -PCK4BITS(2,2,2,2,3,3,3,3), // 80 - 87 -PCK4BITS(4,4,4,4,4,4,4,4), // 88 - 8f -PCK4BITS(4,4,4,4,4,4,4,4), // 90 - 97 -PCK4BITS(4,4,4,4,4,4,4,4), // 98 - 9f -PCK4BITS(5,5,5,5,5,5,5,5), // a0 - a7 -PCK4BITS(5,5,5,5,5,5,5,5), // a8 - af -PCK4BITS(5,5,5,5,5,5,5,5), // b0 - b7 -PCK4BITS(5,5,5,5,5,5,5,5), // b8 - bf -PCK4BITS(0,0,6,6,6,6,6,6), // c0 - c7 -PCK4BITS(6,6,6,6,6,6,6,6), // c8 - cf -PCK4BITS(6,6,6,6,6,6,6,6), // d0 - d7 -PCK4BITS(6,6,6,6,6,6,6,6), // d8 - df -PCK4BITS(7,8,8,8,8,8,8,8), // e0 - e7 -PCK4BITS(8,8,8,8,8,9,8,8), // e8 - ef -PCK4BITS(10,11,11,11,11,11,11,11), // f0 - f7 -PCK4BITS(12,13,13,13,14,15,0,0) // f8 - ff -}; - - -static const PRUint32 UTF8_st [ 26] = { -PCK4BITS(eError,eStart,eError,eError,eError,eError, 12, 10),//00-07 -PCK4BITS( 9, 11, 8, 7, 6, 5, 4, 3),//08-0f -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//10-17 -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//18-1f -PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe),//20-27 -PCK4BITS(eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe),//28-2f -PCK4BITS(eError,eError, 5, 5, 5, 5,eError,eError),//30-37 -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//38-3f -PCK4BITS(eError,eError,eError, 5, 5, 5,eError,eError),//40-47 -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//48-4f -PCK4BITS(eError,eError, 7, 7, 7, 7,eError,eError),//50-57 -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//58-5f -PCK4BITS(eError,eError,eError,eError, 7, 7,eError,eError),//60-67 -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//68-6f -PCK4BITS(eError,eError, 9, 9, 9, 9,eError,eError),//70-77 -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//78-7f -PCK4BITS(eError,eError,eError,eError,eError, 9,eError,eError),//80-87 -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//88-8f -PCK4BITS(eError,eError, 12, 12, 12, 12,eError,eError),//90-97 -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//98-9f -PCK4BITS(eError,eError,eError,eError,eError, 12,eError,eError),//a0-a7 -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//a8-af -PCK4BITS(eError,eError, 12, 12, 12,eError,eError,eError),//b0-b7 -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError),//b8-bf -PCK4BITS(eError,eError,eStart,eStart,eStart,eStart,eError,eError),//c0-c7 -PCK4BITS(eError,eError,eError,eError,eError,eError,eError,eError) //c8-cf -}; - - -static nsVerifier nsUTF8Verifier = { - "UTF-8", - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - UTF8_cls - }, - 16, - { - eIdxSft4bits, - eSftMsk4bits, - eBitSft4bits, - eUnitMsk4bits, - UTF8_st - } -}; diff --git a/intl/chardet/src/nsVerifier.h b/intl/chardet/src/nsVerifier.h deleted file mode 100644 index 2c01b6aff9d..00000000000 --- a/intl/chardet/src/nsVerifier.h +++ /dev/null @@ -1,61 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is mozilla.org code. - * - * The Initial Developer of the Original Code is - * Netscape Communications Corporation. - * Portions created by the Initial Developer are Copyright (C) 1998 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ -#ifndef nsVerifier_h__ -#define nsVerifier_h__ - -#include "nsPkgInt.h" - -typedef enum { - eStart = 0, - eError = 1, - eItsMe = 2 -} nsSMState; - -typedef struct _nsVerifierMutable { - const char* charset; - nsPkgInt cclass; - PRUint32 stFactor; // >= number of cclass. - nsPkgInt states; -} nsVerifierMutable; - -typedef const nsVerifierMutable nsVerifier; - -#define GETCLASS(v,c) GETFROMPCK(((unsigned char)(c)), (v)->cclass) -#define GETNEXTSTATE(v,c,s) \ - GETFROMPCK((s)*((v)->stFactor)+GETCLASS((v),(c)), ((v)->states)) - -#endif /* nsVerifier_h__ */ diff --git a/intl/chardet/tests/DetectCharset.cpp b/intl/chardet/tests/DetectCharset.cpp index b211c5dbd4f..4668dd0cffd 100644 --- a/intl/chardet/tests/DetectCharset.cpp +++ b/intl/chardet/tests/DetectCharset.cpp @@ -352,9 +352,13 @@ int main(int argc, char** argv) { sz = read(0, buf, bs); if(sz > 0) { if(! done) { -printf("call DoIt %d\n",sz); +#ifdef DEBUG_DetectCharset + printf("call DoIt %d\n",sz); +#endif rev = det->DoIt( buf, sz, &done); -printf("DoIt return Done = %d\n",done); +#ifdef DEBUG_DetectCharset + printf("DoIt return Done = %d\n",done); +#endif if(NS_FAILED(rev)) { printf("XPCOM ERROR CODE = %x\n", rev); @@ -368,8 +372,10 @@ printf("DoIt return Done = %d\n",done); } while(sz > 0); if(!done) { -printf("Done = %d\n",done); -printf("call Done %d\n",sz); +#ifdef DEBUG_DetectCharset + printf("Done = %d\n",done); + printf("call Done %d\n",sz); +#endif rev = det->Done(); if(NS_FAILED(rev)) { @@ -381,9 +387,13 @@ printf("call Done %d\n",sz); stat[i]->DataEnd(); stat[i]->Report(); } -printf( "Done\n"); +#ifdef DEBUG_DetectCharset + printf( "Done\n"); +#endif NS_IF_RELEASE(det); -printf( "Done 2\n"); +#ifdef DEBUG_DetectCharset + printf( "Done 2\n"); +#endif return (0); }