From 7abc056d03d96bfeab8d7bca7c9de14ead7c324f Mon Sep 17 00:00:00 2001 From: Jorg K Date: Wed, 24 Apr 2019 10:34:36 +0200 Subject: [PATCH] Bug 1546364 - Reformat to Google coding style in mailnews/extensions. rs=reformat # ignore-this-changeset --- mailnews/base/public/nsIMsgMdnGenerator.idl | 10 - .../src/nsBayesianFilter.cpp | 2956 +++++------ .../src/nsBayesianFilter.h | 583 +-- .../src/nsBayesianFilterCID.h | 10 +- .../src/nsIncompleteGamma.h | 122 +- mailnews/extensions/fts3/src/Normalize.c | 4599 ++++++++++------- mailnews/extensions/fts3/src/fts3_porter.c | 728 ++- mailnews/extensions/fts3/src/fts3_tokenizer.h | 28 +- .../extensions/fts3/src/nsFts3Tokenizer.cpp | 45 +- .../extensions/fts3/src/nsFts3Tokenizer.h | 10 +- .../extensions/fts3/src/nsFts3TokenizerCID.h | 12 +- .../fts3/src/nsGlodaRankerFunction.cpp | 39 +- .../fts3/src/nsGlodaRankerFunction.h | 10 +- .../mailviews/src/nsMsgMailViewList.cpp | 402 +- .../mailviews/src/nsMsgMailViewList.h | 22 +- .../mailviews/src/nsMsgMailViewsCID.h | 13 +- mailnews/extensions/mdn/src/nsMsgMdnCID.h | 13 +- .../extensions/mdn/src/nsMsgMdnGenerator.cpp | 1648 +++--- .../extensions/mdn/src/nsMsgMdnGenerator.h | 29 +- .../extensions/smime/src/nsCertPicker.cpp | 128 +- mailnews/extensions/smime/src/nsCertPicker.h | 18 +- .../smime/src/nsEncryptedSMIMEURIsService.cpp | 20 +- .../smime/src/nsEncryptedSMIMEURIsService.h | 7 +- .../smime/src/nsMsgComposeSecure.cpp | 642 +-- .../extensions/smime/src/nsMsgComposeSecure.h | 32 +- mailnews/extensions/smime/src/nsMsgSMIMECID.h | 34 +- .../extensions/smime/src/nsSMimeJSHelper.cpp | 153 +- .../extensions/smime/src/nsSMimeJSHelper.h | 9 +- 28 files changed, 6254 insertions(+), 6068 deletions(-) diff --git a/mailnews/base/public/nsIMsgMdnGenerator.idl b/mailnews/base/public/nsIMsgMdnGenerator.idl index 905e640e93..55bd936fce 100644 --- a/mailnews/base/public/nsIMsgMdnGenerator.idl +++ b/mailnews/base/public/nsIMsgMdnGenerator.idl @@ -14,16 +14,6 @@ typedef long EDisposeType; typedef long ReceiptHdrType; typedef long MDNIncorporateType; -%{C++ -#define NS_MSGMDNGENERATOR_CONTRACTID \ - "@mozilla.org/messenger-mdn/generator;1" - -#define NS_MSGMDNGENERATOR_CID \ -{ /* ec917b13-8f73-4d4d-9146-d7f7aafe9076 */ \ - 0xec917b13, 0x8f73, 0x4d4d, \ - { 0x91, 0x46, 0xd7, 0xf7, 0xaa, 0xfe, 0x90, 0x76 }} -%} - [scriptable, uuid(440EA3DE-DACA-4886-9875-84E6CD7D7927)] interface nsIMsgMdnGenerator : nsISupports { diff --git a/mailnews/extensions/bayesian-spam-filter/src/nsBayesianFilter.cpp b/mailnews/extensions/bayesian-spam-filter/src/nsBayesianFilter.cpp index 0a2ccf9ccf..3ca87e7e12 100644 --- a/mailnews/extensions/bayesian-spam-filter/src/nsBayesianFilter.cpp +++ b/mailnews/extensions/bayesian-spam-filter/src/nsBayesianFilter.cpp @@ -9,7 +9,7 @@ #include "nsNetUtil.h" #include "nsQuickSort.h" #include "nsIMsgMessageService.h" -#include "nsMsgUtils.h" // for GetMessageServiceFromURI +#include "nsMsgUtils.h" // for GetMessageServiceFromURI #include "prnetdb.h" #include "nsIMsgWindow.h" #include "mozilla/Logging.h" @@ -29,7 +29,7 @@ #include "nsDependentSubstring.h" #include "nsMemory.h" -#include "mozilla/ArenaAllocatorExtensions.h" // for ArenaStrdup +#include "mozilla/ArenaAllocatorExtensions.h" // for ArenaStrdup using namespace mozilla; @@ -47,73 +47,71 @@ using namespace mozilla; #include "nsIMsgTraitService.h" #include "mozilla/Services.h" #include "mozilla/Attributes.h" -#include // for std::abs(int/long) -#include // for std::abs(float/double) +#include // for std::abs(int/long) +#include // for std::abs(float/double) static mozilla::LazyLogModule BayesianFilterLogModule("BayesianFilter"); -#define kDefaultJunkThreshold .99 // we override this value via a pref +#define kDefaultJunkThreshold .99 // we override this value via a pref static const char* kBayesianFilterTokenDelimiters = " \t\n\r\f."; -static unsigned int kMinLengthForToken = 3; // lower bound on the number of characters in a word before we treat it as a token -static unsigned int kMaxLengthForToken = 12; // upper bound on the number of characters in a word to be declared as a token +static unsigned int kMinLengthForToken = + 3; // lower bound on the number of characters in a word before we treat it + // as a token +static unsigned int kMaxLengthForToken = + 12; // upper bound on the number of characters in a word to be declared as + // a token #define FORGED_RECEIVED_HEADER_HINT NS_LITERAL_CSTRING("may be forged") #ifndef M_LN2 -#define M_LN2 0.69314718055994530942 +# define M_LN2 0.69314718055994530942 #endif #ifndef M_E -#define M_E 2.7182818284590452354 +# define M_E 2.7182818284590452354 #endif // provide base implementation of hash lookup of a string -struct BaseToken : public PLDHashEntryHdr -{ - const char* mWord; +struct BaseToken : public PLDHashEntryHdr { + const char* mWord; }; // token for a particular message // mCount, mAnalysisLink are initialized to zero by the hash code struct Token : public BaseToken { - uint32_t mCount; - uint32_t mAnalysisLink; // index in mAnalysisStore of the AnalysisPerToken - // object for the first trait for this token - // Helper to support Tokenizer::copyTokens() - void clone(const Token& other) { - mWord = other.mWord; - mCount = other.mCount; - mAnalysisLink = other.mAnalysisLink; - } + uint32_t mCount; + uint32_t mAnalysisLink; // index in mAnalysisStore of the AnalysisPerToken + // object for the first trait for this token + // Helper to support Tokenizer::copyTokens() + void clone(const Token& other) { + mWord = other.mWord; + mCount = other.mCount; + mAnalysisLink = other.mAnalysisLink; + } }; // token stored in a training file for a group of messages // mTraitLink is initialized to 0 by the hash code -struct CorpusToken : public BaseToken -{ - uint32_t mTraitLink; // index in mTraitStore of the TraitPerToken - // object for the first trait for this token +struct CorpusToken : public BaseToken { + uint32_t mTraitLink; // index in mTraitStore of the TraitPerToken + // object for the first trait for this token }; // set the value of a TraitPerToken object TraitPerToken::TraitPerToken(uint32_t aTraitId, uint32_t aCount) - : mId(aTraitId), mCount(aCount), mNextLink(0) -{ -} + : mId(aTraitId), mCount(aCount), mNextLink(0) {} // shorthand representations of trait ids for junk and good static const uint32_t kJunkTrait = nsIJunkMailPlugin::JUNK_TRAIT; static const uint32_t kGoodTrait = nsIJunkMailPlugin::GOOD_TRAIT; // set the value of an AnalysisPerToken object -AnalysisPerToken::AnalysisPerToken( - uint32_t aTraitIndex, double aDistance, double aProbability) : - mTraitIndex(aTraitIndex), - mDistance(aDistance), - mProbability(aProbability), - mNextLink(0) -{ -} +AnalysisPerToken::AnalysisPerToken(uint32_t aTraitIndex, double aDistance, + double aProbability) + : mTraitIndex(aTraitIndex), + mDistance(aDistance), + mProbability(aProbability), + mNextLink(0) {} // the initial size of the AnalysisPerToken linked list storage const uint32_t kAnalysisStoreCapacity = 2048; @@ -125,118 +123,98 @@ const uint32_t kTraitStoreCapacity = 16384; const uint32_t kTraitAutoCapacity = 10; TokenEnumeration::TokenEnumeration(PLDHashTable* table) - : mIterator(table->Iter()) -{ -} + : mIterator(table->Iter()) {} -inline bool TokenEnumeration::hasMoreTokens() -{ - return !mIterator.Done(); -} +inline bool TokenEnumeration::hasMoreTokens() { return !mIterator.Done(); } -inline BaseToken* TokenEnumeration::nextToken() -{ - auto token = static_cast(mIterator.Get()); - mIterator.Next(); - return token; +inline BaseToken* TokenEnumeration::nextToken() { + auto token = static_cast(mIterator.Get()); + mIterator.Next(); + return token; } // member variables static const PLDHashTableOps gTokenTableOps = { - PLDHashTable::HashStringKey, - PLDHashTable::MatchStringKey, - PLDHashTable::MoveEntryStub, - PLDHashTable::ClearEntryStub, - nullptr -}; + PLDHashTable::HashStringKey, PLDHashTable::MatchStringKey, + PLDHashTable::MoveEntryStub, PLDHashTable::ClearEntryStub, nullptr}; TokenHash::TokenHash(uint32_t aEntrySize) - : mTokenTable(&gTokenTableOps, aEntrySize, 128) -{ - mEntrySize = aEntrySize; + : mTokenTable(&gTokenTableOps, aEntrySize, 128) { + mEntrySize = aEntrySize; } -TokenHash::~TokenHash() -{ +TokenHash::~TokenHash() {} + +nsresult TokenHash::clearTokens() { + // we re-use the tokenizer when classifying multiple messages, + // so this gets called after every message classification. + mTokenTable.ClearAndPrepareForLength(128); + mWordPool.Clear(); + return NS_OK; } -nsresult TokenHash::clearTokens() -{ - // we re-use the tokenizer when classifying multiple messages, - // so this gets called after every message classification. - mTokenTable.ClearAndPrepareForLength(128); - mWordPool.Clear(); - return NS_OK; +char* TokenHash::copyWord(const char* word, uint32_t len) { + return ArenaStrdup(Substring(word, len), mWordPool); } -char* TokenHash::copyWord(const char* word, uint32_t len) -{ - return ArenaStrdup(Substring(word, len), mWordPool); +inline BaseToken* TokenHash::get(const char* word) { + PLDHashEntryHdr* entry = mTokenTable.Search(word); + if (entry) return static_cast(entry); + return NULL; } -inline BaseToken* TokenHash::get(const char* word) -{ - PLDHashEntryHdr* entry = mTokenTable.Search(word); - if (entry) - return static_cast(entry); - return NULL; -} +BaseToken* TokenHash::add(const char* word) { + if (!word || !*word) { + NS_ERROR("Trying to add a null word"); + return nullptr; + } -BaseToken* TokenHash::add(const char* word) -{ - if (!word || !*word) - { - NS_ERROR("Trying to add a null word"); - return nullptr; + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, ("add word: %s", word)); + + PLDHashEntryHdr* entry = mTokenTable.Add(word, mozilla::fallible); + BaseToken* token = static_cast(entry); + if (token) { + if (token->mWord == NULL) { + uint32_t len = strlen(word); + NS_ASSERTION(len != 0, "adding zero length word to tokenizer"); + if (!len) + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, + ("adding zero length word to tokenizer")); + token->mWord = copyWord(word, len); + NS_ASSERTION(token->mWord, "copyWord failed"); + if (!token->mWord) { + MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, + ("copyWord failed: %s (%d)", word, len)); + mTokenTable.RawRemove(entry); + return NULL; + } } - - MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, ("add word: %s", word)); - - PLDHashEntryHdr* entry = mTokenTable.Add(word, mozilla::fallible); - BaseToken* token = static_cast(entry); - if (token) { - if (token->mWord == NULL) { - uint32_t len = strlen(word); - NS_ASSERTION(len != 0, "adding zero length word to tokenizer"); - if (!len) - MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, ("adding zero length word to tokenizer")); - token->mWord = copyWord(word, len); - NS_ASSERTION(token->mWord, "copyWord failed"); - if (!token->mWord) { - MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, ("copyWord failed: %s (%d)", word, len)); - mTokenTable.RawRemove(entry); - return NULL; - } - } - } - return token; + } + return token; } -inline uint32_t TokenHash::countTokens() -{ - return mTokenTable.EntryCount(); -} +inline uint32_t TokenHash::countTokens() { return mTokenTable.EntryCount(); } -inline TokenEnumeration TokenHash::getTokens() -{ +inline TokenEnumeration TokenHash::getTokens() { return TokenEnumeration(&mTokenTable); } -Tokenizer::Tokenizer() : - TokenHash(sizeof(Token)), - mBodyDelimiters(kBayesianFilterTokenDelimiters), - mHeaderDelimiters(kBayesianFilterTokenDelimiters), - mCustomHeaderTokenization(false), - mMaxLengthForToken(kMaxLengthForToken), - mIframeToDiv(false) -{ +Tokenizer::Tokenizer() + : TokenHash(sizeof(Token)), + mBodyDelimiters(kBayesianFilterTokenDelimiters), + mHeaderDelimiters(kBayesianFilterTokenDelimiters), + mCustomHeaderTokenization(false), + mMaxLengthForToken(kMaxLengthForToken), + mIframeToDiv(false) { nsresult rv; - nsCOMPtr prefs = do_GetService(NS_PREFSERVICE_CONTRACTID, &rv); + nsCOMPtr prefs = + do_GetService(NS_PREFSERVICE_CONTRACTID, &rv); NS_ENSURE_SUCCESS_VOID(rv); nsCOMPtr prefBranch; - rv = prefs->GetBranch("mailnews.bayesian_spam_filter.", getter_AddRefs(prefBranch)); - NS_ENSURE_SUCCESS_VOID(rv); // no branch defined, just use defaults + rv = prefs->GetBranch("mailnews.bayesian_spam_filter.", + getter_AddRefs(prefBranch)); + NS_ENSURE_SUCCESS_VOID(rv); // no branch defined, just use defaults /* * RSS feeds store their summary as alternate content of an iframe. But due @@ -244,8 +222,7 @@ Tokenizer::Tokenizer() : * the tokenizer to replace the iframe with div for tokenization. */ rv = prefBranch->GetBoolPref("iframe_to_div", &mIframeToDiv); - if (NS_FAILED(rv)) - mIframeToDiv = false; + if (NS_FAILED(rv)) mIframeToDiv = false; /* * the list of delimiters used to tokenize the message and body @@ -261,7 +238,7 @@ Tokenizer::Tokenizer() : prefBranch->GetCharPref("body_delimiters", mBodyDelimiters); if (!mBodyDelimiters.IsEmpty()) UnescapeCString(mBodyDelimiters); - else // prefBranch empties the result when it fails :( + else // prefBranch empties the result when it fails :( mBodyDelimiters.Assign(kBayesianFilterTokenDelimiters); prefBranch->GetCharPref("header_delimiters", mHeaderDelimiters); @@ -306,29 +283,27 @@ Tokenizer::Tokenizer() : // get customized maximum token length int32_t maxLengthForToken; rv = prefBranch->GetIntPref("maxlengthfortoken", &maxLengthForToken); - mMaxLengthForToken = NS_SUCCEEDED(rv) ? uint32_t(maxLengthForToken) : kMaxLengthForToken; + mMaxLengthForToken = + NS_SUCCEEDED(rv) ? uint32_t(maxLengthForToken) : kMaxLengthForToken; - rv = prefs->GetBranch("mailnews.bayesian_spam_filter.tokenizeheader.", getter_AddRefs(prefBranch)); - if (NS_SUCCEEDED(rv)) - rv = prefBranch->GetChildList("", &count, &headers); + rv = prefs->GetBranch("mailnews.bayesian_spam_filter.tokenizeheader.", + getter_AddRefs(prefBranch)); + if (NS_SUCCEEDED(rv)) rv = prefBranch->GetChildList("", &count, &headers); - if (NS_SUCCEEDED(rv)) - { + if (NS_SUCCEEDED(rv)) { mCustomHeaderTokenization = true; - for (uint32_t i = 0; i < count; i++) - { + for (uint32_t i = 0; i < count; i++) { nsCString value; prefBranch->GetCharPref(headers[i], value); - if (value.EqualsLiteral("false")) - { + if (value.EqualsLiteral("false")) { mDisabledHeaders.AppendElement(headers[i]); continue; } mEnabledHeaders.AppendElement(headers[i]); if (value.EqualsLiteral("standard")) - value.SetIsVoid(true); // Void means use default delimiter + value.SetIsVoid(true); // Void means use default delimiter else if (value.EqualsLiteral("full")) - value.Truncate(); // Empty means add full header + value.Truncate(); // Empty means add full header else UnescapeCString(value); mEnabledHeadersDelimiters.AppendElement(value); @@ -337,96 +312,77 @@ Tokenizer::Tokenizer() : } } -Tokenizer::~Tokenizer() -{ -} +Tokenizer::~Tokenizer() {} -inline Token* Tokenizer::get(const char* word) -{ +inline Token* Tokenizer::get(const char* word) { return static_cast(TokenHash::get(word)); } -Token* Tokenizer::add(const char* word, uint32_t count) -{ - MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, ("add word: %s (count=%d)", - word, count)); +Token* Tokenizer::add(const char* word, uint32_t count) { + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, + ("add word: %s (count=%d)", word, count)); Token* token = static_cast(TokenHash::add(word)); - if (token) - { - token->mCount += count; // hash code initializes this to zero + if (token) { + token->mCount += count; // hash code initializes this to zero MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, - ("adding word to tokenizer: %s (count=%d) (mCount=%d)", - word, count, token->mCount)); + ("adding word to tokenizer: %s (count=%d) (mCount=%d)", word, count, + token->mCount)); } return token; } -static bool isDecimalNumber(const char* word) -{ - const char* p = word; - if (*p == '-') ++p; - char c; - while ((c = *p++)) { - if (!isdigit((unsigned char) c)) - return false; - } - return true; +static bool isDecimalNumber(const char* word) { + const char* p = word; + if (*p == '-') ++p; + char c; + while ((c = *p++)) { + if (!isdigit((unsigned char)c)) return false; + } + return true; } -static bool isASCII(const char* word) -{ - const unsigned char* p = (const unsigned char*)word; - unsigned char c; - while ((c = *p++)) { - if (c > 127) - return false; - } - return true; +static bool isASCII(const char* word) { + const unsigned char* p = (const unsigned char*)word; + unsigned char c; + while ((c = *p++)) { + if (c > 127) return false; + } + return true; } inline bool isUpperCase(char c) { return ('A' <= c) && (c <= 'Z'); } -static char* toLowerCase(char* str) -{ - char c, *p = str; - while ((c = *p++)) { - if (isUpperCase(c)) - p[-1] = c + ('a' - 'A'); - } - return str; +static char* toLowerCase(char* str) { + char c, *p = str; + while ((c = *p++)) { + if (isUpperCase(c)) p[-1] = c + ('a' - 'A'); + } + return str; } -void Tokenizer::addTokenForHeader(const char * aTokenPrefix, nsACString& aValue, - bool aTokenizeValue, const char* aDelimiters) -{ - if (aValue.Length()) - { +void Tokenizer::addTokenForHeader(const char* aTokenPrefix, nsACString& aValue, + bool aTokenizeValue, + const char* aDelimiters) { + if (aValue.Length()) { ToLowerCase(aValue); - if (!aTokenizeValue) - { + if (!aTokenizeValue) { nsCString tmpStr; tmpStr.Assign(aTokenPrefix); tmpStr.Append(':'); tmpStr.Append(aValue); add(tmpStr.get()); - } - else - { + } else { char* word; nsCString str(aValue); - char *next = str.BeginWriting(); - const char* delimiters = !aDelimiters ? - mHeaderDelimiters.get() : aDelimiters; - while ((word = NS_strtok(delimiters, &next)) != NULL) - { - if (strlen(word) < kMinLengthForToken) - continue; - if (isDecimalNumber(word)) - continue; - if (isASCII(word)) - { + char* next = str.BeginWriting(); + const char* delimiters = + !aDelimiters ? mHeaderDelimiters.get() : aDelimiters; + while ((word = NS_strtok(delimiters, &next)) != NULL) { + if (strlen(word) < kMinLengthForToken) continue; + if (isDecimalNumber(word)) continue; + if (isASCII(word)) { nsCString tmpStr; tmpStr.Assign(aTokenPrefix); tmpStr.Append(':'); @@ -438,8 +394,8 @@ void Tokenizer::addTokenForHeader(const char * aTokenPrefix, nsACString& aValue, } } -void Tokenizer::tokenizeAttachment(const char * aContentType, const char * aFileName) -{ +void Tokenizer::tokenizeAttachment(const char* aContentType, + const char* aFileName) { nsAutoCString contentType; nsAutoCString fileName; fileName.Assign(aFileName); @@ -453,25 +409,23 @@ void Tokenizer::tokenizeAttachment(const char * aContentType, const char * aFile addTokenForHeader("attachment/content-type", contentType); } -void Tokenizer::tokenizeHeaders(nsIUTF8StringEnumerator * aHeaderNames, nsIUTF8StringEnumerator * aHeaderValues) -{ +void Tokenizer::tokenizeHeaders(nsIUTF8StringEnumerator* aHeaderNames, + nsIUTF8StringEnumerator* aHeaderValues) { nsCString headerValue; - nsAutoCString headerName; // we'll be normalizing all header names to lower case + nsAutoCString + headerName; // we'll be normalizing all header names to lower case bool hasMore; - while (NS_SUCCEEDED(aHeaderNames->HasMore(&hasMore)) && hasMore) - { + while (NS_SUCCEEDED(aHeaderNames->HasMore(&hasMore)) && hasMore) { aHeaderNames->GetNext(headerName); ToLowerCase(headerName); aHeaderValues->GetNext(headerValue); bool headerProcessed = false; - if (mCustomHeaderTokenization) - { + if (mCustomHeaderTokenization) { // Process any exceptions set from preferences for (uint32_t i = 0; i < mEnabledHeaders.Length(); i++) - if (headerName.Equals(mEnabledHeaders[i])) - { + if (headerName.Equals(mEnabledHeaders[i])) { if (mEnabledHeadersDelimiters[i].IsVoid()) // tokenize with standard delimiters for all headers addTokenForHeader(headerName.get(), headerValue, true); @@ -483,85 +437,86 @@ void Tokenizer::tokenizeHeaders(nsIUTF8StringEnumerator * aHeaderNames, nsIUTF8S addTokenForHeader(headerName.get(), headerValue, true, mEnabledHeadersDelimiters[i].get()); headerProcessed = true; - break; // we found the header, no need to look for more custom values + break; // we found the header, no need to look for more custom values } - for (uint32_t i = 0; i < mDisabledHeaders.Length(); i++) - { - if (headerName.Equals(mDisabledHeaders[i])) - { + for (uint32_t i = 0; i < mDisabledHeaders.Length(); i++) { + if (headerName.Equals(mDisabledHeaders[i])) { headerProcessed = true; break; } } - if (headerProcessed) - continue; + if (headerProcessed) continue; } - switch (headerName.First()) - { - case 'c': - if (headerName.EqualsLiteral("content-type")) - { + switch (headerName.First()) { + case 'c': + if (headerName.EqualsLiteral("content-type")) { nsresult rv; - nsCOMPtr mimehdrpar = do_GetService(NS_MIMEHEADERPARAM_CONTRACTID, &rv); - if (NS_FAILED(rv)) - break; + nsCOMPtr mimehdrpar = + do_GetService(NS_MIMEHEADERPARAM_CONTRACTID, &rv); + if (NS_FAILED(rv)) break; // extract the charset parameter nsCString parameterValue; - mimehdrpar->GetParameterInternal(headerValue.get(), "charset", nullptr, nullptr, getter_Copies(parameterValue)); + mimehdrpar->GetParameterInternal(headerValue.get(), "charset", + nullptr, nullptr, + getter_Copies(parameterValue)); addTokenForHeader("charset", parameterValue); // create a token containing just the content type - mimehdrpar->GetParameterInternal(headerValue.get(), "type", nullptr, nullptr, getter_Copies(parameterValue)); + mimehdrpar->GetParameterInternal(headerValue.get(), "type", nullptr, + nullptr, + getter_Copies(parameterValue)); if (!parameterValue.Length()) - mimehdrpar->GetParameterInternal(headerValue.get(), nullptr /* use first unnamed param */, nullptr, nullptr, getter_Copies(parameterValue)); + mimehdrpar->GetParameterInternal( + headerValue.get(), nullptr /* use first unnamed param */, + nullptr, nullptr, getter_Copies(parameterValue)); addTokenForHeader("content-type/type", parameterValue); - // XXX: should we add a token for the entire content-type header as well or just these parts we have extracted? + // XXX: should we add a token for the entire content-type header as + // well or just these parts we have extracted? } break; - case 'r': - if (headerName.EqualsLiteral("received")) - { - // look for the string "may be forged" in the received headers. sendmail sometimes adds this hint - // This does not compile on linux yet. Need to figure out why. Commenting out for now - // if (FindInReadable(FORGED_RECEIVED_HEADER_HINT, headerValue)) - // addTokenForHeader(headerName.get(), FORGED_RECEIVED_HEADER_HINT); - } + case 'r': + if (headerName.EqualsLiteral("received")) { + // look for the string "may be forged" in the received headers. + // sendmail sometimes adds this hint This does not compile on linux + // yet. Need to figure out why. Commenting out for now if + // (FindInReadable(FORGED_RECEIVED_HEADER_HINT, headerValue)) + // addTokenForHeader(headerName.get(), FORGED_RECEIVED_HEADER_HINT); + } - // leave out reply-to - break; - case 's': - if (headerName.EqualsLiteral("subject")) - { + // leave out reply-to + break; + case 's': + if (headerName.EqualsLiteral("subject")) { // we want to tokenize the subject addTokenForHeader(headerName.get(), headerValue, true); } // important: leave out sender field. Too strong of an indicator break; - case 'x': // (2) X-Mailer / user-agent works best if it is untokenized, just fold the case and any leading/trailing white space - // all headers beginning with x-mozilla are being changed by us, so ignore + case 'x': // (2) X-Mailer / user-agent works best if it is untokenized, + // just fold the case and any leading/trailing white space + // all headers beginning with x-mozilla are being changed by us, so + // ignore if (StringBeginsWith(headerName, NS_LITERAL_CSTRING("x-mozilla"))) break; // fall through MOZ_FALLTHROUGH; - case 'u': + case 'u': addTokenForHeader(headerName.get(), headerValue); break; - default: + default: addTokenForHeader(headerName.get(), headerValue); break; - } // end switch - + } // end switch } } -void Tokenizer::tokenize_ascii_word(char * aWord) -{ +void Tokenizer::tokenize_ascii_word(char* aWord) { // always deal with normalized lower case strings toLowerCase(aWord); uint32_t wordLength = strlen(aWord); @@ -569,27 +524,32 @@ void Tokenizer::tokenize_ascii_word(char * aWord) // if the wordLength is within our accepted token limit, then add it if (wordLength >= kMinLengthForToken && wordLength <= mMaxLengthForToken) add(aWord); - else if (wordLength > mMaxLengthForToken) - { + else if (wordLength > mMaxLengthForToken) { // don't skip over the word if it looks like an email address, // there is value in adding tokens for addresses - nsDependentCString word (aWord, wordLength); // CHEAP, no allocation occurs here... + nsDependentCString word(aWord, + wordLength); // CHEAP, no allocation occurs here... - // XXX: i think the 40 byte check is just for perf reasons...if the email address is longer than that then forget about it. - const char *atSign = strchr(aWord, '@'); - if (wordLength < 40 && strchr(aWord, '.') && atSign && !strchr(atSign + 1, '@')) - { + // XXX: i think the 40 byte check is just for perf reasons...if the email + // address is longer than that then forget about it. + const char* atSign = strchr(aWord, '@'); + if (wordLength < 40 && strchr(aWord, '.') && atSign && + !strchr(atSign + 1, '@')) { uint32_t numBytesToSep = atSign - aWord; - if (numBytesToSep < wordLength - 1) // if the @ sign is the last character, it must not be an email address + if (numBytesToSep < + wordLength - 1) // if the @ sign is the last character, it must not + // be an email address { - // split the john@foo.com into john and foo.com, treat them as separate tokens + // split the john@foo.com into john and foo.com, treat them as separate + // tokens nsCString emailNameToken; emailNameToken.AssignLiteral("email name:"); emailNameToken.Append(Substring(word, 0, numBytesToSep++)); add(emailNameToken.get()); nsCString emailAddrToken; emailAddrToken.AssignLiteral("email addr:"); - emailAddrToken.Append(Substring(word, numBytesToSep, wordLength - numBytesToSep)); + emailAddrToken.Append( + Substring(word, numBytesToSep, wordLength - numBytesToSep)); add(emailAddrToken.get()); return; } @@ -601,101 +561,97 @@ void Tokenizer::tokenize_ascii_word(char * aWord) skipToken.AssignLiteral("skip:"); skipToken.Append(word[0]); skipToken.Append(' '); - skipToken.AppendInt((wordLength/10) * 10); + skipToken.AppendInt((wordLength / 10) * 10); add(skipToken.get()); } } -// one subtract and one conditional jump should be faster than two conditional jump on most recent system. -#define IN_RANGE(x, low, high) ((uint16_t)((x)-(low)) <= (high)-(low)) +// one subtract and one conditional jump should be faster than two conditional +// jump on most recent system. +#define IN_RANGE(x, low, high) ((uint16_t)((x) - (low)) <= (high) - (low)) -#define IS_JA_HIRAGANA(x) IN_RANGE(x, 0x3040, 0x309F) +#define IS_JA_HIRAGANA(x) IN_RANGE(x, 0x3040, 0x309F) // swapping the range using xor operation to reduce conditional jump. -#define IS_JA_KATAKANA(x) (IN_RANGE(x^0x0004, 0x30A0, 0x30FE)||(IN_RANGE(x, 0xFF66, 0xFF9F))) -#define IS_JA_KANJI(x) (IN_RANGE(x, 0x2E80, 0x2FDF)||IN_RANGE(x, 0x4E00, 0x9FAF)) -#define IS_JA_KUTEN(x) (((x)==0x3001)||((x)==0xFF64)||((x)==0xFF0E)) -#define IS_JA_TOUTEN(x) (((x)==0x3002)||((x)==0xFF61)||((x)==0xFF0C)) -#define IS_JA_SPACE(x) ((x)==0x3000) -#define IS_JA_FWLATAIN(x) IN_RANGE(x, 0xFF01, 0xFF5E) -#define IS_JA_FWNUMERAL(x) IN_RANGE(x, 0xFF10, 0xFF19) +#define IS_JA_KATAKANA(x) \ + (IN_RANGE(x ^ 0x0004, 0x30A0, 0x30FE) || (IN_RANGE(x, 0xFF66, 0xFF9F))) +#define IS_JA_KANJI(x) \ + (IN_RANGE(x, 0x2E80, 0x2FDF) || IN_RANGE(x, 0x4E00, 0x9FAF)) +#define IS_JA_KUTEN(x) (((x) == 0x3001) || ((x) == 0xFF64) || ((x) == 0xFF0E)) +#define IS_JA_TOUTEN(x) (((x) == 0x3002) || ((x) == 0xFF61) || ((x) == 0xFF0C)) +#define IS_JA_SPACE(x) ((x) == 0x3000) +#define IS_JA_FWLATAIN(x) IN_RANGE(x, 0xFF01, 0xFF5E) +#define IS_JA_FWNUMERAL(x) IN_RANGE(x, 0xFF10, 0xFF19) -#define IS_JAPANESE_SPECIFIC(x) (IN_RANGE(x, 0x3040, 0x30FF)||IN_RANGE(x, 0xFF01, 0xFF9F)) +#define IS_JAPANESE_SPECIFIC(x) \ + (IN_RANGE(x, 0x3040, 0x30FF) || IN_RANGE(x, 0xFF01, 0xFF9F)) -enum char_class{ - others = 0, - space, - hiragana, - katakana, - kanji, - kuten, - touten, - kigou, - fwlatain, - ascii +enum char_class { + others = 0, + space, + hiragana, + katakana, + kanji, + kuten, + touten, + kigou, + fwlatain, + ascii }; -static char_class getCharClass(char16_t c) -{ +static char_class getCharClass(char16_t c) { char_class charClass = others; - if(IS_JA_HIRAGANA(c)) + if (IS_JA_HIRAGANA(c)) charClass = hiragana; - else if(IS_JA_KATAKANA(c)) + else if (IS_JA_KATAKANA(c)) charClass = katakana; - else if(IS_JA_KANJI(c)) + else if (IS_JA_KANJI(c)) charClass = kanji; - else if(IS_JA_KUTEN(c)) + else if (IS_JA_KUTEN(c)) charClass = kuten; - else if(IS_JA_TOUTEN(c)) + else if (IS_JA_TOUTEN(c)) charClass = touten; - else if(IS_JA_FWLATAIN(c)) + else if (IS_JA_FWLATAIN(c)) charClass = fwlatain; return charClass; } -static bool isJapanese(const char* word) -{ +static bool isJapanese(const char* word) { nsString text = NS_ConvertUTF8toUTF16(word); char16_t* p = (char16_t*)text.get(); char16_t c; // it is japanese chunk if it contains any hiragana or katakana. - while((c = *p++)) - if( IS_JAPANESE_SPECIFIC(c)) - return true; + while ((c = *p++)) + if (IS_JAPANESE_SPECIFIC(c)) return true; return false; } -static bool isFWNumeral(const char16_t* p1, const char16_t* p2) -{ - for(;p1 utils = - do_GetService(NS_PARSERUTILS_CONTRACTID); +nsresult Tokenizer::stripHTML(const nsAString& inString, nsAString& outString) { + uint32_t flags = nsIDocumentEncoder::OutputLFLineBreak | + nsIDocumentEncoder::OutputNoScriptContent | + nsIDocumentEncoder::OutputNoFramesContent | + nsIDocumentEncoder::OutputBodyOnly; + nsCOMPtr utils = do_GetService(NS_PARSERUTILS_CONTRACTID); return utils->ConvertToPlainText(inString, flags, 80, outString); } // Copied from nsSemanticUnitScanner.cpp which was removed in bug 1368418. -nsresult Tokenizer::ScannerNext(const char16_t *text, int32_t length, int32_t pos, bool isLastBuffer, int32_t *begin, int32_t *end, bool *_retval) -{ - if (!mWordBreaker) { - mWordBreaker = mozilla::intl::WordBreaker::Create(); - } +nsresult Tokenizer::ScannerNext(const char16_t* text, int32_t length, + int32_t pos, bool isLastBuffer, int32_t* begin, + int32_t* end, bool* _retval) { + if (!mWordBreaker) { + mWordBreaker = mozilla::intl::WordBreaker::Create(); + } - // if we reach the end, just return - if (pos >= length) { - *begin = pos; - *end = pos; - *_retval = false; - return NS_OK; - } - - mozilla::intl::WordBreakClass char_class = mozilla::intl::WordBreaker::GetClass(text[pos]); - - // If we are in Chinese mode, return one Han letter at a time. - // We should not do this if we are in Japanese or Korean mode. - if (mozilla::intl::kWbClassHanLetter == char_class) { - *begin = pos; - *end = pos+1; - *_retval = true; - return NS_OK; - } - - int32_t next; - // Find the next "word". - next = mWordBreaker->NextWord(text, (uint32_t) length, (uint32_t) pos); - - // If we don't have enough text to make decision, return. - if (next == NS_WORDBREAKER_NEED_MORE_TEXT) { - *begin = pos; - *end = isLastBuffer ? length : pos; - *_retval = isLastBuffer; - return NS_OK; - } - - // If what we got is space or punct, look at the next break. - if (char_class == mozilla::intl::kWbClassSpace || - char_class == mozilla::intl::kWbClassPunct) { - // If the next "word" is not letters, - // call itself recursively with the new pos. - return ScannerNext(text, length, next, isLastBuffer, begin, end, _retval); - } - - // For the rest, return. + // if we reach the end, just return + if (pos >= length) { *begin = pos; - *end = next; + *end = pos; + *_retval = false; + return NS_OK; + } + + mozilla::intl::WordBreakClass char_class = + mozilla::intl::WordBreaker::GetClass(text[pos]); + + // If we are in Chinese mode, return one Han letter at a time. + // We should not do this if we are in Japanese or Korean mode. + if (mozilla::intl::kWbClassHanLetter == char_class) { + *begin = pos; + *end = pos + 1; *_retval = true; return NS_OK; + } + + int32_t next; + // Find the next "word". + next = mWordBreaker->NextWord(text, (uint32_t)length, (uint32_t)pos); + + // If we don't have enough text to make decision, return. + if (next == NS_WORDBREAKER_NEED_MORE_TEXT) { + *begin = pos; + *end = isLastBuffer ? length : pos; + *_retval = isLastBuffer; + return NS_OK; + } + + // If what we got is space or punct, look at the next break. + if (char_class == mozilla::intl::kWbClassSpace || + char_class == mozilla::intl::kWbClassPunct) { + // If the next "word" is not letters, + // call itself recursively with the new pos. + return ScannerNext(text, length, next, isLastBuffer, begin, end, _retval); + } + + // For the rest, return. + *begin = pos; + *end = next; + *_retval = true; + return NS_OK; } -void Tokenizer::tokenize(const char* aText) -{ +void Tokenizer::tokenize(const char* aText) { MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, ("tokenize: %s", aText)); // strip out HTML tags before we begin processing @@ -788,8 +743,7 @@ void Tokenizer::tokenize(const char* aText) // text. We disable by default, since most people won't be applying bayes // to RSS - if (mIframeToDiv) - { + if (mIframeToDiv) { MsgReplaceSubstring(text, NS_LITERAL_STRING(""), @@ -799,17 +753,17 @@ void Tokenizer::tokenize(const char* aText) stripHTML(text, strippedUCS2); // convert 0x3000(full width space) into 0x0020 - char16_t * substr_start = strippedUCS2.BeginWriting(); - char16_t * substr_end = strippedUCS2.EndWriting(); + char16_t* substr_start = strippedUCS2.BeginWriting(); + char16_t* substr_end = strippedUCS2.EndWriting(); while (substr_start != substr_end) { - if (*substr_start == 0x3000) - *substr_start = 0x0020; + if (*substr_start == 0x3000) *substr_start = 0x0020; ++substr_start; } nsCString strippedStr = NS_ConvertUTF16toUTF8(strippedUCS2); - char * strippedText = strippedStr.BeginWriting(); - MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, ("tokenize stripped html: %s", strippedText)); + char* strippedText = strippedStr.BeginWriting(); + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, + ("tokenize stripped html: %s", strippedText)); char* word; char* next = strippedText; @@ -817,35 +771,34 @@ void Tokenizer::tokenize(const char* aText) if (!*word) continue; if (isDecimalNumber(word)) continue; if (isASCII(word)) - tokenize_ascii_word(word); + tokenize_ascii_word(word); else if (isJapanese(word)) - tokenize_japanese_word(word); + tokenize_japanese_word(word); else { - nsresult rv; - // Convert this word from UTF-8 into UCS2. - NS_ConvertUTF8toUTF16 uword(word); - ToLowerCase(uword); - const char16_t* utext = uword.get(); - int32_t len = uword.Length(), pos = 0, begin, end; - bool gotUnit; - while (pos < len) { - rv = ScannerNext(utext, len, pos, true, &begin, &end, &gotUnit); - if (NS_SUCCEEDED(rv) && gotUnit) { - NS_ConvertUTF16toUTF8 utfUnit(utext + begin, end - begin); - add(utfUnit.get()); - // Advance to end of current unit. - pos = end; - } else { - break; - } + nsresult rv; + // Convert this word from UTF-8 into UCS2. + NS_ConvertUTF8toUTF16 uword(word); + ToLowerCase(uword); + const char16_t* utext = uword.get(); + int32_t len = uword.Length(), pos = 0, begin, end; + bool gotUnit; + while (pos < len) { + rv = ScannerNext(utext, len, pos, true, &begin, &end, &gotUnit); + if (NS_SUCCEEDED(rv) && gotUnit) { + NS_ConvertUTF16toUTF8 utfUnit(utext + begin, end - begin); + add(utfUnit.get()); + // Advance to end of current unit. + pos = end; + } else { + break; } + } } } } // helper function to un-escape \n, \t, etc from a CString -void Tokenizer::UnescapeCString(nsCString& aCString) -{ +void Tokenizer::UnescapeCString(nsCString& aCString) { nsAutoCString result; const char* readEnd = aCString.EndReading(); @@ -854,20 +807,16 @@ void Tokenizer::UnescapeCString(nsCString& aCString) char* writeIter = writeStart; bool inEscape = false; - for (const char* readIter = aCString.BeginReading(); readIter != readEnd; readIter++) - { - if (!inEscape) - { + for (const char* readIter = aCString.BeginReading(); readIter != readEnd; + readIter++) { + if (!inEscape) { if (*readIter == '\\') inEscape = true; else *(writeIter++) = *readIter; - } - else - { + } else { inEscape = false; - switch (*readIter) - { + switch (*readIter) { case '\\': *(writeIter++) = '\\'; break; @@ -896,8 +845,7 @@ void Tokenizer::UnescapeCString(nsCString& aCString) aCString.Assign(result); } -Token* Tokenizer::copyTokens() -{ +Token* Tokenizer::copyTokens() { uint32_t count = countTokens(); if (count > 0) { Token* tokens = new Token[count]; @@ -916,20 +864,18 @@ Token* Tokenizer::copyTokens() } class TokenAnalyzer { -public: - virtual ~TokenAnalyzer() {} + public: + virtual ~TokenAnalyzer() {} - virtual void analyzeTokens(Tokenizer& tokenizer) = 0; - void setTokenListener(nsIStreamListener *aTokenListener) - { - mTokenListener = aTokenListener; - } + virtual void analyzeTokens(Tokenizer& tokenizer) = 0; + void setTokenListener(nsIStreamListener* aTokenListener) { + mTokenListener = aTokenListener; + } - void setSource(const char *sourceURI) {mTokenSource = sourceURI;} - - nsCOMPtr mTokenListener; - nsCString mTokenSource; + void setSource(const char* sourceURI) { mTokenSource = sourceURI; } + nsCOMPtr mTokenListener; + nsCString mTokenSource; }; /** @@ -939,455 +885,434 @@ public: * refinement. */ class TokenStreamListener : public nsIStreamListener, nsIMsgHeaderSink { -public: - NS_DECL_ISUPPORTS - NS_DECL_NSIREQUESTOBSERVER - NS_DECL_NSISTREAMLISTENER - NS_DECL_NSIMSGHEADERSINK + public: + NS_DECL_ISUPPORTS + NS_DECL_NSIREQUESTOBSERVER + NS_DECL_NSISTREAMLISTENER + NS_DECL_NSIMSGHEADERSINK - explicit TokenStreamListener(TokenAnalyzer* analyzer); -protected: - virtual ~TokenStreamListener(); - TokenAnalyzer* mAnalyzer; - char* mBuffer; - uint32_t mBufferSize; - uint32_t mLeftOverCount; - Tokenizer mTokenizer; - bool mSetAttachmentFlag; + explicit TokenStreamListener(TokenAnalyzer* analyzer); + + protected: + virtual ~TokenStreamListener(); + TokenAnalyzer* mAnalyzer; + char* mBuffer; + uint32_t mBufferSize; + uint32_t mLeftOverCount; + Tokenizer mTokenizer; + bool mSetAttachmentFlag; }; const uint32_t kBufferSize = 16384; TokenStreamListener::TokenStreamListener(TokenAnalyzer* analyzer) - : mAnalyzer(analyzer), - mBuffer(NULL), mBufferSize(kBufferSize), mLeftOverCount(0), - mSetAttachmentFlag(false) -{ + : mAnalyzer(analyzer), + mBuffer(NULL), + mBufferSize(kBufferSize), + mLeftOverCount(0), + mSetAttachmentFlag(false) {} + +TokenStreamListener::~TokenStreamListener() { + delete[] mBuffer; + delete mAnalyzer; } -TokenStreamListener::~TokenStreamListener() -{ - delete[] mBuffer; - delete mAnalyzer; -} +NS_IMPL_ISUPPORTS(TokenStreamListener, nsIRequestObserver, nsIStreamListener, + nsIMsgHeaderSink) -NS_IMPL_ISUPPORTS(TokenStreamListener, nsIRequestObserver, nsIStreamListener, nsIMsgHeaderSink) - -NS_IMETHODIMP TokenStreamListener::ProcessHeaders(nsIUTF8StringEnumerator *aHeaderNames, nsIUTF8StringEnumerator *aHeaderValues, bool dontCollectAddress) -{ - mTokenizer.tokenizeHeaders(aHeaderNames, aHeaderValues); - return NS_OK; -} - -NS_IMETHODIMP TokenStreamListener::HandleAttachment(const char *contentType, const char *url, const char16_t *displayName, const char *uri, bool aIsExternalAttachment) -{ - mTokenizer.tokenizeAttachment(contentType, NS_ConvertUTF16toUTF8(displayName).get()); - return NS_OK; -} - -NS_IMETHODIMP TokenStreamListener::AddAttachmentField(const char *field, const char *value) -{ - return NS_OK; -} - -NS_IMETHODIMP TokenStreamListener::OnEndAllAttachments() -{ - return NS_OK; -} - -NS_IMETHODIMP TokenStreamListener::OnEndMsgDownload(nsIMsgMailNewsUrl *url) -{ - return NS_OK; -} - - -NS_IMETHODIMP TokenStreamListener::OnMsgHasRemoteContent(nsIMsgDBHdr *aMsgHdr, - nsIURI *aContentURI, - bool aCanOverride) -{ - return NS_OK; -} - -NS_IMETHODIMP TokenStreamListener::OnEndMsgHeaders(nsIMsgMailNewsUrl *url) -{ - return NS_OK; -} - - -NS_IMETHODIMP TokenStreamListener::GetSecurityInfo(nsISupports * *aSecurityInfo) -{ - return NS_OK; -} -NS_IMETHODIMP TokenStreamListener::SetSecurityInfo(nsISupports * aSecurityInfo) -{ - return NS_OK; -} - -NS_IMETHODIMP TokenStreamListener::GetDummyMsgHeader(nsIMsgDBHdr **aMsgDBHdr) -{ - return NS_ERROR_NOT_IMPLEMENTED; -} - -NS_IMETHODIMP TokenStreamListener::ResetProperties() -{ +NS_IMETHODIMP TokenStreamListener::ProcessHeaders( + nsIUTF8StringEnumerator* aHeaderNames, + nsIUTF8StringEnumerator* aHeaderValues, bool dontCollectAddress) { + mTokenizer.tokenizeHeaders(aHeaderNames, aHeaderValues); return NS_OK; } -NS_IMETHODIMP TokenStreamListener::GetProperties(nsIWritablePropertyBag2 * *aProperties) -{ +NS_IMETHODIMP TokenStreamListener::HandleAttachment( + const char* contentType, const char* url, const char16_t* displayName, + const char* uri, bool aIsExternalAttachment) { + mTokenizer.tokenizeAttachment(contentType, + NS_ConvertUTF16toUTF8(displayName).get()); + return NS_OK; +} + +NS_IMETHODIMP TokenStreamListener::AddAttachmentField(const char* field, + const char* value) { + return NS_OK; +} + +NS_IMETHODIMP TokenStreamListener::OnEndAllAttachments() { return NS_OK; } + +NS_IMETHODIMP TokenStreamListener::OnEndMsgDownload(nsIMsgMailNewsUrl* url) { + return NS_OK; +} + +NS_IMETHODIMP TokenStreamListener::OnMsgHasRemoteContent(nsIMsgDBHdr* aMsgHdr, + nsIURI* aContentURI, + bool aCanOverride) { + return NS_OK; +} + +NS_IMETHODIMP TokenStreamListener::OnEndMsgHeaders(nsIMsgMailNewsUrl* url) { + return NS_OK; +} + +NS_IMETHODIMP TokenStreamListener::GetSecurityInfo( + nsISupports** aSecurityInfo) { + return NS_OK; +} +NS_IMETHODIMP TokenStreamListener::SetSecurityInfo(nsISupports* aSecurityInfo) { + return NS_OK; +} + +NS_IMETHODIMP TokenStreamListener::GetDummyMsgHeader(nsIMsgDBHdr** aMsgDBHdr) { + return NS_ERROR_NOT_IMPLEMENTED; +} + +NS_IMETHODIMP TokenStreamListener::ResetProperties() { return NS_OK; } + +NS_IMETHODIMP TokenStreamListener::GetProperties( + nsIWritablePropertyBag2** aProperties) { return NS_ERROR_NOT_IMPLEMENTED; } /* void onStartRequest (in nsIRequest aRequest); */ -NS_IMETHODIMP TokenStreamListener::OnStartRequest(nsIRequest *aRequest) -{ - mLeftOverCount = 0; - if (!mBuffer) - { - mBuffer = new char[mBufferSize]; - NS_ENSURE_TRUE(mBuffer, NS_ERROR_OUT_OF_MEMORY); - } +NS_IMETHODIMP TokenStreamListener::OnStartRequest(nsIRequest* aRequest) { + mLeftOverCount = 0; + if (!mBuffer) { + mBuffer = new char[mBufferSize]; + NS_ENSURE_TRUE(mBuffer, NS_ERROR_OUT_OF_MEMORY); + } - // get the url for the channel and set our nsIMsgHeaderSink on it so we get notified - // about the headers and attachments + // get the url for the channel and set our nsIMsgHeaderSink on it so we get + // notified about the headers and attachments - nsCOMPtr channel (do_QueryInterface(aRequest)); - if (channel) - { - nsCOMPtr uri; - channel->GetURI(getter_AddRefs(uri)); - nsCOMPtr mailUrl = do_QueryInterface(uri); - if (mailUrl) - mailUrl->SetMsgHeaderSink(static_cast(this)); - } + nsCOMPtr channel(do_QueryInterface(aRequest)); + if (channel) { + nsCOMPtr uri; + channel->GetURI(getter_AddRefs(uri)); + nsCOMPtr mailUrl = do_QueryInterface(uri); + if (mailUrl) + mailUrl->SetMsgHeaderSink(static_cast(this)); + } - return NS_OK; + return NS_OK; } -/* void onDataAvailable (in nsIRequest aRequest, in nsIInputStream aInputStream, in unsigned long long aOffset, in unsigned long aCount); */ -NS_IMETHODIMP TokenStreamListener::OnDataAvailable(nsIRequest *aRequest, nsIInputStream *aInputStream, uint64_t aOffset, uint32_t aCount) -{ - nsresult rv = NS_OK; +/* void onDataAvailable (in nsIRequest aRequest, in nsIInputStream aInputStream, + * in unsigned long long aOffset, in unsigned long aCount); */ +NS_IMETHODIMP TokenStreamListener::OnDataAvailable(nsIRequest* aRequest, + nsIInputStream* aInputStream, + uint64_t aOffset, + uint32_t aCount) { + nsresult rv = NS_OK; - while (aCount > 0) { - uint32_t readCount, totalCount = (aCount + mLeftOverCount); - if (totalCount >= mBufferSize) { - readCount = mBufferSize - mLeftOverCount - 1; - } else { - readCount = aCount; - } - - // mBuffer is supposed to be allocated in onStartRequest. But something - // is causing that to not happen, so as a last-ditch attempt we'll - // do it here. - if (!mBuffer) - { - mBuffer = new char[mBufferSize]; - NS_ENSURE_TRUE(mBuffer, NS_ERROR_OUT_OF_MEMORY); - } - - char* buffer = mBuffer; - rv = aInputStream->Read(buffer + mLeftOverCount, readCount, &readCount); - if (NS_FAILED(rv)) - break; - - if (readCount == 0) { - rv = NS_ERROR_UNEXPECTED; - NS_WARNING("failed to tokenize"); - break; - } - - aCount -= readCount; - - /* consume the tokens up to the last legal token delimiter in the buffer. */ - totalCount = (readCount + mLeftOverCount); - buffer[totalCount] = '\0'; - char* lastDelimiter = NULL; - char* scan = buffer + totalCount; - while (scan > buffer) { - if (strchr(mTokenizer.mBodyDelimiters.get(), *--scan)) { - lastDelimiter = scan; - break; - } - } - - if (lastDelimiter) { - *lastDelimiter = '\0'; - mTokenizer.tokenize(buffer); - - uint32_t consumedCount = 1 + (lastDelimiter - buffer); - mLeftOverCount = totalCount - consumedCount; - if (mLeftOverCount) - memmove(buffer, buffer + consumedCount, mLeftOverCount); - } else { - /* didn't find a delimiter, keep the whole buffer around. */ - mLeftOverCount = totalCount; - if (totalCount >= (mBufferSize / 2)) { - uint32_t newBufferSize = mBufferSize * 2; - char* newBuffer = new char[newBufferSize]; - NS_ENSURE_TRUE(newBuffer, NS_ERROR_OUT_OF_MEMORY); - memcpy(newBuffer, mBuffer, mLeftOverCount); - delete[] mBuffer; - mBuffer = newBuffer; - mBufferSize = newBufferSize; - } - } + while (aCount > 0) { + uint32_t readCount, totalCount = (aCount + mLeftOverCount); + if (totalCount >= mBufferSize) { + readCount = mBufferSize - mLeftOverCount - 1; + } else { + readCount = aCount; } - return rv; + // mBuffer is supposed to be allocated in onStartRequest. But something + // is causing that to not happen, so as a last-ditch attempt we'll + // do it here. + if (!mBuffer) { + mBuffer = new char[mBufferSize]; + NS_ENSURE_TRUE(mBuffer, NS_ERROR_OUT_OF_MEMORY); + } + + char* buffer = mBuffer; + rv = aInputStream->Read(buffer + mLeftOverCount, readCount, &readCount); + if (NS_FAILED(rv)) break; + + if (readCount == 0) { + rv = NS_ERROR_UNEXPECTED; + NS_WARNING("failed to tokenize"); + break; + } + + aCount -= readCount; + + /* consume the tokens up to the last legal token delimiter in the buffer. */ + totalCount = (readCount + mLeftOverCount); + buffer[totalCount] = '\0'; + char* lastDelimiter = NULL; + char* scan = buffer + totalCount; + while (scan > buffer) { + if (strchr(mTokenizer.mBodyDelimiters.get(), *--scan)) { + lastDelimiter = scan; + break; + } + } + + if (lastDelimiter) { + *lastDelimiter = '\0'; + mTokenizer.tokenize(buffer); + + uint32_t consumedCount = 1 + (lastDelimiter - buffer); + mLeftOverCount = totalCount - consumedCount; + if (mLeftOverCount) + memmove(buffer, buffer + consumedCount, mLeftOverCount); + } else { + /* didn't find a delimiter, keep the whole buffer around. */ + mLeftOverCount = totalCount; + if (totalCount >= (mBufferSize / 2)) { + uint32_t newBufferSize = mBufferSize * 2; + char* newBuffer = new char[newBufferSize]; + NS_ENSURE_TRUE(newBuffer, NS_ERROR_OUT_OF_MEMORY); + memcpy(newBuffer, mBuffer, mLeftOverCount); + delete[] mBuffer; + mBuffer = newBuffer; + mBufferSize = newBufferSize; + } + } + } + + return rv; } /* void onStopRequest (in nsIRequest aRequest, in nsresult aStatusCode); */ -NS_IMETHODIMP TokenStreamListener::OnStopRequest(nsIRequest *aRequest, nsresult aStatusCode) -{ - if (mLeftOverCount) { - /* assume final buffer is complete. */ - mBuffer[mLeftOverCount] = '\0'; - mTokenizer.tokenize(mBuffer); - } +NS_IMETHODIMP TokenStreamListener::OnStopRequest(nsIRequest* aRequest, + nsresult aStatusCode) { + if (mLeftOverCount) { + /* assume final buffer is complete. */ + mBuffer[mLeftOverCount] = '\0'; + mTokenizer.tokenize(mBuffer); + } - /* finally, analyze the tokenized message. */ - MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, ("analyze the tokenized message")); - if (mAnalyzer) - mAnalyzer->analyzeTokens(mTokenizer); + /* finally, analyze the tokenized message. */ + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, + ("analyze the tokenized message")); + if (mAnalyzer) mAnalyzer->analyzeTokens(mTokenizer); - return NS_OK; + return NS_OK; } /* Implementation file */ -NS_IMPL_ISUPPORTS(nsBayesianFilter, nsIMsgFilterPlugin, - nsIJunkMailPlugin, nsIMsgCorpus, nsISupportsWeakReference, - nsIObserver) +NS_IMPL_ISUPPORTS(nsBayesianFilter, nsIMsgFilterPlugin, nsIJunkMailPlugin, + nsIMsgCorpus, nsISupportsWeakReference, nsIObserver) -nsBayesianFilter::nsBayesianFilter() - : mTrainingDataDirty(false) -{ - int32_t junkThreshold = 0; - nsresult rv; - nsCOMPtr pPrefBranch(do_GetService(NS_PREFSERVICE_CONTRACTID, &rv)); - if (pPrefBranch) - pPrefBranch->GetIntPref("mail.adaptivefilters.junk_threshold", &junkThreshold); +nsBayesianFilter::nsBayesianFilter() : mTrainingDataDirty(false) { + int32_t junkThreshold = 0; + nsresult rv; + nsCOMPtr pPrefBranch( + do_GetService(NS_PREFSERVICE_CONTRACTID, &rv)); + if (pPrefBranch) + pPrefBranch->GetIntPref("mail.adaptivefilters.junk_threshold", + &junkThreshold); - mJunkProbabilityThreshold = (static_cast(junkThreshold)) / 100.0; - if (mJunkProbabilityThreshold == 0 || mJunkProbabilityThreshold >= 1) - mJunkProbabilityThreshold = kDefaultJunkThreshold; + mJunkProbabilityThreshold = (static_cast(junkThreshold)) / 100.0; + if (mJunkProbabilityThreshold == 0 || mJunkProbabilityThreshold >= 1) + mJunkProbabilityThreshold = kDefaultJunkThreshold; - MOZ_LOG(BayesianFilterLogModule, LogLevel::Warning, ("junk probability threshold: %f", mJunkProbabilityThreshold)); + MOZ_LOG(BayesianFilterLogModule, LogLevel::Warning, + ("junk probability threshold: %f", mJunkProbabilityThreshold)); - mCorpus.readTrainingData(); + mCorpus.readTrainingData(); - // get parameters for training data flushing, from the prefs + // get parameters for training data flushing, from the prefs - nsCOMPtr prefBranch; + nsCOMPtr prefBranch; - nsCOMPtr prefs = do_GetService(NS_PREFSERVICE_CONTRACTID, &rv); - NS_ASSERTION(NS_SUCCEEDED(rv),"failed accessing preferences service"); - rv = prefs->GetBranch(nullptr, getter_AddRefs(prefBranch)); - NS_ASSERTION(NS_SUCCEEDED(rv),"failed getting preferences branch"); + nsCOMPtr prefs = + do_GetService(NS_PREFSERVICE_CONTRACTID, &rv); + NS_ASSERTION(NS_SUCCEEDED(rv), "failed accessing preferences service"); + rv = prefs->GetBranch(nullptr, getter_AddRefs(prefBranch)); + NS_ASSERTION(NS_SUCCEEDED(rv), "failed getting preferences branch"); - rv = prefBranch->GetIntPref("mailnews.bayesian_spam_filter.flush.minimum_interval",&mMinFlushInterval); - // it is not a good idea to allow a minimum interval of under 1 second - if (NS_FAILED(rv) || (mMinFlushInterval <= 1000) ) - mMinFlushInterval = DEFAULT_MIN_INTERVAL_BETWEEN_WRITES; + rv = prefBranch->GetIntPref( + "mailnews.bayesian_spam_filter.flush.minimum_interval", + &mMinFlushInterval); + // it is not a good idea to allow a minimum interval of under 1 second + if (NS_FAILED(rv) || (mMinFlushInterval <= 1000)) + mMinFlushInterval = DEFAULT_MIN_INTERVAL_BETWEEN_WRITES; - rv = prefBranch->GetIntPref("mailnews.bayesian_spam_filter.junk_maxtokens", &mMaximumTokenCount); - if (NS_FAILED(rv)) - mMaximumTokenCount = 0; // which means do not limit token counts - MOZ_LOG(BayesianFilterLogModule, LogLevel::Warning, ("maximum junk tokens: %d", mMaximumTokenCount)); + rv = prefBranch->GetIntPref("mailnews.bayesian_spam_filter.junk_maxtokens", + &mMaximumTokenCount); + if (NS_FAILED(rv)) + mMaximumTokenCount = 0; // which means do not limit token counts + MOZ_LOG(BayesianFilterLogModule, LogLevel::Warning, + ("maximum junk tokens: %d", mMaximumTokenCount)); - mTimer = do_CreateInstance(NS_TIMER_CONTRACTID, &rv); - NS_ASSERTION(NS_SUCCEEDED(rv), "unable to create a timer; training data will only be written on exit"); + mTimer = do_CreateInstance(NS_TIMER_CONTRACTID, &rv); + NS_ASSERTION( + NS_SUCCEEDED(rv), + "unable to create a timer; training data will only be written on exit"); - // the timer is not used on object construction, since for - // the time being there are no dirying messages + // the timer is not used on object construction, since for + // the time being there are no dirying messages - // give a default capacity to the memory structure used to store - // per-message/per-trait token data - mAnalysisStore.SetCapacity(kAnalysisStoreCapacity); + // give a default capacity to the memory structure used to store + // per-message/per-trait token data + mAnalysisStore.SetCapacity(kAnalysisStoreCapacity); - // dummy 0th element. Index 0 means "end of list" so we need to - // start from 1 - AnalysisPerToken analysisPT(0, 0.0, 0.0); - mAnalysisStore.AppendElement(analysisPT); - mNextAnalysisIndex = 1; + // dummy 0th element. Index 0 means "end of list" so we need to + // start from 1 + AnalysisPerToken analysisPT(0, 0.0, 0.0); + mAnalysisStore.AppendElement(analysisPT); + mNextAnalysisIndex = 1; } -nsresult nsBayesianFilter::Init() -{ +nsresult nsBayesianFilter::Init() { nsCOMPtr observerService = - mozilla::services::GetObserverService(); + mozilla::services::GetObserverService(); if (observerService) observerService->AddObserver(this, "profile-before-change", true); return NS_OK; } -void -nsBayesianFilter::TimerCallback(nsITimer* aTimer, void* aClosure) -{ - // we will flush the training data to disk after enough time has passed - // since the first time a message has been classified after the last flush +void nsBayesianFilter::TimerCallback(nsITimer* aTimer, void* aClosure) { + // we will flush the training data to disk after enough time has passed + // since the first time a message has been classified after the last flush - nsBayesianFilter *filter = static_cast(aClosure); - filter->mCorpus.writeTrainingData(filter->mMaximumTokenCount); - filter->mTrainingDataDirty = false; + nsBayesianFilter* filter = static_cast(aClosure); + filter->mCorpus.writeTrainingData(filter->mMaximumTokenCount); + filter->mTrainingDataDirty = false; } -nsBayesianFilter::~nsBayesianFilter() -{ - if (mTimer) - { - mTimer->Cancel(); - mTimer = nullptr; - } - // call shutdown when we are going away in case we need - // to flush the training set to disk - Shutdown(); +nsBayesianFilter::~nsBayesianFilter() { + if (mTimer) { + mTimer->Cancel(); + mTimer = nullptr; + } + // call shutdown when we are going away in case we need + // to flush the training set to disk + Shutdown(); } // this object is used for one call to classifyMessage or classifyMessages(). -// So if we're classifying multiple messages, this object will be used for each message. -// It's going to hold a reference to itself, basically, to stay in memory. +// So if we're classifying multiple messages, this object will be used for each +// message. It's going to hold a reference to itself, basically, to stay in +// memory. class MessageClassifier : public TokenAnalyzer { -public: - // full classifier with arbitrary traits - MessageClassifier(nsBayesianFilter* aFilter, - nsIJunkMailClassificationListener* aJunkListener, - nsIMsgTraitClassificationListener* aTraitListener, - nsIMsgTraitDetailListener* aDetailListener, - nsTArray& aProTraits, - nsTArray& aAntiTraits, - nsIMsgWindow *aMsgWindow, - uint32_t aNumMessagesToClassify, - const char **aMessageURIs) - : mFilter(aFilter), + public: + // full classifier with arbitrary traits + MessageClassifier(nsBayesianFilter* aFilter, + nsIJunkMailClassificationListener* aJunkListener, + nsIMsgTraitClassificationListener* aTraitListener, + nsIMsgTraitDetailListener* aDetailListener, + nsTArray& aProTraits, + nsTArray& aAntiTraits, nsIMsgWindow* aMsgWindow, + uint32_t aNumMessagesToClassify, const char** aMessageURIs) + : mFilter(aFilter), mJunkMailPlugin(aFilter), mJunkListener(aJunkListener), mTraitListener(aTraitListener), mDetailListener(aDetailListener), mProTraits(aProTraits), mAntiTraits(aAntiTraits), - mMsgWindow(aMsgWindow) - { - mCurMessageToClassify = 0; - mNumMessagesToClassify = aNumMessagesToClassify; - mMessageURIs = (char **) moz_xmalloc(sizeof(char *) * aNumMessagesToClassify); - for (uint32_t i = 0; i < aNumMessagesToClassify; i++) - mMessageURIs[i] = PL_strdup(aMessageURIs[i]); + mMsgWindow(aMsgWindow) { + mCurMessageToClassify = 0; + mNumMessagesToClassify = aNumMessagesToClassify; + mMessageURIs = (char**)moz_xmalloc(sizeof(char*) * aNumMessagesToClassify); + for (uint32_t i = 0; i < aNumMessagesToClassify; i++) + mMessageURIs[i] = PL_strdup(aMessageURIs[i]); + } - } - - // junk-only classifier - MessageClassifier(nsBayesianFilter* aFilter, - nsIJunkMailClassificationListener* aJunkListener, - nsIMsgWindow *aMsgWindow, - uint32_t aNumMessagesToClassify, - const char **aMessageURIs) - : mFilter(aFilter), + // junk-only classifier + MessageClassifier(nsBayesianFilter* aFilter, + nsIJunkMailClassificationListener* aJunkListener, + nsIMsgWindow* aMsgWindow, uint32_t aNumMessagesToClassify, + const char** aMessageURIs) + : mFilter(aFilter), mJunkMailPlugin(aFilter), mJunkListener(aJunkListener), mTraitListener(nullptr), mDetailListener(nullptr), - mMsgWindow(aMsgWindow) - { - mCurMessageToClassify = 0; - mNumMessagesToClassify = aNumMessagesToClassify; - mMessageURIs = (char **) moz_xmalloc(sizeof(char *) * aNumMessagesToClassify); - for (uint32_t i = 0; i < aNumMessagesToClassify; i++) - mMessageURIs[i] = PL_strdup(aMessageURIs[i]); - mProTraits.AppendElement(kJunkTrait); - mAntiTraits.AppendElement(kGoodTrait); + mMsgWindow(aMsgWindow) { + mCurMessageToClassify = 0; + mNumMessagesToClassify = aNumMessagesToClassify; + mMessageURIs = (char**)moz_xmalloc(sizeof(char*) * aNumMessagesToClassify); + for (uint32_t i = 0; i < aNumMessagesToClassify; i++) + mMessageURIs[i] = PL_strdup(aMessageURIs[i]); + mProTraits.AppendElement(kJunkTrait); + mAntiTraits.AppendElement(kGoodTrait); + } + virtual ~MessageClassifier() { + if (mMessageURIs) { + NS_FREE_XPCOM_ALLOCATED_POINTER_ARRAY(mNumMessagesToClassify, + mMessageURIs); } + } + virtual void analyzeTokens(Tokenizer& tokenizer) { + mFilter->classifyMessage(tokenizer, mTokenSource.get(), mProTraits, + mAntiTraits, mJunkListener, mTraitListener, + mDetailListener); + tokenizer.clearTokens(); + classifyNextMessage(); + } - virtual ~MessageClassifier() - { - if (mMessageURIs) - { - NS_FREE_XPCOM_ALLOCATED_POINTER_ARRAY(mNumMessagesToClassify, mMessageURIs); - } - } - virtual void analyzeTokens(Tokenizer& tokenizer) - { - mFilter->classifyMessage(tokenizer, - mTokenSource.get(), - mProTraits, - mAntiTraits, - mJunkListener, - mTraitListener, - mDetailListener); - tokenizer.clearTokens(); - classifyNextMessage(); + virtual void classifyNextMessage() { + if (++mCurMessageToClassify < mNumMessagesToClassify && + mMessageURIs[mCurMessageToClassify]) { + MOZ_LOG(BayesianFilterLogModule, LogLevel::Warning, + ("classifyNextMessage(%s)", mMessageURIs[mCurMessageToClassify])); + mFilter->tokenizeMessage(mMessageURIs[mCurMessageToClassify], mMsgWindow, + this); + } else { + // call all listeners with null parameters to signify end of batch + if (mJunkListener) + mJunkListener->OnMessageClassified(nullptr, + nsIJunkMailPlugin::UNCLASSIFIED, 0); + if (mTraitListener) + mTraitListener->OnMessageTraitsClassified(nullptr, 0, nullptr, nullptr); + mTokenListener = + nullptr; // this breaks the circular ref that keeps this object alive + // so we will be destroyed as a result. } + } - virtual void classifyNextMessage() - { - - if (++mCurMessageToClassify < mNumMessagesToClassify && mMessageURIs[mCurMessageToClassify]) { - MOZ_LOG(BayesianFilterLogModule, LogLevel::Warning, ("classifyNextMessage(%s)", mMessageURIs[mCurMessageToClassify])); - mFilter->tokenizeMessage(mMessageURIs[mCurMessageToClassify], mMsgWindow, this); - } - else - { - // call all listeners with null parameters to signify end of batch - if (mJunkListener) - mJunkListener->OnMessageClassified(nullptr, nsIJunkMailPlugin::UNCLASSIFIED, 0); - if (mTraitListener) - mTraitListener->OnMessageTraitsClassified(nullptr, 0, nullptr, nullptr); - mTokenListener = nullptr; // this breaks the circular ref that keeps this object alive - // so we will be destroyed as a result. - } - } - -private: - nsBayesianFilter* mFilter; - nsCOMPtr mJunkMailPlugin; - nsCOMPtr mJunkListener; - nsCOMPtr mTraitListener; - nsCOMPtr mDetailListener; - nsTArray mProTraits; - nsTArray mAntiTraits; - nsCOMPtr mMsgWindow; - int32_t mNumMessagesToClassify; - int32_t mCurMessageToClassify; // 0-based index - char **mMessageURIs; + private: + nsBayesianFilter* mFilter; + nsCOMPtr mJunkMailPlugin; + nsCOMPtr mJunkListener; + nsCOMPtr mTraitListener; + nsCOMPtr mDetailListener; + nsTArray mProTraits; + nsTArray mAntiTraits; + nsCOMPtr mMsgWindow; + int32_t mNumMessagesToClassify; + int32_t mCurMessageToClassify; // 0-based index + char** mMessageURIs; }; -nsresult nsBayesianFilter::tokenizeMessage(const char* aMessageURI, nsIMsgWindow *aMsgWindow, TokenAnalyzer* aAnalyzer) -{ - NS_ENSURE_ARG_POINTER(aMessageURI); +nsresult nsBayesianFilter::tokenizeMessage(const char* aMessageURI, + nsIMsgWindow* aMsgWindow, + TokenAnalyzer* aAnalyzer) { + NS_ENSURE_ARG_POINTER(aMessageURI); - nsCOMPtr msgService; - nsresult rv = GetMessageServiceFromURI(nsDependentCString(aMessageURI), getter_AddRefs(msgService)); - NS_ENSURE_SUCCESS(rv, rv); + nsCOMPtr msgService; + nsresult rv = GetMessageServiceFromURI(nsDependentCString(aMessageURI), + getter_AddRefs(msgService)); + NS_ENSURE_SUCCESS(rv, rv); - aAnalyzer->setSource(aMessageURI); - nsCOMPtr dummyNull; - return msgService->StreamMessage(aMessageURI, aAnalyzer->mTokenListener, - aMsgWindow, nullptr, true /* convert data */, - NS_LITERAL_CSTRING("filter"), false, getter_AddRefs(dummyNull)); + aAnalyzer->setSource(aMessageURI); + nsCOMPtr dummyNull; + return msgService->StreamMessage(aMessageURI, aAnalyzer->mTokenListener, + aMsgWindow, nullptr, true /* convert data */, + NS_LITERAL_CSTRING("filter"), false, + getter_AddRefs(dummyNull)); } // a TraitAnalysis is the per-token representation of the statistical // calculations, basically created to group information that is then // sorted by mDistance -struct TraitAnalysis -{ +struct TraitAnalysis { uint32_t mTokenIndex; double mDistance; double mProbability; }; // comparator required to sort an nsTArray -class compareTraitAnalysis -{ -public: - bool Equals(const TraitAnalysis& a, const TraitAnalysis& b) const - { +class compareTraitAnalysis { + public: + bool Equals(const TraitAnalysis& a, const TraitAnalysis& b) const { return a.mDistance == b.mDistance; } - bool LessThan(const TraitAnalysis& a, const TraitAnalysis& b) const - { + bool LessThan(const TraitAnalysis& a, const TraitAnalysis& b) const { return a.mDistance < b.mDistance; } }; @@ -1407,443 +1332,412 @@ inline double dmin(double x, double y) { return (x < y ? x : y); } // Note that converting unsigned ints to floating point can be slow on // some platforms (like Intel) so use signed quantities for the numeric // routines. -static inline double chi2P (double chi2, double nu, int32_t *error) -{ - // domain checks; set error and return a dummy value - if (chi2 < 0.0 || nu <= 0.0) - { - *error = -1; - return 0.0; - } - // reversing the arguments is intentional - return nsIncompleteGammaP (nu/2.0, chi2/2.0, error); +static inline double chi2P(double chi2, double nu, int32_t* error) { + // domain checks; set error and return a dummy value + if (chi2 < 0.0 || nu <= 0.0) { + *error = -1; + return 0.0; + } + // reversing the arguments is intentional + return nsIncompleteGammaP(nu / 2.0, chi2 / 2.0, error); } void nsBayesianFilter::classifyMessage( - Tokenizer& tokenizer, - const char* messageURI, - nsTArray& aProTraits, - nsTArray& aAntiTraits, - nsIJunkMailClassificationListener* listener, - nsIMsgTraitClassificationListener* aTraitListener, - nsIMsgTraitDetailListener* aDetailListener) -{ - Token* tokens = tokenizer.copyTokens(); - uint32_t tokenCount; - if (!tokens) - { - // This can happen with problems with UTF conversion - NS_ERROR("Trying to classify a null or invalid message"); - tokenCount = 0; - // don't return so that we still call the listeners - } - else - { - tokenCount = tokenizer.countTokens(); - } + Tokenizer& tokenizer, const char* messageURI, + nsTArray& aProTraits, nsTArray& aAntiTraits, + nsIJunkMailClassificationListener* listener, + nsIMsgTraitClassificationListener* aTraitListener, + nsIMsgTraitDetailListener* aDetailListener) { + Token* tokens = tokenizer.copyTokens(); + uint32_t tokenCount; + if (!tokens) { + // This can happen with problems with UTF conversion + NS_ERROR("Trying to classify a null or invalid message"); + tokenCount = 0; + // don't return so that we still call the listeners + } else { + tokenCount = tokenizer.countTokens(); + } - if (aProTraits.Length() != aAntiTraits.Length()) - { - NS_ERROR("Each Pro trait needs a matching Anti trait"); - return; - } + if (aProTraits.Length() != aAntiTraits.Length()) { + NS_ERROR("Each Pro trait needs a matching Anti trait"); + return; + } - /* this part is similar to the Graham algorithm with some adjustments. */ - uint32_t traitCount = aProTraits.Length(); + /* this part is similar to the Graham algorithm with some adjustments. */ + uint32_t traitCount = aProTraits.Length(); - // pro message counts per trait index - AutoTArray numProMessages; - // anti message counts per trait index - AutoTArray numAntiMessages; - // array of pro aliases per trait index - AutoTArray proAliasArrays; - // number of pro aliases per trait index - AutoTArray proAliasesLengths; - // array of anti aliases per trait index - AutoTArray antiAliasArrays; - // number of anti aliases per trait index - AutoTArray antiAliasesLengths; - // construct the outgoing listener arrays - AutoTArray traits; - AutoTArray percents; - if (traitCount > kTraitAutoCapacity) - { - traits.SetCapacity(traitCount); - percents.SetCapacity(traitCount); - numProMessages.SetCapacity(traitCount); - numAntiMessages.SetCapacity(traitCount); - proAliasesLengths.SetCapacity(traitCount); - antiAliasesLengths.SetCapacity(traitCount); - proAliasArrays.SetCapacity(traitCount); - antiAliasArrays.SetCapacity(traitCount); - } + // pro message counts per trait index + AutoTArray numProMessages; + // anti message counts per trait index + AutoTArray numAntiMessages; + // array of pro aliases per trait index + AutoTArray proAliasArrays; + // number of pro aliases per trait index + AutoTArray proAliasesLengths; + // array of anti aliases per trait index + AutoTArray antiAliasArrays; + // number of anti aliases per trait index + AutoTArray antiAliasesLengths; + // construct the outgoing listener arrays + AutoTArray traits; + AutoTArray percents; + if (traitCount > kTraitAutoCapacity) { + traits.SetCapacity(traitCount); + percents.SetCapacity(traitCount); + numProMessages.SetCapacity(traitCount); + numAntiMessages.SetCapacity(traitCount); + proAliasesLengths.SetCapacity(traitCount); + antiAliasesLengths.SetCapacity(traitCount); + proAliasArrays.SetCapacity(traitCount); + antiAliasArrays.SetCapacity(traitCount); + } + nsresult rv; + nsCOMPtr traitService( + do_GetService("@mozilla.org/msg-trait-service;1", &rv)); + if (NS_FAILED(rv)) { + NS_ERROR("Failed to get trait service"); + MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, + ("Failed to get trait service")); + } + + // get aliases and message counts for the pro and anti traits + for (uint32_t traitIndex = 0; traitIndex < traitCount; traitIndex++) { nsresult rv; - nsCOMPtr traitService(do_GetService("@mozilla.org/msg-trait-service;1", &rv)); - if (NS_FAILED(rv)) - { - NS_ERROR("Failed to get trait service"); - MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, ("Failed to get trait service")); - } - // get aliases and message counts for the pro and anti traits - for (uint32_t traitIndex = 0; traitIndex < traitCount; traitIndex++) - { - nsresult rv; - - // pro trait - uint32_t proAliasesLength = 0; - uint32_t* proAliases = nullptr; - uint32_t proTrait = aProTraits[traitIndex]; - if (traitService) - { - rv = traitService->GetAliases(proTrait, &proAliasesLength, &proAliases); - if (NS_FAILED(rv)) - { - NS_ERROR("trait service failed to get aliases"); - MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, ("trait service failed to get aliases")); - } + // pro trait + uint32_t proAliasesLength = 0; + uint32_t* proAliases = nullptr; + uint32_t proTrait = aProTraits[traitIndex]; + if (traitService) { + rv = traitService->GetAliases(proTrait, &proAliasesLength, &proAliases); + if (NS_FAILED(rv)) { + NS_ERROR("trait service failed to get aliases"); + MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, + ("trait service failed to get aliases")); } - proAliasesLengths.AppendElement(proAliasesLength); - proAliasArrays.AppendElement(proAliases); - uint32_t proMessageCount = mCorpus.getMessageCount(proTrait); - for (uint32_t aliasIndex = 0; aliasIndex < proAliasesLength; aliasIndex++) - proMessageCount += mCorpus.getMessageCount(proAliases[aliasIndex]); - numProMessages.AppendElement(proMessageCount); - - // anti trait - uint32_t antiAliasesLength = 0; - uint32_t* antiAliases = nullptr; - uint32_t antiTrait = aAntiTraits[traitIndex]; - if (traitService) - { - rv = traitService->GetAliases(antiTrait, &antiAliasesLength, &antiAliases); - if (NS_FAILED(rv)) - { - NS_ERROR("trait service failed to get aliases"); - MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, ("trait service failed to get aliases")); - } - } - antiAliasesLengths.AppendElement(antiAliasesLength); - antiAliasArrays.AppendElement(antiAliases); - uint32_t antiMessageCount = mCorpus.getMessageCount(antiTrait); - for (uint32_t aliasIndex = 0; aliasIndex < antiAliasesLength; aliasIndex++) - antiMessageCount += mCorpus.getMessageCount(antiAliases[aliasIndex]); - numAntiMessages.AppendElement(antiMessageCount); } + proAliasesLengths.AppendElement(proAliasesLength); + proAliasArrays.AppendElement(proAliases); + uint32_t proMessageCount = mCorpus.getMessageCount(proTrait); + for (uint32_t aliasIndex = 0; aliasIndex < proAliasesLength; aliasIndex++) + proMessageCount += mCorpus.getMessageCount(proAliases[aliasIndex]); + numProMessages.AppendElement(proMessageCount); - for (uint32_t i = 0; i < tokenCount; ++i) - { - Token& token = tokens[i]; - CorpusToken* t = mCorpus.get(token.mWord); - if (!t) + // anti trait + uint32_t antiAliasesLength = 0; + uint32_t* antiAliases = nullptr; + uint32_t antiTrait = aAntiTraits[traitIndex]; + if (traitService) { + rv = + traitService->GetAliases(antiTrait, &antiAliasesLength, &antiAliases); + if (NS_FAILED(rv)) { + NS_ERROR("trait service failed to get aliases"); + MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, + ("trait service failed to get aliases")); + } + } + antiAliasesLengths.AppendElement(antiAliasesLength); + antiAliasArrays.AppendElement(antiAliases); + uint32_t antiMessageCount = mCorpus.getMessageCount(antiTrait); + for (uint32_t aliasIndex = 0; aliasIndex < antiAliasesLength; aliasIndex++) + antiMessageCount += mCorpus.getMessageCount(antiAliases[aliasIndex]); + numAntiMessages.AppendElement(antiMessageCount); + } + + for (uint32_t i = 0; i < tokenCount; ++i) { + Token& token = tokens[i]; + CorpusToken* t = mCorpus.get(token.mWord); + if (!t) continue; + for (uint32_t traitIndex = 0; traitIndex < traitCount; traitIndex++) { + uint32_t iProCount = mCorpus.getTraitCount(t, aProTraits[traitIndex]); + // add in any counts for aliases to proTrait + for (uint32_t aliasIndex = 0; aliasIndex < proAliasesLengths[traitIndex]; + aliasIndex++) + iProCount += + mCorpus.getTraitCount(t, proAliasArrays[traitIndex][aliasIndex]); + double proCount = static_cast(iProCount); + + uint32_t iAntiCount = mCorpus.getTraitCount(t, aAntiTraits[traitIndex]); + // add in any counts for aliases to antiTrait + for (uint32_t aliasIndex = 0; aliasIndex < antiAliasesLengths[traitIndex]; + aliasIndex++) + iAntiCount += + mCorpus.getTraitCount(t, antiAliasArrays[traitIndex][aliasIndex]); + double antiCount = static_cast(iAntiCount); + + double prob, denom; + // Prevent a divide by zero error by setting defaults for prob + + // If there are no matching tokens at all, ignore. + if (antiCount == 0.0 && proCount == 0.0) continue; + // if only anti match, set probability to 0% + if (proCount == 0.0) prob = 0.0; + // if only pro match, set probability to 100% + else if (antiCount == 0.0) + prob = 1.0; + // not really needed, but just to be sure check the denom as well + else if ((denom = proCount * numAntiMessages[traitIndex] + + antiCount * numProMessages[traitIndex]) == 0.0) continue; - for (uint32_t traitIndex = 0; traitIndex < traitCount; traitIndex++) - { - uint32_t iProCount = mCorpus.getTraitCount(t, aProTraits[traitIndex]); - // add in any counts for aliases to proTrait - for (uint32_t aliasIndex = 0; aliasIndex < proAliasesLengths[traitIndex]; aliasIndex++) - iProCount += mCorpus.getTraitCount(t, proAliasArrays[traitIndex][aliasIndex]); - double proCount = static_cast(iProCount); - - uint32_t iAntiCount = mCorpus.getTraitCount(t, aAntiTraits[traitIndex]); - // add in any counts for aliases to antiTrait - for (uint32_t aliasIndex = 0; aliasIndex < antiAliasesLengths[traitIndex]; aliasIndex++) - iAntiCount += mCorpus.getTraitCount(t, antiAliasArrays[traitIndex][aliasIndex]); - double antiCount = static_cast(iAntiCount); - - double prob, denom; - // Prevent a divide by zero error by setting defaults for prob - - // If there are no matching tokens at all, ignore. - if (antiCount == 0.0 && proCount == 0.0) - continue; - // if only anti match, set probability to 0% - if (proCount == 0.0) - prob = 0.0; - // if only pro match, set probability to 100% - else if (antiCount == 0.0) - prob = 1.0; - // not really needed, but just to be sure check the denom as well - else if ((denom = proCount * numAntiMessages[traitIndex] + - antiCount * numProMessages[traitIndex]) == 0.0) - continue; - else - prob = (proCount * numAntiMessages[traitIndex]) / denom; - - double n = proCount + antiCount; - prob = (0.225 + n * prob) / (.45 + n); - double distance = std::abs(prob - 0.5); - if (distance >= .1) - { - mozilla::DebugOnly rv = setAnalysis(token, traitIndex, distance, prob); - NS_ASSERTION(NS_SUCCEEDED(rv), "Problem in setAnalysis"); - } - } - } - - for (uint32_t traitIndex = 0; traitIndex < traitCount; traitIndex++) - { - AutoTArray traitAnalyses; - // copy valid tokens into an array to sort - for (uint32_t tokenIndex = 0; tokenIndex < tokenCount; tokenIndex++) - { - uint32_t storeIndex = getAnalysisIndex(tokens[tokenIndex], traitIndex); - if (storeIndex) - { - TraitAnalysis ta = - {tokenIndex, - mAnalysisStore[storeIndex].mDistance, - mAnalysisStore[storeIndex].mProbability}; - traitAnalyses.AppendElement(ta); - } - } - - // sort the array by the distances - traitAnalyses.Sort(compareTraitAnalysis()); - uint32_t count = traitAnalyses.Length(); - uint32_t first, last = count; - const uint32_t kMaxTokens = 150; - first = ( count > kMaxTokens) ? count - kMaxTokens : 0; - - // Setup the arrays to save details if needed - nsTArray sArray; - nsTArray hArray; - uint32_t usedTokenCount = ( count > kMaxTokens) ? kMaxTokens : count; - if (aDetailListener) - { - sArray.SetCapacity(usedTokenCount); - hArray.SetCapacity(usedTokenCount); - } - - double H = 1.0, S = 1.0; - int32_t Hexp = 0, Sexp = 0; - uint32_t goodclues=0; - int e; - - // index from end to analyze most significant first - for (uint32_t ip1 = last; ip1 != first; --ip1) - { - TraitAnalysis& ta = traitAnalyses[ip1 - 1]; - if (ta.mDistance > 0.0) - { - goodclues++; - double value = ta.mProbability; - S *= (1.0 - value); - H *= value; - if ( S < 1e-200 ) - { - S = frexp(S, &e); - Sexp += e; - } - if ( H < 1e-200 ) - { - H = frexp(H, &e); - Hexp += e; - } - MOZ_LOG(BayesianFilterLogModule, LogLevel::Warning, - ("token probability (%s) is %f", - tokens[ta.mTokenIndex].mWord, ta.mProbability)); - } - if (aDetailListener) - { - sArray.AppendElement(log(S) + Sexp * M_LN2); - hArray.AppendElement(log(H) + Hexp * M_LN2); - } - } - - S = log(S) + Sexp * M_LN2; - H = log(H) + Hexp * M_LN2; - - double prob; - if (goodclues > 0) - { - int32_t chi_error; - S = chi2P(-2.0 * S, 2.0 * goodclues, &chi_error); - if (!chi_error) - H = chi2P(-2.0 * H, 2.0 * goodclues, &chi_error); - // if any error toss the entire calculation - if (!chi_error) - prob = (S-H +1.0) / 2.0; - else - prob = 0.5; - } else - prob = 0.5; + prob = (proCount * numAntiMessages[traitIndex]) / denom; - if (aDetailListener) - { - // Prepare output arrays - nsTArray tokenPercents(usedTokenCount); - nsTArray runningPercents(usedTokenCount); - nsTArray tokenStrings(usedTokenCount); - - double clueCount = 1.0; - for (uint32_t tokenIndex = 0; tokenIndex < usedTokenCount; tokenIndex++) - { - TraitAnalysis& ta = traitAnalyses[last - 1 - tokenIndex]; - int32_t chi_error; - S = chi2P(-2.0 * sArray[tokenIndex], 2.0 * clueCount, &chi_error); - if (!chi_error) - H = chi2P(-2.0 * hArray[tokenIndex], 2.0 * clueCount, &chi_error); - clueCount += 1.0; - double runningProb; - if (!chi_error) - runningProb = (S - H + 1.0) / 2.0; - else - runningProb = 0.5; - runningPercents.AppendElement(static_cast(runningProb * - 100. + .5)); - tokenPercents.AppendElement(static_cast(ta.mProbability * - 100. + .5)); - tokenStrings.AppendElement(ToNewUnicode(NS_ConvertUTF8toUTF16( - tokens[ta.mTokenIndex].mWord))); - } - - aDetailListener->OnMessageTraitDetails(messageURI, aProTraits[traitIndex], - usedTokenCount, (const char16_t**)tokenStrings.Elements(), - tokenPercents.Elements(), runningPercents.Elements()); - for (uint32_t tokenIndex = 0; tokenIndex < usedTokenCount; tokenIndex++) - free(tokenStrings[tokenIndex]); + double n = proCount + antiCount; + prob = (0.225 + n * prob) / (.45 + n); + double distance = std::abs(prob - 0.5); + if (distance >= .1) { + mozilla::DebugOnly rv = + setAnalysis(token, traitIndex, distance, prob); + NS_ASSERTION(NS_SUCCEEDED(rv), "Problem in setAnalysis"); } + } + } - uint32_t proPercent = static_cast(prob*100. + .5); - - // directly classify junk to maintain backwards compatibility - if (aProTraits[traitIndex] == kJunkTrait) - { - bool isJunk = (prob >= mJunkProbabilityThreshold); - MOZ_LOG(BayesianFilterLogModule, LogLevel::Info, - ("%s is junk probability = (%f) HAM SCORE:%f SPAM SCORE:%f", - messageURI, prob,H,S)); - - // the algorithm in "A Plan For Spam" assumes that you have a large good - // corpus and a large junk corpus. - // that won't be the case with users who first use the junk mail trait - // so, we do certain things to encourage them to train. - // - // if there are no good tokens, assume the message is junk - // this will "encourage" the user to train - // and if there are no bad tokens, assume the message is not junk - // this will also "encourage" the user to train - // see bug #194238 - - if (listener && !mCorpus.getMessageCount(kGoodTrait)) - isJunk = true; - else if (listener && !mCorpus.getMessageCount(kJunkTrait)) - isJunk = false; - - if (listener) - listener->OnMessageClassified(messageURI, isJunk ? - nsMsgJunkStatus(nsIJunkMailPlugin::JUNK) : - nsMsgJunkStatus(nsIJunkMailPlugin::GOOD), proPercent); + for (uint32_t traitIndex = 0; traitIndex < traitCount; traitIndex++) { + AutoTArray traitAnalyses; + // copy valid tokens into an array to sort + for (uint32_t tokenIndex = 0; tokenIndex < tokenCount; tokenIndex++) { + uint32_t storeIndex = getAnalysisIndex(tokens[tokenIndex], traitIndex); + if (storeIndex) { + TraitAnalysis ta = {tokenIndex, mAnalysisStore[storeIndex].mDistance, + mAnalysisStore[storeIndex].mProbability}; + traitAnalyses.AppendElement(ta); } - - if (aTraitListener) - { - traits.AppendElement(aProTraits[traitIndex]); - percents.AppendElement(proPercent); - } - - // free aliases arrays returned from XPCOM - if (proAliasesLengths[traitIndex]) - free(proAliasArrays[traitIndex]); - if (antiAliasesLengths[traitIndex]) - free(antiAliasArrays[traitIndex]); } - if (aTraitListener) - aTraitListener->OnMessageTraitsClassified(messageURI, - traits.Length(), traits.Elements(), percents.Elements()); + // sort the array by the distances + traitAnalyses.Sort(compareTraitAnalysis()); + uint32_t count = traitAnalyses.Length(); + uint32_t first, last = count; + const uint32_t kMaxTokens = 150; + first = (count > kMaxTokens) ? count - kMaxTokens : 0; - delete[] tokens; - // reuse mAnalysisStore without clearing memory - mNextAnalysisIndex = 1; - // but shrink it back to the default size - if (mAnalysisStore.Length() > kAnalysisStoreCapacity) - mAnalysisStore.RemoveElementsAt(kAnalysisStoreCapacity, - mAnalysisStore.Length() - kAnalysisStoreCapacity); - mAnalysisStore.Compact(); + // Setup the arrays to save details if needed + nsTArray sArray; + nsTArray hArray; + uint32_t usedTokenCount = (count > kMaxTokens) ? kMaxTokens : count; + if (aDetailListener) { + sArray.SetCapacity(usedTokenCount); + hArray.SetCapacity(usedTokenCount); + } + + double H = 1.0, S = 1.0; + int32_t Hexp = 0, Sexp = 0; + uint32_t goodclues = 0; + int e; + + // index from end to analyze most significant first + for (uint32_t ip1 = last; ip1 != first; --ip1) { + TraitAnalysis& ta = traitAnalyses[ip1 - 1]; + if (ta.mDistance > 0.0) { + goodclues++; + double value = ta.mProbability; + S *= (1.0 - value); + H *= value; + if (S < 1e-200) { + S = frexp(S, &e); + Sexp += e; + } + if (H < 1e-200) { + H = frexp(H, &e); + Hexp += e; + } + MOZ_LOG(BayesianFilterLogModule, LogLevel::Warning, + ("token probability (%s) is %f", tokens[ta.mTokenIndex].mWord, + ta.mProbability)); + } + if (aDetailListener) { + sArray.AppendElement(log(S) + Sexp * M_LN2); + hArray.AppendElement(log(H) + Hexp * M_LN2); + } + } + + S = log(S) + Sexp * M_LN2; + H = log(H) + Hexp * M_LN2; + + double prob; + if (goodclues > 0) { + int32_t chi_error; + S = chi2P(-2.0 * S, 2.0 * goodclues, &chi_error); + if (!chi_error) H = chi2P(-2.0 * H, 2.0 * goodclues, &chi_error); + // if any error toss the entire calculation + if (!chi_error) + prob = (S - H + 1.0) / 2.0; + else + prob = 0.5; + } else + prob = 0.5; + + if (aDetailListener) { + // Prepare output arrays + nsTArray tokenPercents(usedTokenCount); + nsTArray runningPercents(usedTokenCount); + nsTArray tokenStrings(usedTokenCount); + + double clueCount = 1.0; + for (uint32_t tokenIndex = 0; tokenIndex < usedTokenCount; tokenIndex++) { + TraitAnalysis& ta = traitAnalyses[last - 1 - tokenIndex]; + int32_t chi_error; + S = chi2P(-2.0 * sArray[tokenIndex], 2.0 * clueCount, &chi_error); + if (!chi_error) + H = chi2P(-2.0 * hArray[tokenIndex], 2.0 * clueCount, &chi_error); + clueCount += 1.0; + double runningProb; + if (!chi_error) + runningProb = (S - H + 1.0) / 2.0; + else + runningProb = 0.5; + runningPercents.AppendElement( + static_cast(runningProb * 100. + .5)); + tokenPercents.AppendElement( + static_cast(ta.mProbability * 100. + .5)); + tokenStrings.AppendElement( + ToNewUnicode(NS_ConvertUTF8toUTF16(tokens[ta.mTokenIndex].mWord))); + } + + aDetailListener->OnMessageTraitDetails( + messageURI, aProTraits[traitIndex], usedTokenCount, + (const char16_t**)tokenStrings.Elements(), tokenPercents.Elements(), + runningPercents.Elements()); + for (uint32_t tokenIndex = 0; tokenIndex < usedTokenCount; tokenIndex++) + free(tokenStrings[tokenIndex]); + } + + uint32_t proPercent = static_cast(prob * 100. + .5); + + // directly classify junk to maintain backwards compatibility + if (aProTraits[traitIndex] == kJunkTrait) { + bool isJunk = (prob >= mJunkProbabilityThreshold); + MOZ_LOG(BayesianFilterLogModule, LogLevel::Info, + ("%s is junk probability = (%f) HAM SCORE:%f SPAM SCORE:%f", + messageURI, prob, H, S)); + + // the algorithm in "A Plan For Spam" assumes that you have a large good + // corpus and a large junk corpus. + // that won't be the case with users who first use the junk mail trait + // so, we do certain things to encourage them to train. + // + // if there are no good tokens, assume the message is junk + // this will "encourage" the user to train + // and if there are no bad tokens, assume the message is not junk + // this will also "encourage" the user to train + // see bug #194238 + + if (listener && !mCorpus.getMessageCount(kGoodTrait)) + isJunk = true; + else if (listener && !mCorpus.getMessageCount(kJunkTrait)) + isJunk = false; + + if (listener) + listener->OnMessageClassified( + messageURI, + isJunk ? nsMsgJunkStatus(nsIJunkMailPlugin::JUNK) + : nsMsgJunkStatus(nsIJunkMailPlugin::GOOD), + proPercent); + } + + if (aTraitListener) { + traits.AppendElement(aProTraits[traitIndex]); + percents.AppendElement(proPercent); + } + + // free aliases arrays returned from XPCOM + if (proAliasesLengths[traitIndex]) free(proAliasArrays[traitIndex]); + if (antiAliasesLengths[traitIndex]) free(antiAliasArrays[traitIndex]); + } + + if (aTraitListener) + aTraitListener->OnMessageTraitsClassified( + messageURI, traits.Length(), traits.Elements(), percents.Elements()); + + delete[] tokens; + // reuse mAnalysisStore without clearing memory + mNextAnalysisIndex = 1; + // but shrink it back to the default size + if (mAnalysisStore.Length() > kAnalysisStoreCapacity) + mAnalysisStore.RemoveElementsAt( + kAnalysisStoreCapacity, + mAnalysisStore.Length() - kAnalysisStoreCapacity); + mAnalysisStore.Compact(); } void nsBayesianFilter::classifyMessage( - Tokenizer& tokens, - const char* messageURI, - nsIJunkMailClassificationListener* aJunkListener) -{ + Tokenizer& tokens, const char* messageURI, + nsIJunkMailClassificationListener* aJunkListener) { AutoTArray proTraits; AutoTArray antiTraits; proTraits.AppendElement(kJunkTrait); antiTraits.AppendElement(kGoodTrait); - classifyMessage(tokens, messageURI, proTraits, antiTraits, - aJunkListener, nullptr, nullptr); + classifyMessage(tokens, messageURI, proTraits, antiTraits, aJunkListener, + nullptr, nullptr); } NS_IMETHODIMP -nsBayesianFilter::Observe(nsISupports *aSubject, const char *aTopic, - const char16_t *someData) -{ - if (!strcmp(aTopic, "profile-before-change")) - Shutdown(); +nsBayesianFilter::Observe(nsISupports* aSubject, const char* aTopic, + const char16_t* someData) { + if (!strcmp(aTopic, "profile-before-change")) Shutdown(); return NS_OK; } /* void shutdown (); */ -NS_IMETHODIMP nsBayesianFilter::Shutdown() -{ - if (mTrainingDataDirty) - mCorpus.writeTrainingData(mMaximumTokenCount); +NS_IMETHODIMP nsBayesianFilter::Shutdown() { + if (mTrainingDataDirty) mCorpus.writeTrainingData(mMaximumTokenCount); mTrainingDataDirty = false; return NS_OK; } /* readonly attribute boolean shouldDownloadAllHeaders; */ -NS_IMETHODIMP nsBayesianFilter::GetShouldDownloadAllHeaders(bool *aShouldDownloadAllHeaders) -{ - // bayesian filters work on the whole msg body currently. - *aShouldDownloadAllHeaders = false; - return NS_OK; +NS_IMETHODIMP nsBayesianFilter::GetShouldDownloadAllHeaders( + bool* aShouldDownloadAllHeaders) { + // bayesian filters work on the whole msg body currently. + *aShouldDownloadAllHeaders = false; + return NS_OK; } -/* void classifyMessage (in string aMsgURL, in nsIJunkMailClassificationListener aListener); */ -NS_IMETHODIMP nsBayesianFilter::ClassifyMessage(const char *aMessageURL, nsIMsgWindow *aMsgWindow, nsIJunkMailClassificationListener *aListener) -{ - MessageClassifier* analyzer = new MessageClassifier(this, aListener, aMsgWindow, 1, &aMessageURL); - NS_ENSURE_TRUE(analyzer, NS_ERROR_OUT_OF_MEMORY); - TokenStreamListener *tokenListener = new TokenStreamListener(analyzer); - NS_ENSURE_TRUE(tokenListener, NS_ERROR_OUT_OF_MEMORY); - analyzer->setTokenListener(tokenListener); - return tokenizeMessage(aMessageURL, aMsgWindow, analyzer); +/* void classifyMessage (in string aMsgURL, in nsIJunkMailClassificationListener + * aListener); */ +NS_IMETHODIMP nsBayesianFilter::ClassifyMessage( + const char* aMessageURL, nsIMsgWindow* aMsgWindow, + nsIJunkMailClassificationListener* aListener) { + MessageClassifier* analyzer = + new MessageClassifier(this, aListener, aMsgWindow, 1, &aMessageURL); + NS_ENSURE_TRUE(analyzer, NS_ERROR_OUT_OF_MEMORY); + TokenStreamListener* tokenListener = new TokenStreamListener(analyzer); + NS_ENSURE_TRUE(tokenListener, NS_ERROR_OUT_OF_MEMORY); + analyzer->setTokenListener(tokenListener); + return tokenizeMessage(aMessageURL, aMsgWindow, analyzer); } -/* void classifyMessages (in unsigned long aCount, [array, size_is (aCount)] in string aMsgURLs, in nsIJunkMailClassificationListener aListener); */ -NS_IMETHODIMP nsBayesianFilter::ClassifyMessages(uint32_t aCount, const char **aMsgURLs, nsIMsgWindow *aMsgWindow, nsIJunkMailClassificationListener *aListener) -{ - NS_ENSURE_ARG_POINTER(aMsgURLs); +/* void classifyMessages (in unsigned long aCount, [array, size_is (aCount)] in + * string aMsgURLs, in nsIJunkMailClassificationListener aListener); */ +NS_IMETHODIMP nsBayesianFilter::ClassifyMessages( + uint32_t aCount, const char** aMsgURLs, nsIMsgWindow* aMsgWindow, + nsIJunkMailClassificationListener* aListener) { + NS_ENSURE_ARG_POINTER(aMsgURLs); - TokenAnalyzer* analyzer = new MessageClassifier(this, aListener, aMsgWindow, aCount, aMsgURLs); - NS_ENSURE_TRUE(analyzer, NS_ERROR_OUT_OF_MEMORY); - TokenStreamListener *tokenListener = new TokenStreamListener(analyzer); - NS_ENSURE_TRUE(tokenListener, NS_ERROR_OUT_OF_MEMORY); - analyzer->setTokenListener(tokenListener); - return tokenizeMessage(aMsgURLs[0], aMsgWindow, analyzer); + TokenAnalyzer* analyzer = + new MessageClassifier(this, aListener, aMsgWindow, aCount, aMsgURLs); + NS_ENSURE_TRUE(analyzer, NS_ERROR_OUT_OF_MEMORY); + TokenStreamListener* tokenListener = new TokenStreamListener(analyzer); + NS_ENSURE_TRUE(tokenListener, NS_ERROR_OUT_OF_MEMORY); + analyzer->setTokenListener(tokenListener); + return tokenizeMessage(aMsgURLs[0], aMsgWindow, analyzer); } nsresult nsBayesianFilter::setAnalysis(Token& token, uint32_t aTraitIndex, - double aDistance, double aProbability) -{ + double aDistance, double aProbability) { uint32_t nextLink = token.mAnalysisLink; uint32_t lastLink = 0; uint32_t linkCount = 0, maxLinks = 100; // try to find an existing element. Limit the search to maxLinks // as a precaution - for (linkCount = 0; nextLink && linkCount < maxLinks; linkCount++) - { - AnalysisPerToken &rAnalysis = mAnalysisStore[nextLink]; - if (rAnalysis.mTraitIndex == aTraitIndex) - { + for (linkCount = 0; nextLink && linkCount < maxLinks; linkCount++) { + AnalysisPerToken& rAnalysis = mAnalysisStore[nextLink]; + if (rAnalysis.mTraitIndex == aTraitIndex) { rAnalysis.mDistance = aDistance; rAnalysis.mProbability = aProbability; return NS_OK; @@ -1851,8 +1745,7 @@ nsresult nsBayesianFilter::setAnalysis(Token& token, uint32_t aTraitIndex, lastLink = nextLink; nextLink = rAnalysis.mNextLink; } - if (linkCount >= maxLinks) - return NS_ERROR_FAILURE; + if (linkCount >= maxLinks) return NS_ERROR_FAILURE; // trait does not exist, so add it @@ -1861,7 +1754,7 @@ nsresult nsBayesianFilter::setAnalysis(Token& token, uint32_t aTraitIndex, mAnalysisStore.InsertElementAt(mNextAnalysisIndex, analysis); else if (mAnalysisStore.Length() > mNextAnalysisIndex) mAnalysisStore.ReplaceElementsAt(mNextAnalysisIndex, 1, analysis); - else // we can only insert at the end of the array + else // we can only insert at the end of the array return NS_ERROR_FAILURE; if (lastLink) @@ -1875,15 +1768,14 @@ nsresult nsBayesianFilter::setAnalysis(Token& token, uint32_t aTraitIndex, return NS_OK; } -uint32_t nsBayesianFilter::getAnalysisIndex(Token& token, uint32_t aTraitIndex) -{ +uint32_t nsBayesianFilter::getAnalysisIndex(Token& token, + uint32_t aTraitIndex) { uint32_t nextLink; uint32_t linkCount = 0, maxLinks = 100; - for (nextLink = token.mAnalysisLink; nextLink && linkCount < maxLinks; linkCount++) - { - AnalysisPerToken &rAnalysis = mAnalysisStore[nextLink]; - if (rAnalysis.mTraitIndex == aTraitIndex) - return nextLink; + for (nextLink = token.mAnalysisLink; nextLink && linkCount < maxLinks; + linkCount++) { + AnalysisPerToken& rAnalysis = mAnalysisStore[nextLink]; + if (rAnalysis.mTraitIndex == aTraitIndex) return nextLink; nextLink = rAnalysis.mNextLink; } NS_ASSERTION(linkCount < maxLinks, "corrupt analysis store"); @@ -1893,43 +1785,35 @@ uint32_t nsBayesianFilter::getAnalysisIndex(Token& token, uint32_t aTraitIndex) } NS_IMETHODIMP nsBayesianFilter::ClassifyTraitsInMessage( - const char *aMsgURI, - uint32_t aTraitCount, - uint32_t *aProTraits, - uint32_t *aAntiTraits, - nsIMsgTraitClassificationListener *aTraitListener, - nsIMsgWindow *aMsgWindow, - nsIJunkMailClassificationListener *aJunkListener) -{ + const char* aMsgURI, uint32_t aTraitCount, uint32_t* aProTraits, + uint32_t* aAntiTraits, nsIMsgTraitClassificationListener* aTraitListener, + nsIMsgWindow* aMsgWindow, + nsIJunkMailClassificationListener* aJunkListener) { return ClassifyTraitsInMessages(1, &aMsgURI, aTraitCount, aProTraits, - aAntiTraits, aTraitListener, aMsgWindow, aJunkListener); + aAntiTraits, aTraitListener, aMsgWindow, + aJunkListener); } NS_IMETHODIMP nsBayesianFilter::ClassifyTraitsInMessages( - uint32_t aCount, - const char **aMsgURIs, - uint32_t aTraitCount, - uint32_t *aProTraits, - uint32_t *aAntiTraits, - nsIMsgTraitClassificationListener *aTraitListener, - nsIMsgWindow *aMsgWindow, - nsIJunkMailClassificationListener *aJunkListener) -{ + uint32_t aCount, const char** aMsgURIs, uint32_t aTraitCount, + uint32_t* aProTraits, uint32_t* aAntiTraits, + nsIMsgTraitClassificationListener* aTraitListener, nsIMsgWindow* aMsgWindow, + nsIJunkMailClassificationListener* aJunkListener) { AutoTArray proTraits; AutoTArray antiTraits; - if (aTraitCount > kTraitAutoCapacity) - { + if (aTraitCount > kTraitAutoCapacity) { proTraits.SetCapacity(aTraitCount); antiTraits.SetCapacity(aTraitCount); } proTraits.AppendElements(aProTraits, aTraitCount); antiTraits.AppendElements(aAntiTraits, aTraitCount); - MessageClassifier* analyzer = new MessageClassifier(this, aJunkListener, - aTraitListener, nullptr, proTraits, antiTraits, aMsgWindow, aCount, aMsgURIs); + MessageClassifier* analyzer = new MessageClassifier( + this, aJunkListener, aTraitListener, nullptr, proTraits, antiTraits, + aMsgWindow, aCount, aMsgURIs); NS_ENSURE_TRUE(analyzer, NS_ERROR_OUT_OF_MEMORY); - TokenStreamListener *tokenListener = new TokenStreamListener(analyzer); + TokenStreamListener* tokenListener = new TokenStreamListener(analyzer); NS_ENSURE_TRUE(tokenListener, NS_ERROR_OUT_OF_MEMORY); analyzer->setTokenListener(tokenListener); @@ -1937,28 +1821,27 @@ NS_IMETHODIMP nsBayesianFilter::ClassifyTraitsInMessages( } class MessageObserver : public TokenAnalyzer { -public: + public: MessageObserver(nsBayesianFilter* filter, nsTArray& aOldClassifications, nsTArray& aNewClassifications, nsIJunkMailClassificationListener* aJunkListener, nsIMsgTraitClassificationListener* aTraitListener) - : mFilter(filter), mJunkMailPlugin(filter), mJunkListener(aJunkListener), - mTraitListener(aTraitListener), - mOldClassifications(aOldClassifications), - mNewClassifications(aNewClassifications) - { - } + : mFilter(filter), + mJunkMailPlugin(filter), + mJunkListener(aJunkListener), + mTraitListener(aTraitListener), + mOldClassifications(aOldClassifications), + mNewClassifications(aNewClassifications) {} - virtual void analyzeTokens(Tokenizer& tokenizer) - { + virtual void analyzeTokens(Tokenizer& tokenizer) { mFilter->observeMessage(tokenizer, mTokenSource.get(), mOldClassifications, mNewClassifications, mJunkListener, mTraitListener); // release reference to listener, which will allow us to go away as well. mTokenListener = nullptr; } -private: + private: nsBayesianFilter* mFilter; nsCOMPtr mJunkMailPlugin; nsCOMPtr mJunkListener; @@ -1968,29 +1851,22 @@ private: }; NS_IMETHODIMP nsBayesianFilter::SetMsgTraitClassification( - const char *aMsgURI, - uint32_t aOldCount, - uint32_t *aOldTraits, - uint32_t aNewCount, - uint32_t *aNewTraits, - nsIMsgTraitClassificationListener *aTraitListener, - nsIMsgWindow *aMsgWindow, - nsIJunkMailClassificationListener *aJunkListener) -{ + const char* aMsgURI, uint32_t aOldCount, uint32_t* aOldTraits, + uint32_t aNewCount, uint32_t* aNewTraits, + nsIMsgTraitClassificationListener* aTraitListener, nsIMsgWindow* aMsgWindow, + nsIJunkMailClassificationListener* aJunkListener) { AutoTArray oldTraits; AutoTArray newTraits; - if (aOldCount > kTraitAutoCapacity) - oldTraits.SetCapacity(aOldCount); - if (aNewCount > kTraitAutoCapacity) - newTraits.SetCapacity(aNewCount); + if (aOldCount > kTraitAutoCapacity) oldTraits.SetCapacity(aOldCount); + if (aNewCount > kTraitAutoCapacity) newTraits.SetCapacity(aNewCount); oldTraits.AppendElements(aOldTraits, aOldCount); newTraits.AppendElements(aNewTraits, aNewCount); - MessageObserver* analyzer = new MessageObserver(this, oldTraits, - newTraits, aJunkListener, aTraitListener); + MessageObserver* analyzer = new MessageObserver( + this, oldTraits, newTraits, aJunkListener, aTraitListener); NS_ENSURE_TRUE(analyzer, NS_ERROR_OUT_OF_MEMORY); - TokenStreamListener *tokenListener = new TokenStreamListener(analyzer); + TokenStreamListener* tokenListener = new TokenStreamListener(analyzer); NS_ENSURE_TRUE(tokenListener, NS_ERROR_OUT_OF_MEMORY); analyzer->setTokenListener(tokenListener); @@ -1999,116 +1875,102 @@ NS_IMETHODIMP nsBayesianFilter::SetMsgTraitClassification( // set new message classifications for a message void nsBayesianFilter::observeMessage( - Tokenizer& tokenizer, - const char* messageURL, + Tokenizer& tokenizer, const char* messageURL, nsTArray& oldClassifications, nsTArray& newClassifications, nsIJunkMailClassificationListener* aJunkListener, - nsIMsgTraitClassificationListener* aTraitListener) -{ + nsIMsgTraitClassificationListener* aTraitListener) { + bool trainingDataWasDirty = mTrainingDataDirty; - bool trainingDataWasDirty = mTrainingDataDirty; - - // Uhoh...if the user is re-training then the message may already be classified and we are classifying it again with the same classification. - // the old code would have removed the tokens for this message then added them back. But this really hurts the message occurrence - // count for tokens if you just removed training.dat and are re-training. See Bug #237095 for more details. - // What can we do here? Well we can skip the token removal step if the classifications are the same and assume the user is - // just re-training. But this then allows users to re-classify the same message on the same training set over and over again - // leading to data skew. But that's all I can think to do right now to address this..... - uint32_t oldLength = oldClassifications.Length(); - for (uint32_t index = 0; index < oldLength; index++) - { - uint32_t trait = oldClassifications.ElementAt(index); - // skip removing if trait is also in the new set - if (newClassifications.Contains(trait)) - continue; - // remove the tokens from the token set it is currently in - uint32_t messageCount; - messageCount = mCorpus.getMessageCount(trait); - if (messageCount > 0) - { - mCorpus.setMessageCount(trait, messageCount - 1); - mCorpus.forgetTokens(tokenizer, trait, 1); - mTrainingDataDirty = true; - } - } - - nsMsgJunkStatus newClassification = nsIJunkMailPlugin::UNCLASSIFIED; - uint32_t junkPercent = 0; // 0 here is no possibility of meeting the classification - uint32_t newLength = newClassifications.Length(); - for (uint32_t index = 0; index < newLength; index++) - { - uint32_t trait = newClassifications.ElementAt(index); - mCorpus.setMessageCount(trait, mCorpus.getMessageCount(trait) + 1); - mCorpus.rememberTokens(tokenizer, trait, 1); + // Uhoh...if the user is re-training then the message may already be + // classified and we are classifying it again with the same classification. + // the old code would have removed the tokens for this message then added them + // back. But this really hurts the message occurrence count for tokens if you + // just removed training.dat and are re-training. See Bug #237095 for more + // details. What can we do here? Well we can skip the token removal step if + // the classifications are the same and assume the user is just re-training. + // But this then allows users to re-classify the same message on the same + // training set over and over again leading to data skew. But that's all I can + // think to do right now to address this..... + uint32_t oldLength = oldClassifications.Length(); + for (uint32_t index = 0; index < oldLength; index++) { + uint32_t trait = oldClassifications.ElementAt(index); + // skip removing if trait is also in the new set + if (newClassifications.Contains(trait)) continue; + // remove the tokens from the token set it is currently in + uint32_t messageCount; + messageCount = mCorpus.getMessageCount(trait); + if (messageCount > 0) { + mCorpus.setMessageCount(trait, messageCount - 1); + mCorpus.forgetTokens(tokenizer, trait, 1); mTrainingDataDirty = true; + } + } - if (aJunkListener) - { - if (trait == kJunkTrait) - { - junkPercent = nsIJunkMailPlugin::IS_SPAM_SCORE; - newClassification = nsIJunkMailPlugin::JUNK; - } - else if (trait == kGoodTrait) - { - junkPercent = nsIJunkMailPlugin::IS_HAM_SCORE; - newClassification = nsIJunkMailPlugin::GOOD; - } + nsMsgJunkStatus newClassification = nsIJunkMailPlugin::UNCLASSIFIED; + uint32_t junkPercent = + 0; // 0 here is no possibility of meeting the classification + uint32_t newLength = newClassifications.Length(); + for (uint32_t index = 0; index < newLength; index++) { + uint32_t trait = newClassifications.ElementAt(index); + mCorpus.setMessageCount(trait, mCorpus.getMessageCount(trait) + 1); + mCorpus.rememberTokens(tokenizer, trait, 1); + mTrainingDataDirty = true; + + if (aJunkListener) { + if (trait == kJunkTrait) { + junkPercent = nsIJunkMailPlugin::IS_SPAM_SCORE; + newClassification = nsIJunkMailPlugin::JUNK; + } else if (trait == kGoodTrait) { + junkPercent = nsIJunkMailPlugin::IS_HAM_SCORE; + newClassification = nsIJunkMailPlugin::GOOD; } } + } - if (aJunkListener) - aJunkListener->OnMessageClassified(messageURL, newClassification, junkPercent); + if (aJunkListener) + aJunkListener->OnMessageClassified(messageURL, newClassification, + junkPercent); - if (aTraitListener) - { - // construct the outgoing listener arrays - AutoTArray traits; - AutoTArray percents; - uint32_t newLength = newClassifications.Length(); - if (newLength > kTraitAutoCapacity) - { - traits.SetCapacity(newLength); - percents.SetCapacity(newLength); - } - traits.AppendElements(newClassifications); - for (uint32_t index = 0; index < newLength; index++) - percents.AppendElement(100); // This is 100 percent, or certainty - aTraitListener->OnMessageTraitsClassified(messageURL, - traits.Length(), traits.Elements(), percents.Elements()); + if (aTraitListener) { + // construct the outgoing listener arrays + AutoTArray traits; + AutoTArray percents; + uint32_t newLength = newClassifications.Length(); + if (newLength > kTraitAutoCapacity) { + traits.SetCapacity(newLength); + percents.SetCapacity(newLength); } + traits.AppendElements(newClassifications); + for (uint32_t index = 0; index < newLength; index++) + percents.AppendElement(100); // This is 100 percent, or certainty + aTraitListener->OnMessageTraitsClassified( + messageURL, traits.Length(), traits.Elements(), percents.Elements()); + } - if (mTrainingDataDirty && !trainingDataWasDirty && ( mTimer != nullptr )) - { - // if training data became dirty just now, schedule flush - // mMinFlushInterval msec from now - MOZ_LOG( - BayesianFilterLogModule, LogLevel::Debug, + if (mTrainingDataDirty && !trainingDataWasDirty && (mTimer != nullptr)) { + // if training data became dirty just now, schedule flush + // mMinFlushInterval msec from now + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, ("starting training data flush timer %i msec", mMinFlushInterval)); - mTimer->InitWithNamedFuncCallback(nsBayesianFilter::TimerCallback, - this, mMinFlushInterval, - nsITimer::TYPE_ONE_SHOT, - "nsBayesianFilter::TimerCallback"); - } + mTimer->InitWithNamedFuncCallback( + nsBayesianFilter::TimerCallback, this, mMinFlushInterval, + nsITimer::TYPE_ONE_SHOT, "nsBayesianFilter::TimerCallback"); + } } -NS_IMETHODIMP nsBayesianFilter::GetUserHasClassified(bool *aResult) -{ - *aResult = ( (mCorpus.getMessageCount(kGoodTrait) + - mCorpus.getMessageCount(kJunkTrait)) && - mCorpus.countTokens()); +NS_IMETHODIMP nsBayesianFilter::GetUserHasClassified(bool* aResult) { + *aResult = ((mCorpus.getMessageCount(kGoodTrait) + + mCorpus.getMessageCount(kJunkTrait)) && + mCorpus.countTokens()); return NS_OK; } // Set message classification (only allows junk and good) NS_IMETHODIMP nsBayesianFilter::SetMessageClassification( - const char *aMsgURL, - nsMsgJunkStatus aOldClassification, - nsMsgJunkStatus aNewClassification, - nsIMsgWindow *aMsgWindow, - nsIJunkMailClassificationListener *aListener) -{ + const char* aMsgURL, nsMsgJunkStatus aOldClassification, + nsMsgJunkStatus aNewClassification, nsIMsgWindow* aMsgWindow, + nsIJunkMailClassificationListener* aListener) { AutoTArray oldClassifications; AutoTArray newClassifications; @@ -2122,36 +1984,35 @@ NS_IMETHODIMP nsBayesianFilter::SetMessageClassification( else if (aNewClassification == nsIJunkMailPlugin::GOOD) newClassifications.AppendElement(kGoodTrait); - MessageObserver* analyzer = new MessageObserver(this, oldClassifications, - newClassifications, aListener, nullptr); + MessageObserver* analyzer = new MessageObserver( + this, oldClassifications, newClassifications, aListener, nullptr); NS_ENSURE_TRUE(analyzer, NS_ERROR_OUT_OF_MEMORY); - TokenStreamListener *tokenListener = new TokenStreamListener(analyzer); + TokenStreamListener* tokenListener = new TokenStreamListener(analyzer); NS_ENSURE_TRUE(tokenListener, NS_ERROR_OUT_OF_MEMORY); analyzer->setTokenListener(tokenListener); return tokenizeMessage(aMsgURL, aMsgWindow, analyzer); } -NS_IMETHODIMP nsBayesianFilter::ResetTrainingData() -{ +NS_IMETHODIMP nsBayesianFilter::ResetTrainingData() { return mCorpus.resetTrainingData(); } -NS_IMETHODIMP nsBayesianFilter::DetailMessage(const char *aMsgURI, - uint32_t aProTrait, uint32_t aAntiTrait, - nsIMsgTraitDetailListener *aDetailListener, nsIMsgWindow *aMsgWindow) -{ +NS_IMETHODIMP nsBayesianFilter::DetailMessage( + const char* aMsgURI, uint32_t aProTrait, uint32_t aAntiTrait, + nsIMsgTraitDetailListener* aDetailListener, nsIMsgWindow* aMsgWindow) { AutoTArray proTraits; AutoTArray antiTraits; proTraits.AppendElement(aProTrait); antiTraits.AppendElement(aAntiTrait); - MessageClassifier* analyzer = new MessageClassifier(this, nullptr, - nullptr, aDetailListener, proTraits, antiTraits, aMsgWindow, 1, &aMsgURI); + MessageClassifier* analyzer = + new MessageClassifier(this, nullptr, nullptr, aDetailListener, proTraits, + antiTraits, aMsgWindow, 1, &aMsgURI); NS_ENSURE_TRUE(analyzer, NS_ERROR_OUT_OF_MEMORY); - TokenStreamListener *tokenListener = new TokenStreamListener(analyzer); + TokenStreamListener* tokenListener = new TokenStreamListener(analyzer); NS_ENSURE_TRUE(tokenListener, NS_ERROR_OUT_OF_MEMORY); analyzer->setTokenListener(tokenListener); @@ -2161,36 +2022,27 @@ NS_IMETHODIMP nsBayesianFilter::DetailMessage(const char *aMsgURI, // nsIMsgCorpus implementation NS_IMETHODIMP nsBayesianFilter::CorpusCounts(uint32_t aTrait, - uint32_t *aMessageCount, - uint32_t *aTokenCount) -{ + uint32_t* aMessageCount, + uint32_t* aTokenCount) { NS_ENSURE_ARG_POINTER(aTokenCount); *aTokenCount = mCorpus.countTokens(); - if (aTrait && aMessageCount) - *aMessageCount = mCorpus.getMessageCount(aTrait); + if (aTrait && aMessageCount) *aMessageCount = mCorpus.getMessageCount(aTrait); return NS_OK; } -NS_IMETHODIMP nsBayesianFilter::ClearTrait(uint32_t aTrait) -{ - return mCorpus.ClearTrait(aTrait); +NS_IMETHODIMP nsBayesianFilter::ClearTrait(uint32_t aTrait) { + return mCorpus.ClearTrait(aTrait); } NS_IMETHODIMP -nsBayesianFilter::UpdateData(nsIFile *aFile, - bool aIsAdd, - uint32_t aRemapCount, - uint32_t *aFromTraits, - uint32_t *aToTraits) -{ +nsBayesianFilter::UpdateData(nsIFile* aFile, bool aIsAdd, uint32_t aRemapCount, + uint32_t* aFromTraits, uint32_t* aToTraits) { return mCorpus.UpdateData(aFile, aIsAdd, aRemapCount, aFromTraits, aToTraits); } NS_IMETHODIMP -nsBayesianFilter::GetTokenCount(const nsACString &aWord, - uint32_t aTrait, - uint32_t *aCount) -{ +nsBayesianFilter::GetTokenCount(const nsACString& aWord, uint32_t aTrait, + uint32_t* aCount) { NS_ENSURE_ARG_POINTER(aCount); CorpusToken* t = mCorpus.get(PromiseFlatCString(aWord).get()); uint32_t count = mCorpus.getTraitCount(t, aTrait); @@ -2221,59 +2073,50 @@ nsBayesianFilter::GetTokenCount(const nsACString &aWord, [length of word]word */ -CorpusStore::CorpusStore() : - TokenHash(sizeof(CorpusToken)), - mNextTraitIndex(1) // skip 0 since index=0 will mean end of linked list +CorpusStore::CorpusStore() + : TokenHash(sizeof(CorpusToken)), + mNextTraitIndex(1) // skip 0 since index=0 will mean end of linked list { getTrainingFile(getter_AddRefs(mTrainingFile)); mTraitStore.SetCapacity(kTraitStoreCapacity); TraitPerToken traitPT(0, 0); - mTraitStore.AppendElement(traitPT); // dummy 0th element + mTraitStore.AppendElement(traitPT); // dummy 0th element } -CorpusStore::~CorpusStore() -{ +CorpusStore::~CorpusStore() {} + +inline int writeUInt32(FILE* stream, uint32_t value) { + value = PR_htonl(value); + return fwrite(&value, sizeof(uint32_t), 1, stream); } -inline int writeUInt32(FILE* stream, uint32_t value) -{ - value = PR_htonl(value); - return fwrite(&value, sizeof(uint32_t), 1, stream); +inline int readUInt32(FILE* stream, uint32_t* value) { + int n = fread(value, sizeof(uint32_t), 1, stream); + if (n == 1) { + *value = PR_ntohl(*value); + } + return n; } -inline int readUInt32(FILE* stream, uint32_t* value) -{ - int n = fread(value, sizeof(uint32_t), 1, stream); - if (n == 1) { - *value = PR_ntohl(*value); - } - return n; -} - -void CorpusStore::forgetTokens(Tokenizer& aTokenizer, - uint32_t aTraitId, uint32_t aCount) -{ +void CorpusStore::forgetTokens(Tokenizer& aTokenizer, uint32_t aTraitId, + uint32_t aCount) { // if we are forgetting the tokens for a message, should only // subtract 1 from the occurrence count for that token in the training set // because we assume we only bumped the training set count once per messages // containing the token. TokenEnumeration tokens = aTokenizer.getTokens(); - while (tokens.hasMoreTokens()) - { + while (tokens.hasMoreTokens()) { CorpusToken* token = static_cast(tokens.nextToken()); remove(token->mWord, aTraitId, aCount); } } -void CorpusStore::rememberTokens(Tokenizer& aTokenizer, - uint32_t aTraitId, uint32_t aCount) -{ +void CorpusStore::rememberTokens(Tokenizer& aTokenizer, uint32_t aTraitId, + uint32_t aCount) { TokenEnumeration tokens = aTokenizer.getTokens(); - while (tokens.hasMoreTokens()) - { + while (tokens.hasMoreTokens()) { CorpusToken* token = static_cast(tokens.nextToken()); - if (!token) - { + if (!token) { NS_ERROR("null token"); continue; } @@ -2281,139 +2124,121 @@ void CorpusStore::rememberTokens(Tokenizer& aTokenizer, } } -bool CorpusStore::writeTokens(FILE* stream, bool shrink, uint32_t aTraitId) -{ +bool CorpusStore::writeTokens(FILE* stream, bool shrink, uint32_t aTraitId) { uint32_t tokenCount = countTokens(); uint32_t newTokenCount = 0; // calculate the tokens for this trait to write TokenEnumeration tokens = getTokens(); - for (uint32_t i = 0; i < tokenCount; ++i) - { + for (uint32_t i = 0; i < tokenCount; ++i) { CorpusToken* token = static_cast(tokens.nextToken()); uint32_t count = getTraitCount(token, aTraitId); - // Shrinking the token database is accomplished by dividing all token counts by 2. - // If shrinking, we'll ignore counts < 2, otherwise only ignore counts of < 1 - if ((shrink && count > 1) || (!shrink && count)) - newTokenCount++; + // Shrinking the token database is accomplished by dividing all token counts + // by 2. If shrinking, we'll ignore counts < 2, otherwise only ignore counts + // of < 1 + if ((shrink && count > 1) || (!shrink && count)) newTokenCount++; } - if (writeUInt32(stream, newTokenCount) != 1) - return false; + if (writeUInt32(stream, newTokenCount) != 1) return false; - if (newTokenCount > 0) - { + if (newTokenCount > 0) { TokenEnumeration tokens = getTokens(); - for (uint32_t i = 0; i < tokenCount; ++i) - { + for (uint32_t i = 0; i < tokenCount; ++i) { CorpusToken* token = static_cast(tokens.nextToken()); uint32_t wordCount = getTraitCount(token, aTraitId); - if (shrink) - wordCount /= 2; - if (!wordCount) - continue; // Don't output zero count words - if (writeUInt32(stream, wordCount) != 1) - return false; + if (shrink) wordCount /= 2; + if (!wordCount) continue; // Don't output zero count words + if (writeUInt32(stream, wordCount) != 1) return false; uint32_t tokenLength = strlen(token->mWord); - if (writeUInt32(stream, tokenLength) != 1) - return false; - if (fwrite(token->mWord, tokenLength, 1, stream) != 1) - return false; + if (writeUInt32(stream, tokenLength) != 1) return false; + if (fwrite(token->mWord, tokenLength, 1, stream) != 1) return false; } } return true; } -bool CorpusStore::readTokens(FILE* stream, int64_t fileSize, - uint32_t aTraitId, bool aIsAdd) -{ - uint32_t tokenCount; - if (readUInt32(stream, &tokenCount) != 1) - return false; +bool CorpusStore::readTokens(FILE* stream, int64_t fileSize, uint32_t aTraitId, + bool aIsAdd) { + uint32_t tokenCount; + if (readUInt32(stream, &tokenCount) != 1) return false; - int64_t fpos = ftell(stream); - if (fpos < 0) - return false; + int64_t fpos = ftell(stream); + if (fpos < 0) return false; - uint32_t bufferSize = 4096; - char* buffer = new char[bufferSize]; - if (!buffer) return false; + uint32_t bufferSize = 4096; + char* buffer = new char[bufferSize]; + if (!buffer) return false; - for (uint32_t i = 0; i < tokenCount; ++i) { - uint32_t count; - if (readUInt32(stream, &count) != 1) - break; - uint32_t size; - if (readUInt32(stream, &size) != 1) - break; - fpos += 8; - if (fpos + size > fileSize) { - delete[] buffer; - return false; - } - if (size >= bufferSize) { - delete[] buffer; - while (size >= bufferSize) { - bufferSize *= 2; - if (bufferSize == 0) - return false; - } - buffer = new char[bufferSize]; - if (!buffer) return false; - } - if (fread(buffer, size, 1, stream) != 1) - break; - fpos += size; - buffer[size] = '\0'; - if (aIsAdd) - add(buffer, aTraitId, count); - else - remove(buffer, aTraitId, count); + for (uint32_t i = 0; i < tokenCount; ++i) { + uint32_t count; + if (readUInt32(stream, &count) != 1) break; + uint32_t size; + if (readUInt32(stream, &size) != 1) break; + fpos += 8; + if (fpos + size > fileSize) { + delete[] buffer; + return false; } + if (size >= bufferSize) { + delete[] buffer; + while (size >= bufferSize) { + bufferSize *= 2; + if (bufferSize == 0) return false; + } + buffer = new char[bufferSize]; + if (!buffer) return false; + } + if (fread(buffer, size, 1, stream) != 1) break; + fpos += size; + buffer[size] = '\0'; + if (aIsAdd) + add(buffer, aTraitId, count); + else + remove(buffer, aTraitId, count); + } - delete[] buffer; + delete[] buffer; - return true; + return true; } -nsresult CorpusStore::getTrainingFile(nsIFile ** aTrainingFile) -{ +nsresult CorpusStore::getTrainingFile(nsIFile** aTrainingFile) { // should we cache the profile manager's directory? nsCOMPtr profileDir; - nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(profileDir)); + nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, + getter_AddRefs(profileDir)); NS_ENSURE_SUCCESS(rv, rv); rv = profileDir->Append(NS_LITERAL_STRING("training.dat")); NS_ENSURE_SUCCESS(rv, rv); - return profileDir->QueryInterface(NS_GET_IID(nsIFile), (void **) aTrainingFile); + return profileDir->QueryInterface(NS_GET_IID(nsIFile), (void**)aTrainingFile); } -nsresult CorpusStore::getTraitFile(nsIFile ** aTraitFile) -{ +nsresult CorpusStore::getTraitFile(nsIFile** aTraitFile) { // should we cache the profile manager's directory? nsCOMPtr profileDir; - nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(profileDir)); + nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, + getter_AddRefs(profileDir)); NS_ENSURE_SUCCESS(rv, rv); rv = profileDir->Append(NS_LITERAL_STRING("traits.dat")); NS_ENSURE_SUCCESS(rv, rv); - return profileDir->QueryInterface(NS_GET_IID(nsIFile), (void **) aTraitFile); + return profileDir->QueryInterface(NS_GET_IID(nsIFile), (void**)aTraitFile); } -static const char kMagicCookie[] = { '\xFE', '\xED', '\xFA', '\xCE' }; +static const char kMagicCookie[] = {'\xFE', '\xED', '\xFA', '\xCE'}; // random string used to identify trait file and version (last byte is version) -static const char kTraitCookie[] = { '\xFC', '\xA9', '\x36', '\x01' }; +static const char kTraitCookie[] = {'\xFC', '\xA9', '\x36', '\x01'}; -void CorpusStore::writeTrainingData(uint32_t aMaximumTokenCount) -{ - MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, ("writeTrainingData() entered")); - if (!mTrainingFile) - return; +void CorpusStore::writeTrainingData(uint32_t aMaximumTokenCount) { + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, + ("writeTrainingData() entered")); + if (!mTrainingFile) return; /* * For backwards compatibility, write the good and junk tokens to @@ -2423,34 +2248,30 @@ void CorpusStore::writeTrainingData(uint32_t aMaximumTokenCount) // open the file, and write out training data FILE* stream; nsresult rv = mTrainingFile->OpenANSIFileDesc("wb", &stream); - if (NS_FAILED(rv)) - return; + if (NS_FAILED(rv)) return; // If the number of tokens exceeds our limit, set the shrink flag bool shrink = false; - if ((aMaximumTokenCount > 0) && // if 0, do not limit tokens - (countTokens() > aMaximumTokenCount)) - { + if ((aMaximumTokenCount > 0) && // if 0, do not limit tokens + (countTokens() > aMaximumTokenCount)) { shrink = true; - MOZ_LOG(BayesianFilterLogModule, LogLevel::Warning, ("shrinking token data file")); + MOZ_LOG(BayesianFilterLogModule, LogLevel::Warning, + ("shrinking token data file")); } // We implement shrink by dividing counts by two uint32_t shrinkFactor = shrink ? 2 : 1; if (!((fwrite(kMagicCookie, sizeof(kMagicCookie), 1, stream) == 1) && - (writeUInt32(stream, getMessageCount(kGoodTrait) / shrinkFactor)) && - (writeUInt32(stream, getMessageCount(kJunkTrait) / shrinkFactor)) && - writeTokens(stream, shrink, kGoodTrait) && - writeTokens(stream, shrink, kJunkTrait))) - { + (writeUInt32(stream, getMessageCount(kGoodTrait) / shrinkFactor)) && + (writeUInt32(stream, getMessageCount(kJunkTrait) / shrinkFactor)) && + writeTokens(stream, shrink, kGoodTrait) && + writeTokens(stream, shrink, kJunkTrait))) { NS_WARNING("failed to write training data."); fclose(stream); // delete the training data file, since it is potentially corrupt. mTrainingFile->Remove(false); - } - else - { + } else { fclose(stream); } @@ -2458,36 +2279,31 @@ void CorpusStore::writeTrainingData(uint32_t aMaximumTokenCount) * Write the remaining data to a second file traits.dat */ - if (!mTraitFile) - { + if (!mTraitFile) { getTraitFile(getter_AddRefs(mTraitFile)); - if (!mTraitFile) - return; + if (!mTraitFile) return; } // open the file, and write out training data rv = mTraitFile->OpenANSIFileDesc("wb", &stream); - if (NS_FAILED(rv)) - return; + if (NS_FAILED(rv)) return; uint32_t numberOfTraits = mMessageCounts.Length(); bool error; - while (1) // break on error or done + while (1) // break on error or done { if ((error = (fwrite(kTraitCookie, sizeof(kTraitCookie), 1, stream) != 1))) break; - for (uint32_t index = 0; index < numberOfTraits; index++) - { + for (uint32_t index = 0; index < numberOfTraits; index++) { uint32_t trait = mMessageCountsId[index]; if (trait == 1 || trait == 2) - continue; // junk traits are stored in training.dat - if ((error = (writeUInt32(stream, trait) != 1))) - break; - if ((error = (writeUInt32(stream, mMessageCounts[index] / shrinkFactor) != 1))) - break; - if ((error = !writeTokens(stream, shrink, trait))) + continue; // junk traits are stored in training.dat + if ((error = (writeUInt32(stream, trait) != 1))) break; + if ((error = (writeUInt32(stream, mMessageCounts[index] / shrinkFactor) != + 1))) break; + if ((error = !writeTokens(stream, shrink, trait))) break; } break; } @@ -2495,53 +2311,44 @@ void CorpusStore::writeTrainingData(uint32_t aMaximumTokenCount) error = writeUInt32(stream, 0) != 1; fclose(stream); - if (error) - { + if (error) { NS_WARNING("failed to write trait data."); // delete the trait data file, since it is probably corrupt. mTraitFile->Remove(false); } - if (shrink) - { + if (shrink) { // We'll clear the tokens, and read them back in from the file. // Yes this is slower than in place, but this is a rare event. - if (countTokens()) - { + if (countTokens()) { clearTokens(); for (uint32_t index = 0; index < numberOfTraits; index++) mMessageCounts[index] = 0; } - readTrainingData(); + readTrainingData(); } } -void CorpusStore::readTrainingData() -{ - +void CorpusStore::readTrainingData() { /* * To maintain backwards compatibility, good and junk traits * are stored in a file "training.dat" */ - if (!mTrainingFile) - return; + if (!mTrainingFile) return; bool exists; nsresult rv = mTrainingFile->Exists(&exists); - if (NS_FAILED(rv) || !exists) - return; + if (NS_FAILED(rv) || !exists) return; FILE* stream; rv = mTrainingFile->OpenANSIFileDesc("rb", &stream); - if (NS_FAILED(rv)) - return; + if (NS_FAILED(rv)) return; int64_t fileSize; rv = mTrainingFile->GetFileSize(&fileSize); - if (NS_FAILED(rv)) - return; + if (NS_FAILED(rv)) return; // FIXME: should make sure that the tokenizers are empty. char cookie[4]; @@ -2550,10 +2357,11 @@ void CorpusStore::readTrainingData() (memcmp(cookie, kMagicCookie, sizeof(cookie)) == 0) && (readUInt32(stream, &goodMessageCount) == 1) && (readUInt32(stream, &junkMessageCount) == 1) && - readTokens(stream, fileSize, kGoodTrait, true) && - readTokens(stream, fileSize, kJunkTrait, true))) { - NS_WARNING("failed to read training data."); - MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, ("failed to read training data.")); + readTokens(stream, fileSize, kGoodTrait, true) && + readTokens(stream, fileSize, kJunkTrait, true))) { + NS_WARNING("failed to read training data."); + MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, + ("failed to read training data.")); } setMessageCount(kGoodTrait, goodMessageCount); setMessageCount(kJunkTrait, junkMessageCount); @@ -2564,79 +2372,67 @@ void CorpusStore::readTrainingData() * Additional traits are stored in traits.dat */ - if (!mTraitFile) - { + if (!mTraitFile) { getTraitFile(getter_AddRefs(mTraitFile)); - if (!mTraitFile) - return; + if (!mTraitFile) return; } rv = mTraitFile->Exists(&exists); - if (NS_FAILED(rv) || !exists) - return; + if (NS_FAILED(rv) || !exists) return; rv = UpdateData(mTraitFile, true, 0, nullptr, nullptr); - if (NS_FAILED(rv)) - { + if (NS_FAILED(rv)) { NS_WARNING("failed to read training data."); - MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, ("failed to read training data.")); + MOZ_LOG(BayesianFilterLogModule, LogLevel::Error, + ("failed to read training data.")); } return; } -nsresult CorpusStore::resetTrainingData() -{ +nsresult CorpusStore::resetTrainingData() { // clear out our in memory training tokens... - if (countTokens()) - clearTokens(); + if (countTokens()) clearTokens(); uint32_t length = mMessageCounts.Length(); - for (uint32_t index = 0 ; index < length; index++) - mMessageCounts[index] = 0; + for (uint32_t index = 0; index < length; index++) mMessageCounts[index] = 0; - if (mTrainingFile) - mTrainingFile->Remove(false); - if (mTraitFile) - mTraitFile->Remove(false); + if (mTrainingFile) mTrainingFile->Remove(false); + if (mTraitFile) mTraitFile->Remove(false); return NS_OK; } -inline CorpusToken* CorpusStore::get(const char* word) -{ +inline CorpusToken* CorpusStore::get(const char* word) { return static_cast(TokenHash::get(word)); } nsresult CorpusStore::updateTrait(CorpusToken* token, uint32_t aTraitId, - int32_t aCountChange) -{ + int32_t aCountChange) { NS_ENSURE_ARG_POINTER(token); uint32_t nextLink = token->mTraitLink; uint32_t lastLink = 0; - uint32_t linkCount, maxLinks = 100; //sanity check - for (linkCount = 0; nextLink && linkCount < maxLinks; linkCount++) - { + uint32_t linkCount, maxLinks = 100; // sanity check + for (linkCount = 0; nextLink && linkCount < maxLinks; linkCount++) { TraitPerToken& traitPT = mTraitStore[nextLink]; - if (traitPT.mId == aTraitId) - { + if (traitPT.mId == aTraitId) { // be careful with signed versus unsigned issues here if (static_cast(traitPT.mCount) + aCountChange > 0) traitPT.mCount += aCountChange; else traitPT.mCount = 0; - // we could delete zero count traits here, but let's not. It's rare anyway. + // we could delete zero count traits here, but let's not. It's rare + // anyway. return NS_OK; } lastLink = nextLink; nextLink = traitPT.mNextLink; } - if (linkCount >= maxLinks) - return NS_ERROR_FAILURE; + if (linkCount >= maxLinks) return NS_ERROR_FAILURE; // trait does not exist, so add it - if (aCountChange > 0) // don't set a negative count + if (aCountChange > 0) // don't set a negative count { TraitPerToken traitPT(aTraitId, aCountChange); if (mTraitStore.Length() == mNextTraitIndex) @@ -2656,18 +2452,14 @@ nsresult CorpusStore::updateTrait(CorpusToken* token, uint32_t aTraitId, return NS_OK; } -uint32_t CorpusStore::getTraitCount(CorpusToken* token, uint32_t aTraitId) -{ +uint32_t CorpusStore::getTraitCount(CorpusToken* token, uint32_t aTraitId) { uint32_t nextLink; - if (!token || !(nextLink = token->mTraitLink)) - return 0; + if (!token || !(nextLink = token->mTraitLink)) return 0; - uint32_t linkCount, maxLinks = 100; //sanity check - for (linkCount = 0; nextLink && linkCount < maxLinks; linkCount++) - { + uint32_t linkCount, maxLinks = 100; // sanity check + for (linkCount = 0; nextLink && linkCount < maxLinks; linkCount++) { TraitPerToken& traitPT = mTraitStore[nextLink]; - if (traitPT.mId == aTraitId) - return traitPT.mCount; + if (traitPT.mId == aTraitId) return traitPT.mCount; nextLink = traitPT.mNextLink; } NS_ASSERTION(linkCount < maxLinks, "Corrupt trait count store"); @@ -2676,60 +2468,46 @@ uint32_t CorpusStore::getTraitCount(CorpusToken* token, uint32_t aTraitId) return 0; } -CorpusToken* CorpusStore::add(const char* word, uint32_t aTraitId, uint32_t aCount) -{ +CorpusToken* CorpusStore::add(const char* word, uint32_t aTraitId, + uint32_t aCount) { CorpusToken* token = static_cast(TokenHash::add(word)); if (token) { MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, - ("adding word to corpus store: %s (Trait=%d) (deltaCount=%d)", - word, aTraitId, aCount)); + ("adding word to corpus store: %s (Trait=%d) (deltaCount=%d)", word, + aTraitId, aCount)); updateTrait(token, aTraitId, aCount); } return token; - } - -void CorpusStore::remove(const char* word, uint32_t aTraitId, uint32_t aCount) -{ - MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, - ("remove word: %s (TraitId=%d) (Count=%d)", - word, aTraitId, aCount)); - CorpusToken* token = get(word); - if (token) - updateTrait(token, aTraitId, -static_cast(aCount)); } -uint32_t CorpusStore::getMessageCount(uint32_t aTraitId) -{ +void CorpusStore::remove(const char* word, uint32_t aTraitId, uint32_t aCount) { + MOZ_LOG(BayesianFilterLogModule, LogLevel::Debug, + ("remove word: %s (TraitId=%d) (Count=%d)", word, aTraitId, aCount)); + CorpusToken* token = get(word); + if (token) updateTrait(token, aTraitId, -static_cast(aCount)); +} + +uint32_t CorpusStore::getMessageCount(uint32_t aTraitId) { size_t index = mMessageCountsId.IndexOf(aTraitId); - if (index == mMessageCountsId.NoIndex) - return 0; + if (index == mMessageCountsId.NoIndex) return 0; return mMessageCounts.ElementAt(index); } -void CorpusStore::setMessageCount(uint32_t aTraitId, uint32_t aCount) -{ +void CorpusStore::setMessageCount(uint32_t aTraitId, uint32_t aCount) { size_t index = mMessageCountsId.IndexOf(aTraitId); - if (index == mMessageCountsId.NoIndex) - { + if (index == mMessageCountsId.NoIndex) { mMessageCounts.AppendElement(aCount); mMessageCountsId.AppendElement(aTraitId); - } - else - { + } else { mMessageCounts[index] = aCount; } } -nsresult -CorpusStore::UpdateData(nsIFile *aFile, - bool aIsAdd, - uint32_t aRemapCount, - uint32_t *aFromTraits, - uint32_t *aToTraits) -{ +nsresult CorpusStore::UpdateData(nsIFile* aFile, bool aIsAdd, + uint32_t aRemapCount, uint32_t* aFromTraits, + uint32_t* aToTraits) { NS_ENSURE_ARG_POINTER(aFile); - if (aRemapCount) - { + if (aRemapCount) { NS_ENSURE_ARG_POINTER(aFromTraits); NS_ENSURE_ARG_POINTER(aToTraits); } @@ -2743,28 +2521,22 @@ CorpusStore::UpdateData(nsIFile *aFile, NS_ENSURE_SUCCESS(rv, rv); bool error; - do // break on error or done + do // break on error or done { char cookie[4]; - if ((error = (fread(cookie, sizeof(cookie), 1, stream) != 1))) - break; + if ((error = (fread(cookie, sizeof(cookie), 1, stream) != 1))) break; - if ((error = memcmp(cookie, kTraitCookie, sizeof(cookie)))) - break; + if ((error = memcmp(cookie, kTraitCookie, sizeof(cookie)))) break; uint32_t fileTrait; - while ( !(error = (readUInt32(stream, &fileTrait) != 1)) && fileTrait) - { + while (!(error = (readUInt32(stream, &fileTrait) != 1)) && fileTrait) { uint32_t count; - if ((error = (readUInt32(stream, &count) != 1))) - break; + if ((error = (readUInt32(stream, &count) != 1))) break; uint32_t localTrait = fileTrait; // remap the trait - for (uint32_t i = 0; i < aRemapCount; i++) - { - if (aFromTraits[i] == fileTrait) - localTrait = aToTraits[i]; + for (uint32_t i = 0; i < aRemapCount; i++) { + if (aFromTraits[i] == fileTrait) localTrait = aToTraits[i]; } uint32_t messageCount = getMessageCount(localTrait); @@ -2776,27 +2548,23 @@ CorpusStore::UpdateData(nsIFile *aFile, messageCount -= count; setMessageCount(localTrait, messageCount); - if ((error = !readTokens(stream, fileSize, localTrait, aIsAdd))) - break; + if ((error = !readTokens(stream, fileSize, localTrait, aIsAdd))) break; } break; } while (0); fclose(stream); - if (error) - return NS_ERROR_FAILURE; + if (error) return NS_ERROR_FAILURE; return NS_OK; } -nsresult CorpusStore::ClearTrait(uint32_t aTrait) -{ +nsresult CorpusStore::ClearTrait(uint32_t aTrait) { // clear message counts setMessageCount(aTrait, 0); TokenEnumeration tokens = getTokens(); - while (tokens.hasMoreTokens()) - { + while (tokens.hasMoreTokens()) { CorpusToken* token = static_cast(tokens.nextToken()); int32_t wordCount = static_cast(getTraitCount(token, aTrait)); updateTrait(token, aTrait, -wordCount); diff --git a/mailnews/extensions/bayesian-spam-filter/src/nsBayesianFilter.h b/mailnews/extensions/bayesian-spam-filter/src/nsBayesianFilter.h index 336154ec30..77d86ad161 100644 --- a/mailnews/extensions/bayesian-spam-filter/src/nsBayesianFilter.h +++ b/mailnews/extensions/bayesian-spam-filter/src/nsBayesianFilter.h @@ -19,7 +19,7 @@ #include "mozilla/ArenaAllocator.h" -#define DEFAULT_MIN_INTERVAL_BETWEEN_WRITES 15*60*1000 +#define DEFAULT_MIN_INTERVAL_BETWEEN_WRITES 15 * 60 * 1000 struct Token; class TokenEnumeration; @@ -36,13 +36,13 @@ struct CorpusToken; * or Remove() is performed on the table. */ class TokenEnumeration { -public: - explicit TokenEnumeration(PLDHashTable* table); - bool hasMoreTokens(); - BaseToken* nextToken(); + public: + explicit TokenEnumeration(PLDHashTable* table); + bool hasMoreTokens(); + BaseToken* nextToken(); -private: - PLDHashTable::Iterator mIterator; + private: + PLDHashTable::Iterator mIterator; }; // A trait is some aspect of a message, like being junk or tagged as @@ -52,13 +52,12 @@ private: // // Traits per token are maintained as a linked list. // -struct TraitPerToken -{ - uint32_t mId; // identifying number for a trait - uint32_t mCount; // count of messages with this token and trait - uint32_t mNextLink; // index in mTraitStore for the next trait, or 0 - // for none - TraitPerToken(uint32_t aId, uint32_t aCount); // inititializer +struct TraitPerToken { + uint32_t mId; // identifying number for a trait + uint32_t mCount; // count of messages with this token and trait + uint32_t mNextLink; // index in mTraitStore for the next trait, or 0 + // for none + TraitPerToken(uint32_t aId, uint32_t aCount); // inititializer }; // An Analysis is the statistical results for a particular message, a @@ -67,90 +66,93 @@ struct TraitPerToken // // Analyses per token are maintained as a linked list. // -struct AnalysisPerToken -{ - uint32_t mTraitIndex; // index representing a protrait/antitrait pair. - // So if we are analyzing 3 different traits, then - // the first trait is 0, the second 1, etc. - double mDistance; // absolute value of mProbability - 0.5 - double mProbability; // relative indicator of match of trait to token - uint32_t mNextLink; // index in mAnalysisStore for the Analysis object - // for the next trait index, or 0 for none. +struct AnalysisPerToken { + uint32_t mTraitIndex; // index representing a protrait/antitrait pair. + // So if we are analyzing 3 different traits, then + // the first trait is 0, the second 1, etc. + double mDistance; // absolute value of mProbability - 0.5 + double mProbability; // relative indicator of match of trait to token + uint32_t mNextLink; // index in mAnalysisStore for the Analysis object + // for the next trait index, or 0 for none. // initializer AnalysisPerToken(uint32_t aTraitIndex, double aDistance, double aProbability); }; class TokenHash { -public: + public: + virtual ~TokenHash(); + /** + * Clears out the previous message tokens. + */ + nsresult clearTokens(); + uint32_t countTokens(); + TokenEnumeration getTokens(); + BaseToken* add(const char* word); - virtual ~TokenHash(); - /** - * Clears out the previous message tokens. - */ - nsresult clearTokens(); - uint32_t countTokens(); - TokenEnumeration getTokens(); - BaseToken* add(const char* word); - -protected: - explicit TokenHash(uint32_t entrySize); - mozilla::ArenaAllocator<16384, 2> mWordPool; - uint32_t mEntrySize; - PLDHashTable mTokenTable; - char* copyWord(const char* word, uint32_t len); - BaseToken* get(const char* word); + protected: + explicit TokenHash(uint32_t entrySize); + mozilla::ArenaAllocator<16384, 2> mWordPool; + uint32_t mEntrySize; + PLDHashTable mTokenTable; + char* copyWord(const char* word, uint32_t len); + BaseToken* get(const char* word); }; -class Tokenizer: public TokenHash { -public: - Tokenizer(); - ~Tokenizer(); +class Tokenizer : public TokenHash { + public: + Tokenizer(); + ~Tokenizer(); - Token* get(const char* word); + Token* get(const char* word); - // The training set keeps an occurrence count on each word. This count - // is supposed to count the # of messages it occurs in. - // When add/remove is called while tokenizing a message and NOT the training set, - // - Token* add(const char* word, uint32_t count = 1); + // The training set keeps an occurrence count on each word. This count + // is supposed to count the # of messages it occurs in. + // When add/remove is called while tokenizing a message and NOT the training + // set, + // + Token* add(const char* word, uint32_t count = 1); - Token* copyTokens(); + Token* copyTokens(); - void tokenize(const char* text); + void tokenize(const char* text); - /** - * Creates specific tokens based on the mime headers for the message being tokenized - */ - void tokenizeHeaders(nsIUTF8StringEnumerator * aHeaderNames, nsIUTF8StringEnumerator * aHeaderValues); + /** + * Creates specific tokens based on the mime headers for the message being + * tokenized + */ + void tokenizeHeaders(nsIUTF8StringEnumerator* aHeaderNames, + nsIUTF8StringEnumerator* aHeaderValues); - void tokenizeAttachment(const char * aContentType, const char * aFileName); + void tokenizeAttachment(const char* aContentType, const char* aFileName); - nsCString mBodyDelimiters; // delimiters for body tokenization - nsCString mHeaderDelimiters; // delimiters for header tokenization + nsCString mBodyDelimiters; // delimiters for body tokenization + nsCString mHeaderDelimiters; // delimiters for header tokenization - // arrays of extra headers to tokenize / to not tokenize - nsTArray mEnabledHeaders; - nsTArray mDisabledHeaders; - // Delimiters used in tokenizing a particular header. - // Parallel array to mEnabledHeaders - nsTArray mEnabledHeadersDelimiters; - bool mCustomHeaderTokenization; // Are there any preference-set tokenization customizations? - uint32_t mMaxLengthForToken; // maximum length of a token - // should we convert iframe to div during tokenization? - bool mIframeToDiv; + // arrays of extra headers to tokenize / to not tokenize + nsTArray mEnabledHeaders; + nsTArray mDisabledHeaders; + // Delimiters used in tokenizing a particular header. + // Parallel array to mEnabledHeaders + nsTArray mEnabledHeadersDelimiters; + bool mCustomHeaderTokenization; // Are there any preference-set tokenization + // customizations? + uint32_t mMaxLengthForToken; // maximum length of a token + // should we convert iframe to div during tokenization? + bool mIframeToDiv; -private: - - void tokenize_ascii_word(char * word); - void tokenize_japanese_word(char* chunk); - inline void addTokenForHeader(const char * aTokenPrefix, nsACString& aValue, - bool aTokenizeValue = false, const char* aDelimiters = nullptr); - nsresult stripHTML(const nsAString& inString, nsAString& outString); - // helper function to escape \n, \t, etc from a CString - void UnescapeCString(nsCString& aCString); - nsresult ScannerNext(const char16_t *text, int32_t length, int32_t pos, - bool isLastBuffer, int32_t *begin, int32_t *end, bool *_retval); - RefPtr mWordBreaker; + private: + void tokenize_ascii_word(char* word); + void tokenize_japanese_word(char* chunk); + inline void addTokenForHeader(const char* aTokenPrefix, nsACString& aValue, + bool aTokenizeValue = false, + const char* aDelimiters = nullptr); + nsresult stripHTML(const nsAString& inString, nsAString& outString); + // helper function to escape \n, \t, etc from a CString + void UnescapeCString(nsCString& aCString); + nsresult ScannerNext(const char16_t* text, int32_t length, int32_t pos, + bool isLastBuffer, int32_t* begin, int32_t* end, + bool* _retval); + RefPtr mWordBreaker; }; /** @@ -158,244 +160,243 @@ private: * a corpus of classified messages */ -class CorpusStore: public TokenHash { -public: - CorpusStore(); - ~CorpusStore(); +class CorpusStore : public TokenHash { + public: + CorpusStore(); + ~CorpusStore(); - /** - * retrieve the token structure for a particular string - * - * @param word the character representation of the token - * - * @return token structure containing counts, null if not found - */ - CorpusToken* get(const char* word); + /** + * retrieve the token structure for a particular string + * + * @param word the character representation of the token + * + * @return token structure containing counts, null if not found + */ + CorpusToken* get(const char* word); - /** - * add tokens to the storage, or increment counts if already exists. - * - * @param aTokenizer tokenizer for the list of tokens to remember - * @param aTraitId id for the trait whose counts will be remembered - * @param aCount number of new messages represented by the token list - */ - void rememberTokens(Tokenizer& aTokenizer, uint32_t aTraitId, uint32_t aCount); + /** + * add tokens to the storage, or increment counts if already exists. + * + * @param aTokenizer tokenizer for the list of tokens to remember + * @param aTraitId id for the trait whose counts will be remembered + * @param aCount number of new messages represented by the token list + */ + void rememberTokens(Tokenizer& aTokenizer, uint32_t aTraitId, + uint32_t aCount); - /** - * decrement counts for tokens in the storage, removing if all counts - * are zero - * - * @param aTokenizer tokenizer for the list of tokens to forget - * @param aTraitId id for the trait whose counts will be removed - * @param aCount number of messages represented by the token list - */ - void forgetTokens(Tokenizer& aTokenizer, uint32_t aTraitId, uint32_t aCount); + /** + * decrement counts for tokens in the storage, removing if all counts + * are zero + * + * @param aTokenizer tokenizer for the list of tokens to forget + * @param aTraitId id for the trait whose counts will be removed + * @param aCount number of messages represented by the token list + */ + void forgetTokens(Tokenizer& aTokenizer, uint32_t aTraitId, uint32_t aCount); - /** - * write the corpus information to file storage - * - * @param aMaximumTokenCount prune tokens if number of tokens exceeds - * this value. == 0 for no pruning - */ - void writeTrainingData(uint32_t aMaximumTokenCount); + /** + * write the corpus information to file storage + * + * @param aMaximumTokenCount prune tokens if number of tokens exceeds + * this value. == 0 for no pruning + */ + void writeTrainingData(uint32_t aMaximumTokenCount); - /** - * read the corpus information from file storage - */ - void readTrainingData(); + /** + * read the corpus information from file storage + */ + void readTrainingData(); - /** - * delete the local corpus storage file and data - */ - nsresult resetTrainingData(); + /** + * delete the local corpus storage file and data + */ + nsresult resetTrainingData(); - /** - * get the count of messages whose tokens are stored that are associated - * with a trait - * - * @param aTraitId identifier for the trait - * @return number of messages for that trait - */ - uint32_t getMessageCount(uint32_t aTraitId); + /** + * get the count of messages whose tokens are stored that are associated + * with a trait + * + * @param aTraitId identifier for the trait + * @return number of messages for that trait + */ + uint32_t getMessageCount(uint32_t aTraitId); - /** - * set the count of messages whose tokens are stored that are associated - * with a trait - * - * @param aTraitId identifier for the trait - * @param aCount number of messages for that trait - */ - void setMessageCount(uint32_t aTraitId, uint32_t aCount); + /** + * set the count of messages whose tokens are stored that are associated + * with a trait + * + * @param aTraitId identifier for the trait + * @param aCount number of messages for that trait + */ + void setMessageCount(uint32_t aTraitId, uint32_t aCount); - /** - * get the count of messages associated with a particular token and trait - * - * @param token the token string and associated counts - * @param aTraitId identifier for the trait - */ - uint32_t getTraitCount(CorpusToken *token, uint32_t aTraitId); + /** + * get the count of messages associated with a particular token and trait + * + * @param token the token string and associated counts + * @param aTraitId identifier for the trait + */ + uint32_t getTraitCount(CorpusToken* token, uint32_t aTraitId); - /** - * Add (or remove) data from a particular file to the corpus data. - * - * @param aFile the file with the data, in the format: - * - * Format of the trait file for version 1: - * [0xFCA93601] (the 01 is the version) - * for each trait to write: - * [id of trait to write] (0 means end of list) - * [number of messages per trait] - * for each token with non-zero count - * [count] - * [length of word]word - * - * @param aIsAdd should the data be added, or removed? true if adding, - * else removing. - * - * @param aRemapCount number of items in the parallel arrays aFromTraits, - * aToTraits. These arrays allow conversion of the - * trait id stored in the file (which may be originated - * externally) to the trait id used in the local corpus - * (which is defined locally using nsIMsgTraitService). - * - * @param aFromTraits array of trait ids used in aFile. If aFile contains - * trait ids that are not in this array, they are not - * remapped, but assumed to be local trait ids. - * - * @param aToTraits array of trait ids, corresponding to elements of - * aFromTraits, that represent the local trait ids to be - * used in storing data from aFile into the local corpus. - * - */ - nsresult UpdateData(nsIFile *aFile, bool aIsAdd, - uint32_t aRemapCount, uint32_t *aFromTraits, - uint32_t *aToTraits); + /** + * Add (or remove) data from a particular file to the corpus data. + * + * @param aFile the file with the data, in the format: + * + * Format of the trait file for version 1: + * [0xFCA93601] (the 01 is the version) + * for each trait to write: + * [id of trait to write] (0 means end of list) + * [number of messages per trait] + * for each token with non-zero count + * [count] + * [length of word]word + * + * @param aIsAdd should the data be added, or removed? true if adding, + * else removing. + * + * @param aRemapCount number of items in the parallel arrays aFromTraits, + * aToTraits. These arrays allow conversion of the + * trait id stored in the file (which may be originated + * externally) to the trait id used in the local corpus + * (which is defined locally using nsIMsgTraitService). + * + * @param aFromTraits array of trait ids used in aFile. If aFile contains + * trait ids that are not in this array, they are not + * remapped, but assumed to be local trait ids. + * + * @param aToTraits array of trait ids, corresponding to elements of + * aFromTraits, that represent the local trait ids to be + * used in storing data from aFile into the local corpus. + * + */ + nsresult UpdateData(nsIFile* aFile, bool aIsAdd, uint32_t aRemapCount, + uint32_t* aFromTraits, uint32_t* aToTraits); - /** - * remove all counts (message and tokens) for a trait id - * - * @param aTrait trait id for the trait to remove - */ - nsresult ClearTrait(uint32_t aTrait); + /** + * remove all counts (message and tokens) for a trait id + * + * @param aTrait trait id for the trait to remove + */ + nsresult ClearTrait(uint32_t aTrait); -protected: + protected: + /** + * return the local corpus storage file for junk traits + */ + nsresult getTrainingFile(nsIFile** aFile); - /** - * return the local corpus storage file for junk traits - */ - nsresult getTrainingFile(nsIFile ** aFile); + /** + * return the local corpus storage file for non-junk traits + */ + nsresult getTraitFile(nsIFile** aFile); - /** - * return the local corpus storage file for non-junk traits - */ - nsresult getTraitFile(nsIFile ** aFile); + /** + * read token strings from the data file + * + * @param stream file stream with token data + * @param fileSize file size + * @param aTraitId id for the trait whose counts will be read + * @param aIsAdd true to add the counts, false to remove them + * + * @return true if successful, false if error + */ + bool readTokens(FILE* stream, int64_t fileSize, uint32_t aTraitId, + bool aIsAdd); - /** - * read token strings from the data file - * - * @param stream file stream with token data - * @param fileSize file size - * @param aTraitId id for the trait whose counts will be read - * @param aIsAdd true to add the counts, false to remove them - * - * @return true if successful, false if error - */ - bool readTokens(FILE* stream, int64_t fileSize, uint32_t aTraitId, - bool aIsAdd); + /** + * write token strings to the data file + */ + bool writeTokens(FILE* stream, bool shrink, uint32_t aTraitId); - /** - * write token strings to the data file - */ - bool writeTokens(FILE* stream, bool shrink, uint32_t aTraitId); + /** + * remove counts for a token string + */ + void remove(const char* word, uint32_t aTraitId, uint32_t aCount); - /** - * remove counts for a token string - */ - void remove(const char* word, uint32_t aTraitId, uint32_t aCount); + /** + * add counts for a token string, adding the token string if new + */ + CorpusToken* add(const char* word, uint32_t aTraitId, uint32_t aCount); - /** - * add counts for a token string, adding the token string if new - */ - CorpusToken* add(const char* word, uint32_t aTraitId, uint32_t aCount); - - /** - * change counts in a trait in the traits array, adding the trait if needed - */ - nsresult updateTrait(CorpusToken* token, uint32_t aTraitId, - int32_t aCountChange); - nsCOMPtr mTrainingFile; // file used to store junk training data - nsCOMPtr mTraitFile; // file used to store non-junk - // training data - nsTArray mTraitStore; // memory for linked-list of counts - uint32_t mNextTraitIndex; // index in mTraitStore to first empty - // TraitPerToken - nsTArray mMessageCounts; // count of messages per trait - // represented in the store - nsTArray mMessageCountsId; // Parallel array to mMessageCounts, with - // the corresponding trait ID + /** + * change counts in a trait in the traits array, adding the trait if needed + */ + nsresult updateTrait(CorpusToken* token, uint32_t aTraitId, + int32_t aCountChange); + nsCOMPtr mTrainingFile; // file used to store junk training data + nsCOMPtr mTraitFile; // file used to store non-junk + // training data + nsTArray mTraitStore; // memory for linked-list of counts + uint32_t mNextTraitIndex; // index in mTraitStore to first empty + // TraitPerToken + nsTArray mMessageCounts; // count of messages per trait + // represented in the store + nsTArray mMessageCountsId; // Parallel array to mMessageCounts, + // with the corresponding trait ID }; -class nsBayesianFilter : public nsIJunkMailPlugin, nsIMsgCorpus, - nsIObserver, nsSupportsWeakReference { -public: - NS_DECL_ISUPPORTS - NS_DECL_NSIMSGFILTERPLUGIN - NS_DECL_NSIJUNKMAILPLUGIN - NS_DECL_NSIMSGCORPUS - NS_DECL_NSIOBSERVER +class nsBayesianFilter : public nsIJunkMailPlugin, + nsIMsgCorpus, + nsIObserver, + nsSupportsWeakReference { + public: + NS_DECL_ISUPPORTS + NS_DECL_NSIMSGFILTERPLUGIN + NS_DECL_NSIJUNKMAILPLUGIN + NS_DECL_NSIMSGCORPUS + NS_DECL_NSIOBSERVER - nsBayesianFilter(); + nsBayesianFilter(); - nsresult Init(); + nsresult Init(); - nsresult tokenizeMessage(const char* messageURI, nsIMsgWindow *aMsgWindow, TokenAnalyzer* analyzer); - void classifyMessage(Tokenizer& tokens, const char* messageURI, - nsIJunkMailClassificationListener* listener); + nsresult tokenizeMessage(const char* messageURI, nsIMsgWindow* aMsgWindow, + TokenAnalyzer* analyzer); + void classifyMessage(Tokenizer& tokens, const char* messageURI, + nsIJunkMailClassificationListener* listener); - void classifyMessage( - Tokenizer& tokenizer, - const char* messageURI, - nsTArray& aProTraits, - nsTArray& aAntiTraits, - nsIJunkMailClassificationListener* listener, - nsIMsgTraitClassificationListener* aTraitListener, - nsIMsgTraitDetailListener* aDetailListener); + void classifyMessage(Tokenizer& tokenizer, const char* messageURI, + nsTArray& aProTraits, + nsTArray& aAntiTraits, + nsIJunkMailClassificationListener* listener, + nsIMsgTraitClassificationListener* aTraitListener, + nsIMsgTraitDetailListener* aDetailListener); - void observeMessage(Tokenizer& tokens, const char* messageURI, - nsTArray& oldClassifications, - nsTArray& newClassifications, - nsIJunkMailClassificationListener* listener, - nsIMsgTraitClassificationListener* aTraitListener); + void observeMessage(Tokenizer& tokens, const char* messageURI, + nsTArray& oldClassifications, + nsTArray& newClassifications, + nsIJunkMailClassificationListener* listener, + nsIMsgTraitClassificationListener* aTraitListener); + protected: + virtual ~nsBayesianFilter(); -protected: - virtual ~nsBayesianFilter(); + static void TimerCallback(nsITimer* aTimer, void* aClosure); - static void TimerCallback(nsITimer* aTimer, void* aClosure); + CorpusStore mCorpus; + double mJunkProbabilityThreshold; + int32_t mMaximumTokenCount; + bool mTrainingDataDirty; + int32_t mMinFlushInterval; // in milliseconds, must be positive + // and not too close to 0 + nsCOMPtr mTimer; - CorpusStore mCorpus; - double mJunkProbabilityThreshold; - int32_t mMaximumTokenCount; - bool mTrainingDataDirty; - int32_t mMinFlushInterval; // in milliseconds, must be positive - //and not too close to 0 - nsCOMPtr mTimer; - - // index in mAnalysisStore for first empty AnalysisPerToken - uint32_t mNextAnalysisIndex; - // memory for linked list of AnalysisPerToken objects - nsTArray mAnalysisStore; - /** - * Determine the location in mAnalysisStore where the AnalysisPerToken - * object for a particular token and trait is stored - */ - uint32_t getAnalysisIndex(Token& token, uint32_t aTraitIndex); - /** - * Set the value of the AnalysisPerToken object for a particular - * token and trait - */ - nsresult setAnalysis(Token& token, uint32_t aTraitIndex, - double aDistance, double aProbability); + // index in mAnalysisStore for first empty AnalysisPerToken + uint32_t mNextAnalysisIndex; + // memory for linked list of AnalysisPerToken objects + nsTArray mAnalysisStore; + /** + * Determine the location in mAnalysisStore where the AnalysisPerToken + * object for a particular token and trait is stored + */ + uint32_t getAnalysisIndex(Token& token, uint32_t aTraitIndex); + /** + * Set the value of the AnalysisPerToken object for a particular + * token and trait + */ + nsresult setAnalysis(Token& token, uint32_t aTraitIndex, double aDistance, + double aProbability); }; -#endif // _nsBayesianFilter_h__ +#endif // _nsBayesianFilter_h__ diff --git a/mailnews/extensions/bayesian-spam-filter/src/nsBayesianFilterCID.h b/mailnews/extensions/bayesian-spam-filter/src/nsBayesianFilterCID.h index deb5d43e58..02f8201fc4 100644 --- a/mailnews/extensions/bayesian-spam-filter/src/nsBayesianFilterCID.h +++ b/mailnews/extensions/bayesian-spam-filter/src/nsBayesianFilterCID.h @@ -8,9 +8,11 @@ #define NS_BAYESIANFILTER_CONTRACTID \ "@mozilla.org/messenger/filter-plugin;1?name=bayesianfilter" -#define NS_BAYESIANFILTER_CID \ -{ /* F1070BFA-D539-11D6-90CA-00039310A47A */ \ - 0xF1070BFA, 0xD539, 0x11D6, \ - { 0x90, 0xCA, 0x00, 0x03, 0x93, 0x10, 0xA4, 0x7A }} +#define NS_BAYESIANFILTER_CID \ + { /* F1070BFA-D539-11D6-90CA-00039310A47A */ \ + 0xF1070BFA, 0xD539, 0x11D6, { \ + 0x90, 0xCA, 0x00, 0x03, 0x93, 0x10, 0xA4, 0x7A \ + } \ + } #endif /* nsBayesianFilterCID_h__ */ diff --git a/mailnews/extensions/bayesian-spam-filter/src/nsIncompleteGamma.h b/mailnews/extensions/bayesian-spam-filter/src/nsIncompleteGamma.h index 24785a4acf..f367e3477c 100644 --- a/mailnews/extensions/bayesian-spam-filter/src/nsIncompleteGamma.h +++ b/mailnews/extensions/bayesian-spam-filter/src/nsIncompleteGamma.h @@ -56,7 +56,7 @@ #include // the main routine -static double nsIncompleteGammaP (double a, double x, int *error); +static double nsIncompleteGammaP(double a, double x, int *error); // nsLnGamma(z): either a wrapper around lgamma_r or the internal function. // C_m = B[2*m]/(2*m*(2*m-1)) where B is a Bernoulli number @@ -73,20 +73,27 @@ static const double C_10 = -174611.0 / 125400.0; static const double C_11 = 77683.0 / 5796.0; // truncated asymptotic series in 1/z -static inline double lngamma_asymp (double z) -{ +static inline double lngamma_asymp(double z) { double w, w2, sum; w = 1.0 / z; w2 = w * w; - sum = w * (w2 * (w2 * (w2 * (w2 * (w2 * (w2 * (w2 * (w2 * (w2 - * (C_11 * w2 + C_10) + C_9) + C_8) + C_7) + C_6) - + C_5) + C_4) + C_3) + C_2) + C_1); + sum = + w * + (w2 * (w2 * (w2 * (w2 * (w2 * (w2 * (w2 * (w2 * (w2 * (C_11 * w2 + C_10) + + C_9) + + C_8) + + C_7) + + C_6) + + C_5) + + C_4) + + C_3) + + C_2) + + C_1); return sum; } -struct fact_table_s -{ +struct fact_table_s { double fact; double lnfact; }; @@ -111,9 +118,8 @@ static const struct fact_table_s FactTable[] = { {1307674368000.000, 2.7899271383840891566988e+01}, {20922789888000.00, 3.0671860106080672803835e+01}, {355687428096000.0, 3.3505073450136888885825e+01}, - {6402373705728000., 3.6395445208033053576674e+01} -}; -#define FactTableLength (int)(sizeof(FactTable)/sizeof(FactTable[0])) + {6402373705728000., 3.6395445208033053576674e+01}}; +#define FactTableLength (int)(sizeof(FactTable) / sizeof(FactTable[0])) // for speed static const double ln_2pi_2 = 0.918938533204672741803; // log(2*PI)/2 @@ -127,133 +133,107 @@ static const double ln_2pi_2 = 0.918938533204672741803; // log(2*PI)/2 < 1e-10. In two small regions, 1 +/- .001 and 2 +/- .001 errors increase quickly. */ -static double nsLnGamma (double z_in, int *gsign) -{ +static double nsLnGamma(double z_in, int *gsign) { double scale, z, sum, result; *gsign = 1; - int zi = (int) z_in; - if (z_in == (double) zi) - { + int zi = (int)z_in; + if (z_in == (double)zi) { if (0 < zi && zi <= FactTableLength) return FactTable[zi - 1].lnfact; // gamma(z) = (z-1)! } - for (scale = 1.0, z = z_in; z < 8.0; ++z) - scale *= z; + for (scale = 1.0, z = z_in; z < 8.0; ++z) scale *= z; - sum = lngamma_asymp (z); - result = (z - 0.5) * log (z) - z + ln_2pi_2 - log (scale); + sum = lngamma_asymp(z); + result = (z - 0.5) * log(z) - z + ln_2pi_2 - log(scale); result += sum; return result; } // log( e^(-x)*x^a/Gamma(a) ) -static inline double lnPQfactor (double a, double x) -{ +static inline double lnPQfactor(double a, double x) { int gsign; // ignored because a > 0 - return a * log (x) - x - nsLnGamma (a, &gsign); + return a * log(x) - x - nsLnGamma(a, &gsign); } -static double Pseries (double a, double x, int *error) -{ +static double Pseries(double a, double x, int *error) { double sum, term; const double eps = 2.0 * DBL_EPSILON; const int imax = 5000; int i; sum = term = 1.0 / a; - for (i = 1; i < imax; ++i) - { + for (i = 1; i < imax; ++i) { term *= x / (a + i); sum += term; - if (fabs (term) < eps * fabs (sum)) - break; + if (fabs(term) < eps * fabs(sum)) break; } - if (i >= imax) - *error = 1; + if (i >= imax) *error = 1; return sum; } -static double Qcontfrac (double a, double x, int *error) -{ +static double Qcontfrac(double a, double x, int *error) { double result, D, C, e, f, term; const double eps = 2.0 * DBL_EPSILON; - const double small = - DBL_EPSILON * DBL_EPSILON * DBL_EPSILON * DBL_EPSILON; + const double small = DBL_EPSILON * DBL_EPSILON * DBL_EPSILON * DBL_EPSILON; const int imax = 5000; int i; // modified Lentz method f = x - a + 1.0; - if (fabs (f) < small) - f = small; + if (fabs(f) < small) f = small; C = f + 1.0 / small; D = 1.0 / f; result = D; - for (i = 1; i < imax; ++i) - { + for (i = 1; i < imax; ++i) { e = i * (a - i); f += 2.0; D = f + e * D; - if (fabs (D) < small) - D = small; + if (fabs(D) < small) D = small; D = 1.0 / D; C = f + e / C; - if (fabs (C) < small) - C = small; + if (fabs(C) < small) C = small; term = C * D; result *= term; - if (fabs (term - 1.0) < eps) - break; + if (fabs(term - 1.0) < eps) break; } - if (i >= imax) - *error = 1; + if (i >= imax) *error = 1; return result; } -static double nsIncompleteGammaP (double a, double x, int *error) -{ +static double nsIncompleteGammaP(double a, double x, int *error) { double result, dom, ldom; // domain errors. the return values are meaningless but have // to return something. *error = -1; - if (a <= 0.0) - return 1.0; - if (x < 0.0) - return 0.0; + if (a <= 0.0) return 1.0; + if (x < 0.0) return 0.0; *error = 0; - if (x == 0.0) - return 0.0; + if (x == 0.0) return 0.0; - ldom = lnPQfactor (a, x); - dom = exp (ldom); + ldom = lnPQfactor(a, x); + dom = exp(ldom); // might need to adjust the crossover point - if (a <= 0.5) - { + if (a <= 0.5) { if (x < a + 1.0) - result = dom * Pseries (a, x, error); + result = dom * Pseries(a, x, error); else - result = 1.0 - dom * Qcontfrac (a, x, error); - } - else - { + result = 1.0 - dom * Qcontfrac(a, x, error); + } else { if (x < a) - result = dom * Pseries (a, x, error); + result = dom * Pseries(a, x, error); else - result = 1.0 - dom * Qcontfrac (a, x, error); + result = 1.0 - dom * Qcontfrac(a, x, error); } // not clear if this can ever happen - if (result > 1.0) - result = 1.0; - if (result < 0.0) - result = 0.0; + if (result > 1.0) result = 1.0; + if (result < 0.0) result = 0.0; return result; } #endif - diff --git a/mailnews/extensions/fts3/src/Normalize.c b/mailnews/extensions/fts3/src/Normalize.c index 92ac400e2f..984a03bd08 100644 --- a/mailnews/extensions/fts3/src/Normalize.c +++ b/mailnews/extensions/fts3/src/Normalize.c @@ -6,1924 +6,2689 @@ /* THIS FILE IS GENERATED BY generate_table.py. DON'T EDIT THIS */ static const unsigned short gNormalizeTable0040[] = { - /* U+0040 */ - 0x0040, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, - 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, - 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, - 0x0078, 0x0079, 0x007a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, - 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, - 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, - 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, - 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, - }; - -static const unsigned short gNormalizeTable0080[] = { - /* U+0080 */ - 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, - 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, - 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, - 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, - 0x0020, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, - 0x0020, 0x00a9, 0x0061, 0x00ab, 0x00ac, 0x0020, 0x00ae, 0x0020, - 0x00b0, 0x00b1, 0x0032, 0x0033, 0x0020, 0x03bc, 0x00b6, 0x00b7, - 0x0020, 0x0031, 0x006f, 0x00bb, 0x0031, 0x0031, 0x0033, 0x00bf, - }; - -static const unsigned short gNormalizeTable00c0[] = { - /* U+00c0 */ - 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x00e6, 0x0063, - 0x0065, 0x0065, 0x0065, 0x0065, 0x0069, 0x0069, 0x0069, 0x0069, - 0x00f0, 0x006e, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x00d7, - 0x00f8, 0x0075, 0x0075, 0x0075, 0x0075, 0x0079, 0x00fe, 0x0073, - 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x00e6, 0x0063, - 0x0065, 0x0065, 0x0065, 0x0065, 0x0069, 0x0069, 0x0069, 0x0069, - 0x00f0, 0x006e, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x00f7, - 0x00f8, 0x0075, 0x0075, 0x0075, 0x0075, 0x0079, 0x00fe, 0x0079, - }; - -static const unsigned short gNormalizeTable0100[] = { - /* U+0100 */ - 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0063, 0x0063, - 0x0063, 0x0063, 0x0063, 0x0063, 0x0063, 0x0063, 0x0064, 0x0064, - 0x0111, 0x0111, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, - 0x0065, 0x0065, 0x0065, 0x0065, 0x0067, 0x0067, 0x0067, 0x0067, - 0x0067, 0x0067, 0x0067, 0x0067, 0x0068, 0x0068, 0x0127, 0x0127, - 0x0069, 0x0069, 0x0069, 0x0069, 0x0069, 0x0069, 0x0069, 0x0069, - 0x0069, 0x0131, 0x0069, 0x0069, 0x006a, 0x006a, 0x006b, 0x006b, - 0x0138, 0x006c, 0x006c, 0x006c, 0x006c, 0x006c, 0x006c, 0x006c, - }; - -static const unsigned short gNormalizeTable0140[] = { - /* U+0140 */ - 0x006c, 0x0142, 0x0142, 0x006e, 0x006e, 0x006e, 0x006e, 0x006e, - 0x006e, 0x02bc, 0x014b, 0x014b, 0x006f, 0x006f, 0x006f, 0x006f, - 0x006f, 0x006f, 0x0153, 0x0153, 0x0072, 0x0072, 0x0072, 0x0072, - 0x0072, 0x0072, 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, - 0x0073, 0x0073, 0x0074, 0x0074, 0x0074, 0x0074, 0x0167, 0x0167, - 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, - 0x0075, 0x0075, 0x0075, 0x0075, 0x0077, 0x0077, 0x0079, 0x0079, - 0x0079, 0x007a, 0x007a, 0x007a, 0x007a, 0x007a, 0x007a, 0x0073, - }; - -static const unsigned short gNormalizeTable0180[] = { - /* U+0180 */ - 0x0180, 0x0253, 0x0183, 0x0183, 0x0185, 0x0185, 0x0254, 0x0188, - 0x0188, 0x0256, 0x0257, 0x018c, 0x018c, 0x018d, 0x01dd, 0x0259, - 0x025b, 0x0192, 0x0192, 0x0260, 0x0263, 0x0195, 0x0269, 0x0268, - 0x0199, 0x0199, 0x019a, 0x019b, 0x026f, 0x0272, 0x019e, 0x0275, - 0x006f, 0x006f, 0x01a3, 0x01a3, 0x01a5, 0x01a5, 0x0280, 0x01a8, - 0x01a8, 0x0283, 0x01aa, 0x01ab, 0x01ad, 0x01ad, 0x0288, 0x0075, - 0x0075, 0x028a, 0x028b, 0x01b4, 0x01b4, 0x01b6, 0x01b6, 0x0292, - 0x01b9, 0x01b9, 0x01ba, 0x01bb, 0x01bd, 0x01bd, 0x01be, 0x01bf, - }; - -static const unsigned short gNormalizeTable01c0[] = { - /* U+01c0 */ - 0x01c0, 0x01c1, 0x01c2, 0x01c3, 0x0064, 0x0064, 0x0064, 0x006c, - 0x006c, 0x006c, 0x006e, 0x006e, 0x006e, 0x0061, 0x0061, 0x0069, - 0x0069, 0x006f, 0x006f, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, - 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x01dd, 0x0061, 0x0061, - 0x0061, 0x0061, 0x00e6, 0x00e6, 0x01e5, 0x01e5, 0x0067, 0x0067, - 0x006b, 0x006b, 0x006f, 0x006f, 0x006f, 0x006f, 0x0292, 0x0292, - 0x006a, 0x0064, 0x0064, 0x0064, 0x0067, 0x0067, 0x0195, 0x01bf, - 0x006e, 0x006e, 0x0061, 0x0061, 0x00e6, 0x00e6, 0x00f8, 0x00f8, - }; - -static const unsigned short gNormalizeTable0200[] = { - /* U+0200 */ - 0x0061, 0x0061, 0x0061, 0x0061, 0x0065, 0x0065, 0x0065, 0x0065, - 0x0069, 0x0069, 0x0069, 0x0069, 0x006f, 0x006f, 0x006f, 0x006f, - 0x0072, 0x0072, 0x0072, 0x0072, 0x0075, 0x0075, 0x0075, 0x0075, - 0x0073, 0x0073, 0x0074, 0x0074, 0x021d, 0x021d, 0x0068, 0x0068, - 0x019e, 0x0221, 0x0223, 0x0223, 0x0225, 0x0225, 0x0061, 0x0061, - 0x0065, 0x0065, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, - 0x006f, 0x006f, 0x0079, 0x0079, 0x0234, 0x0235, 0x0236, 0x0237, - 0x0238, 0x0239, 0x2c65, 0x023c, 0x023c, 0x019a, 0x2c66, 0x023f, - }; - -static const unsigned short gNormalizeTable0240[] = { - /* U+0240 */ - 0x0240, 0x0242, 0x0242, 0x0180, 0x0289, 0x028c, 0x0247, 0x0247, - 0x0249, 0x0249, 0x024b, 0x024b, 0x024d, 0x024d, 0x024f, 0x024f, - 0x0250, 0x0251, 0x0252, 0x0253, 0x0254, 0x0255, 0x0256, 0x0257, - 0x0258, 0x0259, 0x025a, 0x025b, 0x025c, 0x025d, 0x025e, 0x025f, - 0x0260, 0x0261, 0x0262, 0x0263, 0x0264, 0x0265, 0x0266, 0x0267, - 0x0268, 0x0269, 0x026a, 0x026b, 0x026c, 0x026d, 0x026e, 0x026f, - 0x0270, 0x0271, 0x0272, 0x0273, 0x0274, 0x0275, 0x0276, 0x0277, - 0x0278, 0x0279, 0x027a, 0x027b, 0x027c, 0x027d, 0x027e, 0x027f, - }; - -static const unsigned short gNormalizeTable0280[] = { - /* U+0280 */ - 0x0280, 0x0281, 0x0282, 0x0283, 0x0284, 0x0285, 0x0286, 0x0287, - 0x0288, 0x0289, 0x028a, 0x028b, 0x028c, 0x028d, 0x028e, 0x028f, - 0x0290, 0x0291, 0x0292, 0x0293, 0x0294, 0x0295, 0x0296, 0x0297, - 0x0298, 0x0299, 0x029a, 0x029b, 0x029c, 0x029d, 0x029e, 0x029f, - 0x02a0, 0x02a1, 0x02a2, 0x02a3, 0x02a4, 0x02a5, 0x02a6, 0x02a7, - 0x02a8, 0x02a9, 0x02aa, 0x02ab, 0x02ac, 0x02ad, 0x02ae, 0x02af, - 0x0068, 0x0266, 0x006a, 0x0072, 0x0279, 0x027b, 0x0281, 0x0077, - 0x0079, 0x02b9, 0x02ba, 0x02bb, 0x02bc, 0x02bd, 0x02be, 0x02bf, - }; - -static const unsigned short gNormalizeTable02c0[] = { - /* U+02c0 */ - 0x02c0, 0x02c1, 0x02c2, 0x02c3, 0x02c4, 0x02c5, 0x02c6, 0x02c7, - 0x02c8, 0x02c9, 0x02ca, 0x02cb, 0x02cc, 0x02cd, 0x02ce, 0x02cf, - 0x02d0, 0x02d1, 0x02d2, 0x02d3, 0x02d4, 0x02d5, 0x02d6, 0x02d7, - 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x02de, 0x02df, - 0x0263, 0x006c, 0x0073, 0x0078, 0x0295, 0x02e5, 0x02e6, 0x02e7, - 0x02e8, 0x02e9, 0x02ea, 0x02eb, 0x02ec, 0x02ed, 0x02ee, 0x02ef, - 0x02f0, 0x02f1, 0x02f2, 0x02f3, 0x02f4, 0x02f5, 0x02f6, 0x02f7, - 0x02f8, 0x02f9, 0x02fa, 0x02fb, 0x02fc, 0x02fd, 0x02fe, 0x02ff, - }; - -static const unsigned short gNormalizeTable0340[] = { - /* U+0340 */ - 0x0300, 0x0301, 0x0342, 0x0313, 0x0308, 0x03b9, 0x0346, 0x0347, - 0x0348, 0x0349, 0x034a, 0x034b, 0x034c, 0x034d, 0x034e, 0x0020, - 0x0350, 0x0351, 0x0352, 0x0353, 0x0354, 0x0355, 0x0356, 0x0357, - 0x0358, 0x0359, 0x035a, 0x035b, 0x035c, 0x035d, 0x035e, 0x035f, - 0x0360, 0x0361, 0x0362, 0x0363, 0x0364, 0x0365, 0x0366, 0x0367, - 0x0368, 0x0369, 0x036a, 0x036b, 0x036c, 0x036d, 0x036e, 0x036f, - 0x0371, 0x0371, 0x0373, 0x0373, 0x02b9, 0x0375, 0x0377, 0x0377, - 0x0378, 0x0379, 0x0020, 0x037b, 0x037c, 0x037d, 0x003b, 0x037f, - }; - -static const unsigned short gNormalizeTable0380[] = { - /* U+0380 */ - 0x0380, 0x0381, 0x0382, 0x0383, 0x0020, 0x0020, 0x03b1, 0x00b7, - 0x03b5, 0x03b7, 0x03b9, 0x038b, 0x03bf, 0x038d, 0x03c5, 0x03c9, - 0x03b9, 0x03b1, 0x03b2, 0x03b3, 0x03b4, 0x03b5, 0x03b6, 0x03b7, - 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd, 0x03be, 0x03bf, - 0x03c0, 0x03c1, 0x03a2, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7, - 0x03c8, 0x03c9, 0x03b9, 0x03c5, 0x03b1, 0x03b5, 0x03b7, 0x03b9, - 0x03c5, 0x03b1, 0x03b2, 0x03b3, 0x03b4, 0x03b5, 0x03b6, 0x03b7, - 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd, 0x03be, 0x03bf, - }; - -static const unsigned short gNormalizeTable03c0[] = { - /* U+03c0 */ - 0x03c0, 0x03c1, 0x03c3, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7, - 0x03c8, 0x03c9, 0x03b9, 0x03c5, 0x03bf, 0x03c5, 0x03c9, 0x03d7, - 0x03b2, 0x03b8, 0x03c5, 0x03c5, 0x03c5, 0x03c6, 0x03c0, 0x03d7, - 0x03d9, 0x03d9, 0x03db, 0x03db, 0x03dd, 0x03dd, 0x03df, 0x03df, - 0x03e1, 0x03e1, 0x03e3, 0x03e3, 0x03e5, 0x03e5, 0x03e7, 0x03e7, - 0x03e9, 0x03e9, 0x03eb, 0x03eb, 0x03ed, 0x03ed, 0x03ef, 0x03ef, - 0x03ba, 0x03c1, 0x03c3, 0x03f3, 0x03b8, 0x03b5, 0x03f6, 0x03f8, - 0x03f8, 0x03c3, 0x03fb, 0x03fb, 0x03fc, 0x037b, 0x037c, 0x037d, - }; - -static const unsigned short gNormalizeTable0400[] = { - /* U+0400 */ - 0x0435, 0x0435, 0x0452, 0x0433, 0x0454, 0x0455, 0x0456, 0x0456, - 0x0458, 0x0459, 0x045a, 0x045b, 0x043a, 0x0438, 0x0443, 0x045f, - 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, - 0x0438, 0x0438, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, - 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, - 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f, - 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, - 0x0438, 0x0438, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, - }; - -static const unsigned short gNormalizeTable0440[] = { - /* U+0440 */ - 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, - 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f, - 0x0435, 0x0435, 0x0452, 0x0433, 0x0454, 0x0455, 0x0456, 0x0456, - 0x0458, 0x0459, 0x045a, 0x045b, 0x043a, 0x0438, 0x0443, 0x045f, - 0x0461, 0x0461, 0x0463, 0x0463, 0x0465, 0x0465, 0x0467, 0x0467, - 0x0469, 0x0469, 0x046b, 0x046b, 0x046d, 0x046d, 0x046f, 0x046f, - 0x0471, 0x0471, 0x0473, 0x0473, 0x0475, 0x0475, 0x0475, 0x0475, - 0x0479, 0x0479, 0x047b, 0x047b, 0x047d, 0x047d, 0x047f, 0x047f, - }; - -static const unsigned short gNormalizeTable0480[] = { - /* U+0480 */ - 0x0481, 0x0481, 0x0482, 0x0483, 0x0484, 0x0485, 0x0486, 0x0487, - 0x0488, 0x0489, 0x048b, 0x048b, 0x048d, 0x048d, 0x048f, 0x048f, - 0x0491, 0x0491, 0x0493, 0x0493, 0x0495, 0x0495, 0x0497, 0x0497, - 0x0499, 0x0499, 0x049b, 0x049b, 0x049d, 0x049d, 0x049f, 0x049f, - 0x04a1, 0x04a1, 0x04a3, 0x04a3, 0x04a5, 0x04a5, 0x04a7, 0x04a7, - 0x04a9, 0x04a9, 0x04ab, 0x04ab, 0x04ad, 0x04ad, 0x04af, 0x04af, - 0x04b1, 0x04b1, 0x04b3, 0x04b3, 0x04b5, 0x04b5, 0x04b7, 0x04b7, - 0x04b9, 0x04b9, 0x04bb, 0x04bb, 0x04bd, 0x04bd, 0x04bf, 0x04bf, - }; - -static const unsigned short gNormalizeTable04c0[] = { - /* U+04c0 */ - 0x04cf, 0x0436, 0x0436, 0x04c4, 0x04c4, 0x04c6, 0x04c6, 0x04c8, - 0x04c8, 0x04ca, 0x04ca, 0x04cc, 0x04cc, 0x04ce, 0x04ce, 0x04cf, - 0x0430, 0x0430, 0x0430, 0x0430, 0x04d5, 0x04d5, 0x0435, 0x0435, - 0x04d9, 0x04d9, 0x04d9, 0x04d9, 0x0436, 0x0436, 0x0437, 0x0437, - 0x04e1, 0x04e1, 0x0438, 0x0438, 0x0438, 0x0438, 0x043e, 0x043e, - 0x04e9, 0x04e9, 0x04e9, 0x04e9, 0x044d, 0x044d, 0x0443, 0x0443, - 0x0443, 0x0443, 0x0443, 0x0443, 0x0447, 0x0447, 0x04f7, 0x04f7, - 0x044b, 0x044b, 0x04fb, 0x04fb, 0x04fd, 0x04fd, 0x04ff, 0x04ff, - }; - -static const unsigned short gNormalizeTable0500[] = { - /* U+0500 */ - 0x0501, 0x0501, 0x0503, 0x0503, 0x0505, 0x0505, 0x0507, 0x0507, - 0x0509, 0x0509, 0x050b, 0x050b, 0x050d, 0x050d, 0x050f, 0x050f, - 0x0511, 0x0511, 0x0513, 0x0513, 0x0515, 0x0515, 0x0517, 0x0517, - 0x0519, 0x0519, 0x051b, 0x051b, 0x051d, 0x051d, 0x051f, 0x051f, - 0x0521, 0x0521, 0x0523, 0x0523, 0x0525, 0x0525, 0x0526, 0x0527, - 0x0528, 0x0529, 0x052a, 0x052b, 0x052c, 0x052d, 0x052e, 0x052f, - 0x0530, 0x0561, 0x0562, 0x0563, 0x0564, 0x0565, 0x0566, 0x0567, - 0x0568, 0x0569, 0x056a, 0x056b, 0x056c, 0x056d, 0x056e, 0x056f, - }; - -static const unsigned short gNormalizeTable0540[] = { - /* U+0540 */ - 0x0570, 0x0571, 0x0572, 0x0573, 0x0574, 0x0575, 0x0576, 0x0577, - 0x0578, 0x0579, 0x057a, 0x057b, 0x057c, 0x057d, 0x057e, 0x057f, - 0x0580, 0x0581, 0x0582, 0x0583, 0x0584, 0x0585, 0x0586, 0x0557, - 0x0558, 0x0559, 0x055a, 0x055b, 0x055c, 0x055d, 0x055e, 0x055f, - 0x0560, 0x0561, 0x0562, 0x0563, 0x0564, 0x0565, 0x0566, 0x0567, - 0x0568, 0x0569, 0x056a, 0x056b, 0x056c, 0x056d, 0x056e, 0x056f, - 0x0570, 0x0571, 0x0572, 0x0573, 0x0574, 0x0575, 0x0576, 0x0577, - 0x0578, 0x0579, 0x057a, 0x057b, 0x057c, 0x057d, 0x057e, 0x057f, - }; - -static const unsigned short gNormalizeTable0580[] = { - /* U+0580 */ - 0x0580, 0x0581, 0x0582, 0x0583, 0x0584, 0x0585, 0x0586, 0x0565, - 0x0588, 0x0589, 0x058a, 0x058b, 0x058c, 0x058d, 0x058e, 0x058f, - 0x0590, 0x0591, 0x0592, 0x0593, 0x0594, 0x0595, 0x0596, 0x0597, - 0x0598, 0x0599, 0x059a, 0x059b, 0x059c, 0x059d, 0x059e, 0x059f, - 0x05a0, 0x05a1, 0x05a2, 0x05a3, 0x05a4, 0x05a5, 0x05a6, 0x05a7, - 0x05a8, 0x05a9, 0x05aa, 0x05ab, 0x05ac, 0x05ad, 0x05ae, 0x05af, - 0x05b0, 0x05b1, 0x05b2, 0x05b3, 0x05b4, 0x05b5, 0x05b6, 0x05b7, - 0x05b8, 0x05b9, 0x05ba, 0x05bb, 0x05bc, 0x05bd, 0x05be, 0x05bf, - }; - -static const unsigned short gNormalizeTable0600[] = { - /* U+0600 */ - 0x0600, 0x0601, 0x0602, 0x0603, 0x0604, 0x0605, 0x0606, 0x0607, - 0x0608, 0x0609, 0x060a, 0x060b, 0x060c, 0x060d, 0x060e, 0x060f, - 0x0610, 0x0611, 0x0612, 0x0613, 0x0614, 0x0615, 0x0616, 0x0617, - 0x0618, 0x0619, 0x061a, 0x061b, 0x061c, 0x061d, 0x061e, 0x061f, - 0x0620, 0x0621, 0x0627, 0x0627, 0x0648, 0x0627, 0x064a, 0x0627, - 0x0628, 0x0629, 0x062a, 0x062b, 0x062c, 0x062d, 0x062e, 0x062f, - 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x0637, - 0x0638, 0x0639, 0x063a, 0x063b, 0x063c, 0x063d, 0x063e, 0x063f, - }; - -static const unsigned short gNormalizeTable0640[] = { - /* U+0640 */ - 0x0640, 0x0641, 0x0642, 0x0643, 0x0644, 0x0645, 0x0646, 0x0647, - 0x0648, 0x0649, 0x064a, 0x064b, 0x064c, 0x064d, 0x064e, 0x064f, - 0x0650, 0x0651, 0x0652, 0x0653, 0x0654, 0x0655, 0x0656, 0x0657, - 0x0658, 0x0659, 0x065a, 0x065b, 0x065c, 0x065d, 0x065e, 0x065f, - 0x0660, 0x0661, 0x0662, 0x0663, 0x0664, 0x0665, 0x0666, 0x0667, - 0x0668, 0x0669, 0x066a, 0x066b, 0x066c, 0x066d, 0x066e, 0x066f, - 0x0670, 0x0671, 0x0672, 0x0673, 0x0674, 0x0627, 0x0648, 0x06c7, - 0x064a, 0x0679, 0x067a, 0x067b, 0x067c, 0x067d, 0x067e, 0x067f, - }; - -static const unsigned short gNormalizeTable06c0[] = { - /* U+06c0 */ - 0x06d5, 0x06c1, 0x06c1, 0x06c3, 0x06c4, 0x06c5, 0x06c6, 0x06c7, - 0x06c8, 0x06c9, 0x06ca, 0x06cb, 0x06cc, 0x06cd, 0x06ce, 0x06cf, - 0x06d0, 0x06d1, 0x06d2, 0x06d2, 0x06d4, 0x06d5, 0x06d6, 0x06d7, - 0x06d8, 0x06d9, 0x06da, 0x06db, 0x06dc, 0x06dd, 0x06de, 0x06df, - 0x06e0, 0x06e1, 0x06e2, 0x06e3, 0x06e4, 0x06e5, 0x06e6, 0x06e7, - 0x06e8, 0x06e9, 0x06ea, 0x06eb, 0x06ec, 0x06ed, 0x06ee, 0x06ef, - 0x06f0, 0x06f1, 0x06f2, 0x06f3, 0x06f4, 0x06f5, 0x06f6, 0x06f7, - 0x06f8, 0x06f9, 0x06fa, 0x06fb, 0x06fc, 0x06fd, 0x06fe, 0x06ff, - }; - -static const unsigned short gNormalizeTable0900[] = { - /* U+0900 */ - 0x0900, 0x0901, 0x0902, 0x0903, 0x0904, 0x0905, 0x0906, 0x0907, - 0x0908, 0x0909, 0x090a, 0x090b, 0x090c, 0x090d, 0x090e, 0x090f, - 0x0910, 0x0911, 0x0912, 0x0913, 0x0914, 0x0915, 0x0916, 0x0917, - 0x0918, 0x0919, 0x091a, 0x091b, 0x091c, 0x091d, 0x091e, 0x091f, - 0x0920, 0x0921, 0x0922, 0x0923, 0x0924, 0x0925, 0x0926, 0x0927, - 0x0928, 0x0928, 0x092a, 0x092b, 0x092c, 0x092d, 0x092e, 0x092f, - 0x0930, 0x0930, 0x0932, 0x0933, 0x0933, 0x0935, 0x0936, 0x0937, - 0x0938, 0x0939, 0x093a, 0x093b, 0x093c, 0x093d, 0x093e, 0x093f, - }; - -static const unsigned short gNormalizeTable0940[] = { - /* U+0940 */ - 0x0940, 0x0941, 0x0942, 0x0943, 0x0944, 0x0945, 0x0946, 0x0947, - 0x0948, 0x0949, 0x094a, 0x094b, 0x094c, 0x094d, 0x094e, 0x094f, - 0x0950, 0x0951, 0x0952, 0x0953, 0x0954, 0x0955, 0x0956, 0x0957, - 0x0915, 0x0916, 0x0917, 0x091c, 0x0921, 0x0922, 0x092b, 0x092f, - 0x0960, 0x0961, 0x0962, 0x0963, 0x0964, 0x0965, 0x0966, 0x0967, - 0x0968, 0x0969, 0x096a, 0x096b, 0x096c, 0x096d, 0x096e, 0x096f, - 0x0970, 0x0971, 0x0972, 0x0973, 0x0974, 0x0975, 0x0976, 0x0977, - 0x0978, 0x0979, 0x097a, 0x097b, 0x097c, 0x097d, 0x097e, 0x097f, - }; - -static const unsigned short gNormalizeTable09c0[] = { - /* U+09c0 */ - 0x09c0, 0x09c1, 0x09c2, 0x09c3, 0x09c4, 0x09c5, 0x09c6, 0x09c7, - 0x09c8, 0x09c9, 0x09ca, 0x09c7, 0x09c7, 0x09cd, 0x09ce, 0x09cf, - 0x09d0, 0x09d1, 0x09d2, 0x09d3, 0x09d4, 0x09d5, 0x09d6, 0x09d7, - 0x09d8, 0x09d9, 0x09da, 0x09db, 0x09a1, 0x09a2, 0x09de, 0x09af, - 0x09e0, 0x09e1, 0x09e2, 0x09e3, 0x09e4, 0x09e5, 0x09e6, 0x09e7, - 0x09e8, 0x09e9, 0x09ea, 0x09eb, 0x09ec, 0x09ed, 0x09ee, 0x09ef, - 0x09f0, 0x09f1, 0x09f2, 0x09f3, 0x09f4, 0x09f5, 0x09f6, 0x09f7, - 0x09f8, 0x09f9, 0x09fa, 0x09fb, 0x09fc, 0x09fd, 0x09fe, 0x09ff, - }; - -static const unsigned short gNormalizeTable0a00[] = { - /* U+0a00 */ - 0x0a00, 0x0a01, 0x0a02, 0x0a03, 0x0a04, 0x0a05, 0x0a06, 0x0a07, - 0x0a08, 0x0a09, 0x0a0a, 0x0a0b, 0x0a0c, 0x0a0d, 0x0a0e, 0x0a0f, - 0x0a10, 0x0a11, 0x0a12, 0x0a13, 0x0a14, 0x0a15, 0x0a16, 0x0a17, - 0x0a18, 0x0a19, 0x0a1a, 0x0a1b, 0x0a1c, 0x0a1d, 0x0a1e, 0x0a1f, - 0x0a20, 0x0a21, 0x0a22, 0x0a23, 0x0a24, 0x0a25, 0x0a26, 0x0a27, - 0x0a28, 0x0a29, 0x0a2a, 0x0a2b, 0x0a2c, 0x0a2d, 0x0a2e, 0x0a2f, - 0x0a30, 0x0a31, 0x0a32, 0x0a32, 0x0a34, 0x0a35, 0x0a38, 0x0a37, - 0x0a38, 0x0a39, 0x0a3a, 0x0a3b, 0x0a3c, 0x0a3d, 0x0a3e, 0x0a3f, - }; - -static const unsigned short gNormalizeTable0a40[] = { - /* U+0a40 */ - 0x0a40, 0x0a41, 0x0a42, 0x0a43, 0x0a44, 0x0a45, 0x0a46, 0x0a47, - 0x0a48, 0x0a49, 0x0a4a, 0x0a4b, 0x0a4c, 0x0a4d, 0x0a4e, 0x0a4f, - 0x0a50, 0x0a51, 0x0a52, 0x0a53, 0x0a54, 0x0a55, 0x0a56, 0x0a57, - 0x0a58, 0x0a16, 0x0a17, 0x0a1c, 0x0a5c, 0x0a5d, 0x0a2b, 0x0a5f, - 0x0a60, 0x0a61, 0x0a62, 0x0a63, 0x0a64, 0x0a65, 0x0a66, 0x0a67, - 0x0a68, 0x0a69, 0x0a6a, 0x0a6b, 0x0a6c, 0x0a6d, 0x0a6e, 0x0a6f, - 0x0a70, 0x0a71, 0x0a72, 0x0a73, 0x0a74, 0x0a75, 0x0a76, 0x0a77, - 0x0a78, 0x0a79, 0x0a7a, 0x0a7b, 0x0a7c, 0x0a7d, 0x0a7e, 0x0a7f, - }; - -static const unsigned short gNormalizeTable0b40[] = { - /* U+0b40 */ - 0x0b40, 0x0b41, 0x0b42, 0x0b43, 0x0b44, 0x0b45, 0x0b46, 0x0b47, - 0x0b47, 0x0b49, 0x0b4a, 0x0b47, 0x0b47, 0x0b4d, 0x0b4e, 0x0b4f, - 0x0b50, 0x0b51, 0x0b52, 0x0b53, 0x0b54, 0x0b55, 0x0b56, 0x0b57, - 0x0b58, 0x0b59, 0x0b5a, 0x0b5b, 0x0b21, 0x0b22, 0x0b5e, 0x0b5f, - 0x0b60, 0x0b61, 0x0b62, 0x0b63, 0x0b64, 0x0b65, 0x0b66, 0x0b67, - 0x0b68, 0x0b69, 0x0b6a, 0x0b6b, 0x0b6c, 0x0b6d, 0x0b6e, 0x0b6f, - 0x0b70, 0x0b71, 0x0b72, 0x0b73, 0x0b74, 0x0b75, 0x0b76, 0x0b77, - 0x0b78, 0x0b79, 0x0b7a, 0x0b7b, 0x0b7c, 0x0b7d, 0x0b7e, 0x0b7f, - }; - -static const unsigned short gNormalizeTable0b80[] = { - /* U+0b80 */ - 0x0b80, 0x0b81, 0x0b82, 0x0b83, 0x0b84, 0x0b85, 0x0b86, 0x0b87, - 0x0b88, 0x0b89, 0x0b8a, 0x0b8b, 0x0b8c, 0x0b8d, 0x0b8e, 0x0b8f, - 0x0b90, 0x0b91, 0x0b92, 0x0b93, 0x0b92, 0x0b95, 0x0b96, 0x0b97, - 0x0b98, 0x0b99, 0x0b9a, 0x0b9b, 0x0b9c, 0x0b9d, 0x0b9e, 0x0b9f, - 0x0ba0, 0x0ba1, 0x0ba2, 0x0ba3, 0x0ba4, 0x0ba5, 0x0ba6, 0x0ba7, - 0x0ba8, 0x0ba9, 0x0baa, 0x0bab, 0x0bac, 0x0bad, 0x0bae, 0x0baf, - 0x0bb0, 0x0bb1, 0x0bb2, 0x0bb3, 0x0bb4, 0x0bb5, 0x0bb6, 0x0bb7, - 0x0bb8, 0x0bb9, 0x0bba, 0x0bbb, 0x0bbc, 0x0bbd, 0x0bbe, 0x0bbf, - }; - -static const unsigned short gNormalizeTable0bc0[] = { - /* U+0bc0 */ - 0x0bc0, 0x0bc1, 0x0bc2, 0x0bc3, 0x0bc4, 0x0bc5, 0x0bc6, 0x0bc7, - 0x0bc8, 0x0bc9, 0x0bc6, 0x0bc7, 0x0bc6, 0x0bcd, 0x0bce, 0x0bcf, - 0x0bd0, 0x0bd1, 0x0bd2, 0x0bd3, 0x0bd4, 0x0bd5, 0x0bd6, 0x0bd7, - 0x0bd8, 0x0bd9, 0x0bda, 0x0bdb, 0x0bdc, 0x0bdd, 0x0bde, 0x0bdf, - 0x0be0, 0x0be1, 0x0be2, 0x0be3, 0x0be4, 0x0be5, 0x0be6, 0x0be7, - 0x0be8, 0x0be9, 0x0bea, 0x0beb, 0x0bec, 0x0bed, 0x0bee, 0x0bef, - 0x0bf0, 0x0bf1, 0x0bf2, 0x0bf3, 0x0bf4, 0x0bf5, 0x0bf6, 0x0bf7, - 0x0bf8, 0x0bf9, 0x0bfa, 0x0bfb, 0x0bfc, 0x0bfd, 0x0bfe, 0x0bff, - }; - -static const unsigned short gNormalizeTable0c40[] = { - /* U+0c40 */ - 0x0c40, 0x0c41, 0x0c42, 0x0c43, 0x0c44, 0x0c45, 0x0c46, 0x0c47, - 0x0c46, 0x0c49, 0x0c4a, 0x0c4b, 0x0c4c, 0x0c4d, 0x0c4e, 0x0c4f, - 0x0c50, 0x0c51, 0x0c52, 0x0c53, 0x0c54, 0x0c55, 0x0c56, 0x0c57, - 0x0c58, 0x0c59, 0x0c5a, 0x0c5b, 0x0c5c, 0x0c5d, 0x0c5e, 0x0c5f, - 0x0c60, 0x0c61, 0x0c62, 0x0c63, 0x0c64, 0x0c65, 0x0c66, 0x0c67, - 0x0c68, 0x0c69, 0x0c6a, 0x0c6b, 0x0c6c, 0x0c6d, 0x0c6e, 0x0c6f, - 0x0c70, 0x0c71, 0x0c72, 0x0c73, 0x0c74, 0x0c75, 0x0c76, 0x0c77, - 0x0c78, 0x0c79, 0x0c7a, 0x0c7b, 0x0c7c, 0x0c7d, 0x0c7e, 0x0c7f, - }; - -static const unsigned short gNormalizeTable0cc0[] = { - /* U+0cc0 */ - 0x0cbf, 0x0cc1, 0x0cc2, 0x0cc3, 0x0cc4, 0x0cc5, 0x0cc6, 0x0cc6, - 0x0cc6, 0x0cc9, 0x0cc6, 0x0cc6, 0x0ccc, 0x0ccd, 0x0cce, 0x0ccf, - 0x0cd0, 0x0cd1, 0x0cd2, 0x0cd3, 0x0cd4, 0x0cd5, 0x0cd6, 0x0cd7, - 0x0cd8, 0x0cd9, 0x0cda, 0x0cdb, 0x0cdc, 0x0cdd, 0x0cde, 0x0cdf, - 0x0ce0, 0x0ce1, 0x0ce2, 0x0ce3, 0x0ce4, 0x0ce5, 0x0ce6, 0x0ce7, - 0x0ce8, 0x0ce9, 0x0cea, 0x0ceb, 0x0cec, 0x0ced, 0x0cee, 0x0cef, - 0x0cf0, 0x0cf1, 0x0cf2, 0x0cf3, 0x0cf4, 0x0cf5, 0x0cf6, 0x0cf7, - 0x0cf8, 0x0cf9, 0x0cfa, 0x0cfb, 0x0cfc, 0x0cfd, 0x0cfe, 0x0cff, - }; - -static const unsigned short gNormalizeTable0d40[] = { - /* U+0d40 */ - 0x0d40, 0x0d41, 0x0d42, 0x0d43, 0x0d44, 0x0d45, 0x0d46, 0x0d47, - 0x0d48, 0x0d49, 0x0d46, 0x0d47, 0x0d46, 0x0d4d, 0x0d4e, 0x0d4f, - 0x0d50, 0x0d51, 0x0d52, 0x0d53, 0x0d54, 0x0d55, 0x0d56, 0x0d57, - 0x0d58, 0x0d59, 0x0d5a, 0x0d5b, 0x0d5c, 0x0d5d, 0x0d5e, 0x0d5f, - 0x0d60, 0x0d61, 0x0d62, 0x0d63, 0x0d64, 0x0d65, 0x0d66, 0x0d67, - 0x0d68, 0x0d69, 0x0d6a, 0x0d6b, 0x0d6c, 0x0d6d, 0x0d6e, 0x0d6f, - 0x0d70, 0x0d71, 0x0d72, 0x0d73, 0x0d74, 0x0d75, 0x0d76, 0x0d77, - 0x0d78, 0x0d79, 0x0d7a, 0x0d7b, 0x0d7c, 0x0d7d, 0x0d7e, 0x0d7f, - }; - -static const unsigned short gNormalizeTable0dc0[] = { - /* U+0dc0 */ - 0x0dc0, 0x0dc1, 0x0dc2, 0x0dc3, 0x0dc4, 0x0dc5, 0x0dc6, 0x0dc7, - 0x0dc8, 0x0dc9, 0x0dca, 0x0dcb, 0x0dcc, 0x0dcd, 0x0dce, 0x0dcf, - 0x0dd0, 0x0dd1, 0x0dd2, 0x0dd3, 0x0dd4, 0x0dd5, 0x0dd6, 0x0dd7, - 0x0dd8, 0x0dd9, 0x0dd9, 0x0ddb, 0x0dd9, 0x0dd9, 0x0dd9, 0x0ddf, - 0x0de0, 0x0de1, 0x0de2, 0x0de3, 0x0de4, 0x0de5, 0x0de6, 0x0de7, - 0x0de8, 0x0de9, 0x0dea, 0x0deb, 0x0dec, 0x0ded, 0x0dee, 0x0def, - 0x0df0, 0x0df1, 0x0df2, 0x0df3, 0x0df4, 0x0df5, 0x0df6, 0x0df7, - 0x0df8, 0x0df9, 0x0dfa, 0x0dfb, 0x0dfc, 0x0dfd, 0x0dfe, 0x0dff, - }; - -static const unsigned short gNormalizeTable0e00[] = { - /* U+0e00 */ - 0x0e00, 0x0e01, 0x0e02, 0x0e03, 0x0e04, 0x0e05, 0x0e06, 0x0e07, - 0x0e08, 0x0e09, 0x0e0a, 0x0e0b, 0x0e0c, 0x0e0d, 0x0e0e, 0x0e0f, - 0x0e10, 0x0e11, 0x0e12, 0x0e13, 0x0e14, 0x0e15, 0x0e16, 0x0e17, - 0x0e18, 0x0e19, 0x0e1a, 0x0e1b, 0x0e1c, 0x0e1d, 0x0e1e, 0x0e1f, - 0x0e20, 0x0e21, 0x0e22, 0x0e23, 0x0e24, 0x0e25, 0x0e26, 0x0e27, - 0x0e28, 0x0e29, 0x0e2a, 0x0e2b, 0x0e2c, 0x0e2d, 0x0e2e, 0x0e2f, - 0x0e30, 0x0e31, 0x0e32, 0x0e4d, 0x0e34, 0x0e35, 0x0e36, 0x0e37, - 0x0e38, 0x0e39, 0x0e3a, 0x0e3b, 0x0e3c, 0x0e3d, 0x0e3e, 0x0e3f, - }; - -static const unsigned short gNormalizeTable0e80[] = { - /* U+0e80 */ - 0x0e80, 0x0e81, 0x0e82, 0x0e83, 0x0e84, 0x0e85, 0x0e86, 0x0e87, - 0x0e88, 0x0e89, 0x0e8a, 0x0e8b, 0x0e8c, 0x0e8d, 0x0e8e, 0x0e8f, - 0x0e90, 0x0e91, 0x0e92, 0x0e93, 0x0e94, 0x0e95, 0x0e96, 0x0e97, - 0x0e98, 0x0e99, 0x0e9a, 0x0e9b, 0x0e9c, 0x0e9d, 0x0e9e, 0x0e9f, - 0x0ea0, 0x0ea1, 0x0ea2, 0x0ea3, 0x0ea4, 0x0ea5, 0x0ea6, 0x0ea7, - 0x0ea8, 0x0ea9, 0x0eaa, 0x0eab, 0x0eac, 0x0ead, 0x0eae, 0x0eaf, - 0x0eb0, 0x0eb1, 0x0eb2, 0x0ecd, 0x0eb4, 0x0eb5, 0x0eb6, 0x0eb7, - 0x0eb8, 0x0eb9, 0x0eba, 0x0ebb, 0x0ebc, 0x0ebd, 0x0ebe, 0x0ebf, - }; - -static const unsigned short gNormalizeTable0ec0[] = { - /* U+0ec0 */ - 0x0ec0, 0x0ec1, 0x0ec2, 0x0ec3, 0x0ec4, 0x0ec5, 0x0ec6, 0x0ec7, - 0x0ec8, 0x0ec9, 0x0eca, 0x0ecb, 0x0ecc, 0x0ecd, 0x0ece, 0x0ecf, - 0x0ed0, 0x0ed1, 0x0ed2, 0x0ed3, 0x0ed4, 0x0ed5, 0x0ed6, 0x0ed7, - 0x0ed8, 0x0ed9, 0x0eda, 0x0edb, 0x0eab, 0x0eab, 0x0ede, 0x0edf, - 0x0ee0, 0x0ee1, 0x0ee2, 0x0ee3, 0x0ee4, 0x0ee5, 0x0ee6, 0x0ee7, - 0x0ee8, 0x0ee9, 0x0eea, 0x0eeb, 0x0eec, 0x0eed, 0x0eee, 0x0eef, - 0x0ef0, 0x0ef1, 0x0ef2, 0x0ef3, 0x0ef4, 0x0ef5, 0x0ef6, 0x0ef7, - 0x0ef8, 0x0ef9, 0x0efa, 0x0efb, 0x0efc, 0x0efd, 0x0efe, 0x0eff, - }; - -static const unsigned short gNormalizeTable0f00[] = { - /* U+0f00 */ - 0x0f00, 0x0f01, 0x0f02, 0x0f03, 0x0f04, 0x0f05, 0x0f06, 0x0f07, - 0x0f08, 0x0f09, 0x0f0a, 0x0f0b, 0x0f0b, 0x0f0d, 0x0f0e, 0x0f0f, - 0x0f10, 0x0f11, 0x0f12, 0x0f13, 0x0f14, 0x0f15, 0x0f16, 0x0f17, - 0x0f18, 0x0f19, 0x0f1a, 0x0f1b, 0x0f1c, 0x0f1d, 0x0f1e, 0x0f1f, - 0x0f20, 0x0f21, 0x0f22, 0x0f23, 0x0f24, 0x0f25, 0x0f26, 0x0f27, - 0x0f28, 0x0f29, 0x0f2a, 0x0f2b, 0x0f2c, 0x0f2d, 0x0f2e, 0x0f2f, - 0x0f30, 0x0f31, 0x0f32, 0x0f33, 0x0f34, 0x0f35, 0x0f36, 0x0f37, - 0x0f38, 0x0f39, 0x0f3a, 0x0f3b, 0x0f3c, 0x0f3d, 0x0f3e, 0x0f3f, - }; - -static const unsigned short gNormalizeTable0f40[] = { - /* U+0f40 */ - 0x0f40, 0x0f41, 0x0f42, 0x0f42, 0x0f44, 0x0f45, 0x0f46, 0x0f47, - 0x0f48, 0x0f49, 0x0f4a, 0x0f4b, 0x0f4c, 0x0f4c, 0x0f4e, 0x0f4f, - 0x0f50, 0x0f51, 0x0f51, 0x0f53, 0x0f54, 0x0f55, 0x0f56, 0x0f56, - 0x0f58, 0x0f59, 0x0f5a, 0x0f5b, 0x0f5b, 0x0f5d, 0x0f5e, 0x0f5f, - 0x0f60, 0x0f61, 0x0f62, 0x0f63, 0x0f64, 0x0f65, 0x0f66, 0x0f67, - 0x0f68, 0x0f40, 0x0f6a, 0x0f6b, 0x0f6c, 0x0f6d, 0x0f6e, 0x0f6f, - 0x0f70, 0x0f71, 0x0f72, 0x0f71, 0x0f74, 0x0f71, 0x0fb2, 0x0fb2, - 0x0fb3, 0x0fb3, 0x0f7a, 0x0f7b, 0x0f7c, 0x0f7d, 0x0f7e, 0x0f7f, - }; - -static const unsigned short gNormalizeTable0f80[] = { - /* U+0f80 */ - 0x0f80, 0x0f71, 0x0f82, 0x0f83, 0x0f84, 0x0f85, 0x0f86, 0x0f87, - 0x0f88, 0x0f89, 0x0f8a, 0x0f8b, 0x0f8c, 0x0f8d, 0x0f8e, 0x0f8f, - 0x0f90, 0x0f91, 0x0f92, 0x0f92, 0x0f94, 0x0f95, 0x0f96, 0x0f97, - 0x0f98, 0x0f99, 0x0f9a, 0x0f9b, 0x0f9c, 0x0f9c, 0x0f9e, 0x0f9f, - 0x0fa0, 0x0fa1, 0x0fa1, 0x0fa3, 0x0fa4, 0x0fa5, 0x0fa6, 0x0fa6, - 0x0fa8, 0x0fa9, 0x0faa, 0x0fab, 0x0fab, 0x0fad, 0x0fae, 0x0faf, - 0x0fb0, 0x0fb1, 0x0fb2, 0x0fb3, 0x0fb4, 0x0fb5, 0x0fb6, 0x0fb7, - 0x0fb8, 0x0f90, 0x0fba, 0x0fbb, 0x0fbc, 0x0fbd, 0x0fbe, 0x0fbf, - }; - -static const unsigned short gNormalizeTable1000[] = { - /* U+1000 */ - 0x1000, 0x1001, 0x1002, 0x1003, 0x1004, 0x1005, 0x1006, 0x1007, - 0x1008, 0x1009, 0x100a, 0x100b, 0x100c, 0x100d, 0x100e, 0x100f, - 0x1010, 0x1011, 0x1012, 0x1013, 0x1014, 0x1015, 0x1016, 0x1017, - 0x1018, 0x1019, 0x101a, 0x101b, 0x101c, 0x101d, 0x101e, 0x101f, - 0x1020, 0x1021, 0x1022, 0x1023, 0x1024, 0x1025, 0x1025, 0x1027, - 0x1028, 0x1029, 0x102a, 0x102b, 0x102c, 0x102d, 0x102e, 0x102f, - 0x1030, 0x1031, 0x1032, 0x1033, 0x1034, 0x1035, 0x1036, 0x1037, - 0x1038, 0x1039, 0x103a, 0x103b, 0x103c, 0x103d, 0x103e, 0x103f, - }; - -static const unsigned short gNormalizeTable1080[] = { - /* U+1080 */ - 0x1080, 0x1081, 0x1082, 0x1083, 0x1084, 0x1085, 0x1086, 0x1087, - 0x1088, 0x1089, 0x108a, 0x108b, 0x108c, 0x108d, 0x108e, 0x108f, - 0x1090, 0x1091, 0x1092, 0x1093, 0x1094, 0x1095, 0x1096, 0x1097, - 0x1098, 0x1099, 0x109a, 0x109b, 0x109c, 0x109d, 0x109e, 0x109f, - 0x2d00, 0x2d01, 0x2d02, 0x2d03, 0x2d04, 0x2d05, 0x2d06, 0x2d07, - 0x2d08, 0x2d09, 0x2d0a, 0x2d0b, 0x2d0c, 0x2d0d, 0x2d0e, 0x2d0f, - 0x2d10, 0x2d11, 0x2d12, 0x2d13, 0x2d14, 0x2d15, 0x2d16, 0x2d17, - 0x2d18, 0x2d19, 0x2d1a, 0x2d1b, 0x2d1c, 0x2d1d, 0x2d1e, 0x2d1f, - }; - -static const unsigned short gNormalizeTable10c0[] = { - /* U+10c0 */ - 0x2d20, 0x2d21, 0x2d22, 0x2d23, 0x2d24, 0x2d25, 0x10c6, 0x10c7, - 0x10c8, 0x10c9, 0x10ca, 0x10cb, 0x10cc, 0x10cd, 0x10ce, 0x10cf, - 0x10d0, 0x10d1, 0x10d2, 0x10d3, 0x10d4, 0x10d5, 0x10d6, 0x10d7, - 0x10d8, 0x10d9, 0x10da, 0x10db, 0x10dc, 0x10dd, 0x10de, 0x10df, - 0x10e0, 0x10e1, 0x10e2, 0x10e3, 0x10e4, 0x10e5, 0x10e6, 0x10e7, - 0x10e8, 0x10e9, 0x10ea, 0x10eb, 0x10ec, 0x10ed, 0x10ee, 0x10ef, - 0x10f0, 0x10f1, 0x10f2, 0x10f3, 0x10f4, 0x10f5, 0x10f6, 0x10f7, - 0x10f8, 0x10f9, 0x10fa, 0x10fb, 0x10dc, 0x10fd, 0x10fe, 0x10ff, - }; - -static const unsigned short gNormalizeTable1140[] = { - /* U+1140 */ - 0x1140, 0x1141, 0x1142, 0x1143, 0x1144, 0x1145, 0x1146, 0x1147, - 0x1148, 0x1149, 0x114a, 0x114b, 0x114c, 0x114d, 0x114e, 0x114f, - 0x1150, 0x1151, 0x1152, 0x1153, 0x1154, 0x1155, 0x1156, 0x1157, - 0x1158, 0x1159, 0x115a, 0x115b, 0x115c, 0x115d, 0x115e, 0x0020, - 0x0020, 0x1161, 0x1162, 0x1163, 0x1164, 0x1165, 0x1166, 0x1167, - 0x1168, 0x1169, 0x116a, 0x116b, 0x116c, 0x116d, 0x116e, 0x116f, - 0x1170, 0x1171, 0x1172, 0x1173, 0x1174, 0x1175, 0x1176, 0x1177, - 0x1178, 0x1179, 0x117a, 0x117b, 0x117c, 0x117d, 0x117e, 0x117f, - }; - -static const unsigned short gNormalizeTable1780[] = { - /* U+1780 */ - 0x1780, 0x1781, 0x1782, 0x1783, 0x1784, 0x1785, 0x1786, 0x1787, - 0x1788, 0x1789, 0x178a, 0x178b, 0x178c, 0x178d, 0x178e, 0x178f, - 0x1790, 0x1791, 0x1792, 0x1793, 0x1794, 0x1795, 0x1796, 0x1797, - 0x1798, 0x1799, 0x179a, 0x179b, 0x179c, 0x179d, 0x179e, 0x179f, - 0x17a0, 0x17a1, 0x17a2, 0x17a3, 0x17a4, 0x17a5, 0x17a6, 0x17a7, - 0x17a8, 0x17a9, 0x17aa, 0x17ab, 0x17ac, 0x17ad, 0x17ae, 0x17af, - 0x17b0, 0x17b1, 0x17b2, 0x17b3, 0x0020, 0x0020, 0x17b6, 0x17b7, - 0x17b8, 0x17b9, 0x17ba, 0x17bb, 0x17bc, 0x17bd, 0x17be, 0x17bf, - }; - -static const unsigned short gNormalizeTable1800[] = { - /* U+1800 */ - 0x1800, 0x1801, 0x1802, 0x1803, 0x1804, 0x1805, 0x1806, 0x1807, - 0x1808, 0x1809, 0x180a, 0x0020, 0x0020, 0x0020, 0x180e, 0x180f, - 0x1810, 0x1811, 0x1812, 0x1813, 0x1814, 0x1815, 0x1816, 0x1817, - 0x1818, 0x1819, 0x181a, 0x181b, 0x181c, 0x181d, 0x181e, 0x181f, - 0x1820, 0x1821, 0x1822, 0x1823, 0x1824, 0x1825, 0x1826, 0x1827, - 0x1828, 0x1829, 0x182a, 0x182b, 0x182c, 0x182d, 0x182e, 0x182f, - 0x1830, 0x1831, 0x1832, 0x1833, 0x1834, 0x1835, 0x1836, 0x1837, - 0x1838, 0x1839, 0x183a, 0x183b, 0x183c, 0x183d, 0x183e, 0x183f, - }; - -static const unsigned short gNormalizeTable1b00[] = { - /* U+1b00 */ - 0x1b00, 0x1b01, 0x1b02, 0x1b03, 0x1b04, 0x1b05, 0x1b05, 0x1b07, - 0x1b07, 0x1b09, 0x1b09, 0x1b0b, 0x1b0b, 0x1b0d, 0x1b0d, 0x1b0f, - 0x1b10, 0x1b11, 0x1b11, 0x1b13, 0x1b14, 0x1b15, 0x1b16, 0x1b17, - 0x1b18, 0x1b19, 0x1b1a, 0x1b1b, 0x1b1c, 0x1b1d, 0x1b1e, 0x1b1f, - 0x1b20, 0x1b21, 0x1b22, 0x1b23, 0x1b24, 0x1b25, 0x1b26, 0x1b27, - 0x1b28, 0x1b29, 0x1b2a, 0x1b2b, 0x1b2c, 0x1b2d, 0x1b2e, 0x1b2f, - 0x1b30, 0x1b31, 0x1b32, 0x1b33, 0x1b34, 0x1b35, 0x1b36, 0x1b37, - 0x1b38, 0x1b39, 0x1b3a, 0x1b3a, 0x1b3c, 0x1b3c, 0x1b3e, 0x1b3f, - }; - -static const unsigned short gNormalizeTable1b40[] = { - /* U+1b40 */ - 0x1b3e, 0x1b3f, 0x1b42, 0x1b42, 0x1b44, 0x1b45, 0x1b46, 0x1b47, - 0x1b48, 0x1b49, 0x1b4a, 0x1b4b, 0x1b4c, 0x1b4d, 0x1b4e, 0x1b4f, - 0x1b50, 0x1b51, 0x1b52, 0x1b53, 0x1b54, 0x1b55, 0x1b56, 0x1b57, - 0x1b58, 0x1b59, 0x1b5a, 0x1b5b, 0x1b5c, 0x1b5d, 0x1b5e, 0x1b5f, - 0x1b60, 0x1b61, 0x1b62, 0x1b63, 0x1b64, 0x1b65, 0x1b66, 0x1b67, - 0x1b68, 0x1b69, 0x1b6a, 0x1b6b, 0x1b6c, 0x1b6d, 0x1b6e, 0x1b6f, - 0x1b70, 0x1b71, 0x1b72, 0x1b73, 0x1b74, 0x1b75, 0x1b76, 0x1b77, - 0x1b78, 0x1b79, 0x1b7a, 0x1b7b, 0x1b7c, 0x1b7d, 0x1b7e, 0x1b7f, - }; - -static const unsigned short gNormalizeTable1d00[] = { - /* U+1d00 */ - 0x1d00, 0x1d01, 0x1d02, 0x1d03, 0x1d04, 0x1d05, 0x1d06, 0x1d07, - 0x1d08, 0x1d09, 0x1d0a, 0x1d0b, 0x1d0c, 0x1d0d, 0x1d0e, 0x1d0f, - 0x1d10, 0x1d11, 0x1d12, 0x1d13, 0x1d14, 0x1d15, 0x1d16, 0x1d17, - 0x1d18, 0x1d19, 0x1d1a, 0x1d1b, 0x1d1c, 0x1d1d, 0x1d1e, 0x1d1f, - 0x1d20, 0x1d21, 0x1d22, 0x1d23, 0x1d24, 0x1d25, 0x1d26, 0x1d27, - 0x1d28, 0x1d29, 0x1d2a, 0x1d2b, 0x0061, 0x00e6, 0x0062, 0x1d2f, - 0x0064, 0x0065, 0x01dd, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, - 0x006c, 0x006d, 0x006e, 0x1d3b, 0x006f, 0x0223, 0x0070, 0x0072, - }; - -static const unsigned short gNormalizeTable1d40[] = { - /* U+1d40 */ - 0x0074, 0x0075, 0x0077, 0x0061, 0x0250, 0x0251, 0x1d02, 0x0062, - 0x0064, 0x0065, 0x0259, 0x025b, 0x025c, 0x0067, 0x1d4e, 0x006b, - 0x006d, 0x014b, 0x006f, 0x0254, 0x1d16, 0x1d17, 0x0070, 0x0074, - 0x0075, 0x1d1d, 0x026f, 0x0076, 0x1d25, 0x03b2, 0x03b3, 0x03b4, - 0x03c6, 0x03c7, 0x0069, 0x0072, 0x0075, 0x0076, 0x03b2, 0x03b3, - 0x03c1, 0x03c6, 0x03c7, 0x1d6b, 0x1d6c, 0x1d6d, 0x1d6e, 0x1d6f, - 0x1d70, 0x1d71, 0x1d72, 0x1d73, 0x1d74, 0x1d75, 0x1d76, 0x1d77, - 0x043d, 0x1d79, 0x1d7a, 0x1d7b, 0x1d7c, 0x1d7d, 0x1d7e, 0x1d7f, - }; - -static const unsigned short gNormalizeTable1d80[] = { - /* U+1d80 */ - 0x1d80, 0x1d81, 0x1d82, 0x1d83, 0x1d84, 0x1d85, 0x1d86, 0x1d87, - 0x1d88, 0x1d89, 0x1d8a, 0x1d8b, 0x1d8c, 0x1d8d, 0x1d8e, 0x1d8f, - 0x1d90, 0x1d91, 0x1d92, 0x1d93, 0x1d94, 0x1d95, 0x1d96, 0x1d97, - 0x1d98, 0x1d99, 0x1d9a, 0x0252, 0x0063, 0x0255, 0x00f0, 0x025c, - 0x0066, 0x025f, 0x0261, 0x0265, 0x0268, 0x0269, 0x026a, 0x1d7b, - 0x029d, 0x026d, 0x1d85, 0x029f, 0x0271, 0x0270, 0x0272, 0x0273, - 0x0274, 0x0275, 0x0278, 0x0282, 0x0283, 0x01ab, 0x0289, 0x028a, - 0x1d1c, 0x028b, 0x028c, 0x007a, 0x0290, 0x0291, 0x0292, 0x03b8, - }; - -static const unsigned short gNormalizeTable1e00[] = { - /* U+1e00 */ - 0x0061, 0x0061, 0x0062, 0x0062, 0x0062, 0x0062, 0x0062, 0x0062, - 0x0063, 0x0063, 0x0064, 0x0064, 0x0064, 0x0064, 0x0064, 0x0064, - 0x0064, 0x0064, 0x0064, 0x0064, 0x0065, 0x0065, 0x0065, 0x0065, - 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0066, 0x0066, - 0x0067, 0x0067, 0x0068, 0x0068, 0x0068, 0x0068, 0x0068, 0x0068, - 0x0068, 0x0068, 0x0068, 0x0068, 0x0069, 0x0069, 0x0069, 0x0069, - 0x006b, 0x006b, 0x006b, 0x006b, 0x006b, 0x006b, 0x006c, 0x006c, - 0x006c, 0x006c, 0x006c, 0x006c, 0x006c, 0x006c, 0x006d, 0x006d, - }; - -static const unsigned short gNormalizeTable1e40[] = { - /* U+1e40 */ - 0x006d, 0x006d, 0x006d, 0x006d, 0x006e, 0x006e, 0x006e, 0x006e, - 0x006e, 0x006e, 0x006e, 0x006e, 0x006f, 0x006f, 0x006f, 0x006f, - 0x006f, 0x006f, 0x006f, 0x006f, 0x0070, 0x0070, 0x0070, 0x0070, - 0x0072, 0x0072, 0x0072, 0x0072, 0x0072, 0x0072, 0x0072, 0x0072, - 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, - 0x0073, 0x0073, 0x0074, 0x0074, 0x0074, 0x0074, 0x0074, 0x0074, - 0x0074, 0x0074, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, - 0x0075, 0x0075, 0x0075, 0x0075, 0x0076, 0x0076, 0x0076, 0x0076, - }; - -static const unsigned short gNormalizeTable1e80[] = { - /* U+1e80 */ - 0x0077, 0x0077, 0x0077, 0x0077, 0x0077, 0x0077, 0x0077, 0x0077, - 0x0077, 0x0077, 0x0078, 0x0078, 0x0078, 0x0078, 0x0079, 0x0079, - 0x007a, 0x007a, 0x007a, 0x007a, 0x007a, 0x007a, 0x0068, 0x0074, - 0x0077, 0x0079, 0x0061, 0x0073, 0x1e9c, 0x1e9d, 0x0073, 0x1e9f, - 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, - 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, - 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, - 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, - }; - -static const unsigned short gNormalizeTable1ec0[] = { - /* U+1ec0 */ - 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, - 0x0069, 0x0069, 0x0069, 0x0069, 0x006f, 0x006f, 0x006f, 0x006f, - 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, - 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, - 0x006f, 0x006f, 0x006f, 0x006f, 0x0075, 0x0075, 0x0075, 0x0075, - 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, - 0x0075, 0x0075, 0x0079, 0x0079, 0x0079, 0x0079, 0x0079, 0x0079, - 0x0079, 0x0079, 0x1efb, 0x1efb, 0x1efd, 0x1efd, 0x1eff, 0x1eff, - }; - -static const unsigned short gNormalizeTable1f00[] = { - /* U+1f00 */ - 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, - 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, - 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x1f16, 0x1f17, - 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x1f1e, 0x1f1f, - 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, - 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, - 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, - 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, - }; - -static const unsigned short gNormalizeTable1f40[] = { - /* U+1f40 */ - 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x1f46, 0x1f47, - 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x1f4e, 0x1f4f, - 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c5, - 0x1f58, 0x03c5, 0x1f5a, 0x03c5, 0x1f5c, 0x03c5, 0x1f5e, 0x03c5, - 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, - 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, - 0x03b1, 0x03b1, 0x03b5, 0x03b5, 0x03b7, 0x03b7, 0x03b9, 0x03b9, - 0x03bf, 0x03bf, 0x03c5, 0x03c5, 0x03c9, 0x03c9, 0x1f7e, 0x1f7f, - }; - -static const unsigned short gNormalizeTable1f80[] = { - /* U+1f80 */ - 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, - 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, - 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, - 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, - 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, - 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, - 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x1fb5, 0x03b1, 0x03b1, - 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x0020, 0x03b9, 0x0020, - }; - -static const unsigned short gNormalizeTable1fc0[] = { - /* U+1fc0 */ - 0x0020, 0x0020, 0x03b7, 0x03b7, 0x03b7, 0x1fc5, 0x03b7, 0x03b7, - 0x03b5, 0x03b5, 0x03b7, 0x03b7, 0x03b7, 0x0020, 0x0020, 0x0020, - 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x1fd4, 0x1fd5, 0x03b9, 0x03b9, - 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x1fdc, 0x0020, 0x0020, 0x0020, - 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c1, 0x03c1, 0x03c5, 0x03c5, - 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c1, 0x0020, 0x0020, 0x0060, - 0x1ff0, 0x1ff1, 0x03c9, 0x03c9, 0x03c9, 0x1ff5, 0x03c9, 0x03c9, - 0x03bf, 0x03bf, 0x03c9, 0x03c9, 0x03c9, 0x0020, 0x0020, 0x1fff, - }; - -static const unsigned short gNormalizeTable2000[] = { - /* U+2000 */ - 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, - 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, - 0x2010, 0x2010, 0x2012, 0x2013, 0x2014, 0x2015, 0x2016, 0x0020, - 0x2018, 0x2019, 0x201a, 0x201b, 0x201c, 0x201d, 0x201e, 0x201f, - 0x2020, 0x2021, 0x2022, 0x2023, 0x002e, 0x002e, 0x002e, 0x2027, - 0x2028, 0x2029, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, - 0x2030, 0x2031, 0x2032, 0x2032, 0x2032, 0x2035, 0x2035, 0x2035, - 0x2038, 0x2039, 0x203a, 0x203b, 0x0021, 0x203d, 0x0020, 0x203f, - }; - -static const unsigned short gNormalizeTable2040[] = { - /* U+2040 */ - 0x2040, 0x2041, 0x2042, 0x2043, 0x2044, 0x2045, 0x2046, 0x003f, - 0x003f, 0x0021, 0x204a, 0x204b, 0x204c, 0x204d, 0x204e, 0x204f, - 0x2050, 0x2051, 0x2052, 0x2053, 0x2054, 0x2055, 0x2056, 0x2032, - 0x2058, 0x2059, 0x205a, 0x205b, 0x205c, 0x205d, 0x205e, 0x0020, - 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, - 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, - 0x0030, 0x0069, 0x2072, 0x2073, 0x0034, 0x0035, 0x0036, 0x0037, - 0x0038, 0x0039, 0x002b, 0x2212, 0x003d, 0x0028, 0x0029, 0x006e, - }; - -static const unsigned short gNormalizeTable2080[] = { - /* U+2080 */ - 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, - 0x0038, 0x0039, 0x002b, 0x2212, 0x003d, 0x0028, 0x0029, 0x208f, - 0x0061, 0x0065, 0x006f, 0x0078, 0x0259, 0x2095, 0x2096, 0x2097, - 0x2098, 0x2099, 0x209a, 0x209b, 0x209c, 0x209d, 0x209e, 0x209f, - 0x20a0, 0x20a1, 0x20a2, 0x20a3, 0x20a4, 0x20a5, 0x20a6, 0x20a7, - 0x0072, 0x20a9, 0x20aa, 0x20ab, 0x20ac, 0x20ad, 0x20ae, 0x20af, - 0x20b0, 0x20b1, 0x20b2, 0x20b3, 0x20b4, 0x20b5, 0x20b6, 0x20b7, - 0x20b8, 0x20b9, 0x20ba, 0x20bb, 0x20bc, 0x20bd, 0x20be, 0x20bf, - }; - -static const unsigned short gNormalizeTable2100[] = { - /* U+2100 */ - 0x0061, 0x0061, 0x0063, 0x00b0, 0x2104, 0x0063, 0x0063, 0x025b, - 0x2108, 0x00b0, 0x0067, 0x0068, 0x0068, 0x0068, 0x0068, 0x0127, - 0x0069, 0x0069, 0x006c, 0x006c, 0x2114, 0x006e, 0x006e, 0x2117, - 0x2118, 0x0070, 0x0071, 0x0072, 0x0072, 0x0072, 0x211e, 0x211f, - 0x0073, 0x0074, 0x0074, 0x2123, 0x007a, 0x2125, 0x03c9, 0x2127, - 0x007a, 0x2129, 0x006b, 0x0061, 0x0062, 0x0063, 0x212e, 0x0065, - 0x0065, 0x0066, 0x214e, 0x006d, 0x006f, 0x05d0, 0x05d1, 0x05d2, - 0x05d3, 0x0069, 0x213a, 0x0066, 0x03c0, 0x03b3, 0x03b3, 0x03c0, - }; - -static const unsigned short gNormalizeTable2140[] = { - /* U+2140 */ - 0x2211, 0x2141, 0x2142, 0x2143, 0x2144, 0x0064, 0x0064, 0x0065, - 0x0069, 0x006a, 0x214a, 0x214b, 0x214c, 0x214d, 0x214e, 0x214f, - 0x0031, 0x0031, 0x0031, 0x0031, 0x0032, 0x0031, 0x0032, 0x0033, - 0x0034, 0x0031, 0x0035, 0x0031, 0x0033, 0x0035, 0x0037, 0x0031, - 0x0069, 0x0069, 0x0069, 0x0069, 0x0076, 0x0076, 0x0076, 0x0076, - 0x0069, 0x0078, 0x0078, 0x0078, 0x006c, 0x0063, 0x0064, 0x006d, - 0x0069, 0x0069, 0x0069, 0x0069, 0x0076, 0x0076, 0x0076, 0x0076, - 0x0069, 0x0078, 0x0078, 0x0078, 0x006c, 0x0063, 0x0064, 0x006d, - }; - -static const unsigned short gNormalizeTable2180[] = { - /* U+2180 */ - 0x2180, 0x2181, 0x2182, 0x2184, 0x2184, 0x2185, 0x2186, 0x2187, - 0x2188, 0x0030, 0x218a, 0x218b, 0x218c, 0x218d, 0x218e, 0x218f, - 0x2190, 0x2191, 0x2192, 0x2193, 0x2194, 0x2195, 0x2196, 0x2197, - 0x2198, 0x2199, 0x2190, 0x2192, 0x219c, 0x219d, 0x219e, 0x219f, - 0x21a0, 0x21a1, 0x21a2, 0x21a3, 0x21a4, 0x21a5, 0x21a6, 0x21a7, - 0x21a8, 0x21a9, 0x21aa, 0x21ab, 0x21ac, 0x21ad, 0x2194, 0x21af, - 0x21b0, 0x21b1, 0x21b2, 0x21b3, 0x21b4, 0x21b5, 0x21b6, 0x21b7, - 0x21b8, 0x21b9, 0x21ba, 0x21bb, 0x21bc, 0x21bd, 0x21be, 0x21bf, - }; - -static const unsigned short gNormalizeTable21c0[] = { - /* U+21c0 */ - 0x21c0, 0x21c1, 0x21c2, 0x21c3, 0x21c4, 0x21c5, 0x21c6, 0x21c7, - 0x21c8, 0x21c9, 0x21ca, 0x21cb, 0x21cc, 0x21d0, 0x21d4, 0x21d2, - 0x21d0, 0x21d1, 0x21d2, 0x21d3, 0x21d4, 0x21d5, 0x21d6, 0x21d7, - 0x21d8, 0x21d9, 0x21da, 0x21db, 0x21dc, 0x21dd, 0x21de, 0x21df, - 0x21e0, 0x21e1, 0x21e2, 0x21e3, 0x21e4, 0x21e5, 0x21e6, 0x21e7, - 0x21e8, 0x21e9, 0x21ea, 0x21eb, 0x21ec, 0x21ed, 0x21ee, 0x21ef, - 0x21f0, 0x21f1, 0x21f2, 0x21f3, 0x21f4, 0x21f5, 0x21f6, 0x21f7, - 0x21f8, 0x21f9, 0x21fa, 0x21fb, 0x21fc, 0x21fd, 0x21fe, 0x21ff, - }; - -static const unsigned short gNormalizeTable2200[] = { - /* U+2200 */ - 0x2200, 0x2201, 0x2202, 0x2203, 0x2203, 0x2205, 0x2206, 0x2207, - 0x2208, 0x2208, 0x220a, 0x220b, 0x220b, 0x220d, 0x220e, 0x220f, - 0x2210, 0x2211, 0x2212, 0x2213, 0x2214, 0x2215, 0x2216, 0x2217, - 0x2218, 0x2219, 0x221a, 0x221b, 0x221c, 0x221d, 0x221e, 0x221f, - 0x2220, 0x2221, 0x2222, 0x2223, 0x2223, 0x2225, 0x2225, 0x2227, - 0x2228, 0x2229, 0x222a, 0x222b, 0x222b, 0x222b, 0x222e, 0x222e, - 0x222e, 0x2231, 0x2232, 0x2233, 0x2234, 0x2235, 0x2236, 0x2237, - 0x2238, 0x2239, 0x223a, 0x223b, 0x223c, 0x223d, 0x223e, 0x223f, - }; - -static const unsigned short gNormalizeTable2240[] = { - /* U+2240 */ - 0x2240, 0x223c, 0x2242, 0x2243, 0x2243, 0x2245, 0x2246, 0x2245, - 0x2248, 0x2248, 0x224a, 0x224b, 0x224c, 0x224d, 0x224e, 0x224f, - 0x2250, 0x2251, 0x2252, 0x2253, 0x2254, 0x2255, 0x2256, 0x2257, - 0x2258, 0x2259, 0x225a, 0x225b, 0x225c, 0x225d, 0x225e, 0x225f, - 0x003d, 0x2261, 0x2261, 0x2263, 0x2264, 0x2265, 0x2266, 0x2267, - 0x2268, 0x2269, 0x226a, 0x226b, 0x226c, 0x224d, 0x003c, 0x003e, - 0x2264, 0x2265, 0x2272, 0x2273, 0x2272, 0x2273, 0x2276, 0x2277, - 0x2276, 0x2277, 0x227a, 0x227b, 0x227c, 0x227d, 0x227e, 0x227f, - }; - -static const unsigned short gNormalizeTable2280[] = { - /* U+2280 */ - 0x227a, 0x227b, 0x2282, 0x2283, 0x2282, 0x2283, 0x2286, 0x2287, - 0x2286, 0x2287, 0x228a, 0x228b, 0x228c, 0x228d, 0x228e, 0x228f, - 0x2290, 0x2291, 0x2292, 0x2293, 0x2294, 0x2295, 0x2296, 0x2297, - 0x2298, 0x2299, 0x229a, 0x229b, 0x229c, 0x229d, 0x229e, 0x229f, - 0x22a0, 0x22a1, 0x22a2, 0x22a3, 0x22a4, 0x22a5, 0x22a6, 0x22a7, - 0x22a8, 0x22a9, 0x22aa, 0x22ab, 0x22a2, 0x22a8, 0x22a9, 0x22ab, - 0x22b0, 0x22b1, 0x22b2, 0x22b3, 0x22b4, 0x22b5, 0x22b6, 0x22b7, - 0x22b8, 0x22b9, 0x22ba, 0x22bb, 0x22bc, 0x22bd, 0x22be, 0x22bf, - }; - -static const unsigned short gNormalizeTable22c0[] = { - /* U+22c0 */ - 0x22c0, 0x22c1, 0x22c2, 0x22c3, 0x22c4, 0x22c5, 0x22c6, 0x22c7, - 0x22c8, 0x22c9, 0x22ca, 0x22cb, 0x22cc, 0x22cd, 0x22ce, 0x22cf, - 0x22d0, 0x22d1, 0x22d2, 0x22d3, 0x22d4, 0x22d5, 0x22d6, 0x22d7, - 0x22d8, 0x22d9, 0x22da, 0x22db, 0x22dc, 0x22dd, 0x22de, 0x22df, - 0x227c, 0x227d, 0x2291, 0x2292, 0x22e4, 0x22e5, 0x22e6, 0x22e7, - 0x22e8, 0x22e9, 0x22b2, 0x22b3, 0x22b4, 0x22b5, 0x22ee, 0x22ef, - 0x22f0, 0x22f1, 0x22f2, 0x22f3, 0x22f4, 0x22f5, 0x22f6, 0x22f7, - 0x22f8, 0x22f9, 0x22fa, 0x22fb, 0x22fc, 0x22fd, 0x22fe, 0x22ff, - }; - -static const unsigned short gNormalizeTable2300[] = { - /* U+2300 */ - 0x2300, 0x2301, 0x2302, 0x2303, 0x2304, 0x2305, 0x2306, 0x2307, - 0x2308, 0x2309, 0x230a, 0x230b, 0x230c, 0x230d, 0x230e, 0x230f, - 0x2310, 0x2311, 0x2312, 0x2313, 0x2314, 0x2315, 0x2316, 0x2317, - 0x2318, 0x2319, 0x231a, 0x231b, 0x231c, 0x231d, 0x231e, 0x231f, - 0x2320, 0x2321, 0x2322, 0x2323, 0x2324, 0x2325, 0x2326, 0x2327, - 0x2328, 0x3008, 0x3009, 0x232b, 0x232c, 0x232d, 0x232e, 0x232f, - 0x2330, 0x2331, 0x2332, 0x2333, 0x2334, 0x2335, 0x2336, 0x2337, - 0x2338, 0x2339, 0x233a, 0x233b, 0x233c, 0x233d, 0x233e, 0x233f, - }; - -static const unsigned short gNormalizeTable2440[] = { - /* U+2440 */ - 0x2440, 0x2441, 0x2442, 0x2443, 0x2444, 0x2445, 0x2446, 0x2447, - 0x2448, 0x2449, 0x244a, 0x244b, 0x244c, 0x244d, 0x244e, 0x244f, - 0x2450, 0x2451, 0x2452, 0x2453, 0x2454, 0x2455, 0x2456, 0x2457, - 0x2458, 0x2459, 0x245a, 0x245b, 0x245c, 0x245d, 0x245e, 0x245f, - 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, - 0x0039, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, - 0x0031, 0x0031, 0x0031, 0x0032, 0x0028, 0x0028, 0x0028, 0x0028, - 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, - }; - -static const unsigned short gNormalizeTable2480[] = { - /* U+2480 */ - 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, - 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, - 0x0039, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, - 0x0031, 0x0031, 0x0031, 0x0032, 0x0028, 0x0028, 0x0028, 0x0028, - 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, - 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, - 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0061, 0x0062, - 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, - }; - -static const unsigned short gNormalizeTable24c0[] = { - /* U+24c0 */ - 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, 0x0070, 0x0071, 0x0072, - 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, - 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, - 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, 0x0070, - 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, - 0x0079, 0x007a, 0x0030, 0x24eb, 0x24ec, 0x24ed, 0x24ee, 0x24ef, - 0x24f0, 0x24f1, 0x24f2, 0x24f3, 0x24f4, 0x24f5, 0x24f6, 0x24f7, - 0x24f8, 0x24f9, 0x24fa, 0x24fb, 0x24fc, 0x24fd, 0x24fe, 0x24ff, - }; - -static const unsigned short gNormalizeTable2a00[] = { - /* U+2a00 */ - 0x2a00, 0x2a01, 0x2a02, 0x2a03, 0x2a04, 0x2a05, 0x2a06, 0x2a07, - 0x2a08, 0x2a09, 0x2a0a, 0x2a0b, 0x222b, 0x2a0d, 0x2a0e, 0x2a0f, - 0x2a10, 0x2a11, 0x2a12, 0x2a13, 0x2a14, 0x2a15, 0x2a16, 0x2a17, - 0x2a18, 0x2a19, 0x2a1a, 0x2a1b, 0x2a1c, 0x2a1d, 0x2a1e, 0x2a1f, - 0x2a20, 0x2a21, 0x2a22, 0x2a23, 0x2a24, 0x2a25, 0x2a26, 0x2a27, - 0x2a28, 0x2a29, 0x2a2a, 0x2a2b, 0x2a2c, 0x2a2d, 0x2a2e, 0x2a2f, - 0x2a30, 0x2a31, 0x2a32, 0x2a33, 0x2a34, 0x2a35, 0x2a36, 0x2a37, - 0x2a38, 0x2a39, 0x2a3a, 0x2a3b, 0x2a3c, 0x2a3d, 0x2a3e, 0x2a3f, - }; - -static const unsigned short gNormalizeTable2a40[] = { - /* U+2a40 */ - 0x2a40, 0x2a41, 0x2a42, 0x2a43, 0x2a44, 0x2a45, 0x2a46, 0x2a47, - 0x2a48, 0x2a49, 0x2a4a, 0x2a4b, 0x2a4c, 0x2a4d, 0x2a4e, 0x2a4f, - 0x2a50, 0x2a51, 0x2a52, 0x2a53, 0x2a54, 0x2a55, 0x2a56, 0x2a57, - 0x2a58, 0x2a59, 0x2a5a, 0x2a5b, 0x2a5c, 0x2a5d, 0x2a5e, 0x2a5f, - 0x2a60, 0x2a61, 0x2a62, 0x2a63, 0x2a64, 0x2a65, 0x2a66, 0x2a67, - 0x2a68, 0x2a69, 0x2a6a, 0x2a6b, 0x2a6c, 0x2a6d, 0x2a6e, 0x2a6f, - 0x2a70, 0x2a71, 0x2a72, 0x2a73, 0x003a, 0x003d, 0x003d, 0x2a77, - 0x2a78, 0x2a79, 0x2a7a, 0x2a7b, 0x2a7c, 0x2a7d, 0x2a7e, 0x2a7f, - }; - -static const unsigned short gNormalizeTable2ac0[] = { - /* U+2ac0 */ - 0x2ac0, 0x2ac1, 0x2ac2, 0x2ac3, 0x2ac4, 0x2ac5, 0x2ac6, 0x2ac7, - 0x2ac8, 0x2ac9, 0x2aca, 0x2acb, 0x2acc, 0x2acd, 0x2ace, 0x2acf, - 0x2ad0, 0x2ad1, 0x2ad2, 0x2ad3, 0x2ad4, 0x2ad5, 0x2ad6, 0x2ad7, - 0x2ad8, 0x2ad9, 0x2ada, 0x2adb, 0x2add, 0x2add, 0x2ade, 0x2adf, - 0x2ae0, 0x2ae1, 0x2ae2, 0x2ae3, 0x2ae4, 0x2ae5, 0x2ae6, 0x2ae7, - 0x2ae8, 0x2ae9, 0x2aea, 0x2aeb, 0x2aec, 0x2aed, 0x2aee, 0x2aef, - 0x2af0, 0x2af1, 0x2af2, 0x2af3, 0x2af4, 0x2af5, 0x2af6, 0x2af7, - 0x2af8, 0x2af9, 0x2afa, 0x2afb, 0x2afc, 0x2afd, 0x2afe, 0x2aff, - }; - -static const unsigned short gNormalizeTable2c00[] = { - /* U+2c00 */ - 0x2c30, 0x2c31, 0x2c32, 0x2c33, 0x2c34, 0x2c35, 0x2c36, 0x2c37, - 0x2c38, 0x2c39, 0x2c3a, 0x2c3b, 0x2c3c, 0x2c3d, 0x2c3e, 0x2c3f, - 0x2c40, 0x2c41, 0x2c42, 0x2c43, 0x2c44, 0x2c45, 0x2c46, 0x2c47, - 0x2c48, 0x2c49, 0x2c4a, 0x2c4b, 0x2c4c, 0x2c4d, 0x2c4e, 0x2c4f, - 0x2c50, 0x2c51, 0x2c52, 0x2c53, 0x2c54, 0x2c55, 0x2c56, 0x2c57, - 0x2c58, 0x2c59, 0x2c5a, 0x2c5b, 0x2c5c, 0x2c5d, 0x2c5e, 0x2c2f, - 0x2c30, 0x2c31, 0x2c32, 0x2c33, 0x2c34, 0x2c35, 0x2c36, 0x2c37, - 0x2c38, 0x2c39, 0x2c3a, 0x2c3b, 0x2c3c, 0x2c3d, 0x2c3e, 0x2c3f, - }; - -static const unsigned short gNormalizeTable2c40[] = { - /* U+2c40 */ - 0x2c40, 0x2c41, 0x2c42, 0x2c43, 0x2c44, 0x2c45, 0x2c46, 0x2c47, - 0x2c48, 0x2c49, 0x2c4a, 0x2c4b, 0x2c4c, 0x2c4d, 0x2c4e, 0x2c4f, - 0x2c50, 0x2c51, 0x2c52, 0x2c53, 0x2c54, 0x2c55, 0x2c56, 0x2c57, - 0x2c58, 0x2c59, 0x2c5a, 0x2c5b, 0x2c5c, 0x2c5d, 0x2c5e, 0x2c5f, - 0x2c61, 0x2c61, 0x026b, 0x1d7d, 0x027d, 0x2c65, 0x2c66, 0x2c68, - 0x2c68, 0x2c6a, 0x2c6a, 0x2c6c, 0x2c6c, 0x0251, 0x0271, 0x0250, - 0x0252, 0x2c71, 0x2c73, 0x2c73, 0x2c74, 0x2c76, 0x2c76, 0x2c77, - 0x2c78, 0x2c79, 0x2c7a, 0x2c7b, 0x006a, 0x0076, 0x023f, 0x0240, - }; - -static const unsigned short gNormalizeTable2c80[] = { - /* U+2c80 */ - 0x2c81, 0x2c81, 0x2c83, 0x2c83, 0x2c85, 0x2c85, 0x2c87, 0x2c87, - 0x2c89, 0x2c89, 0x2c8b, 0x2c8b, 0x2c8d, 0x2c8d, 0x2c8f, 0x2c8f, - 0x2c91, 0x2c91, 0x2c93, 0x2c93, 0x2c95, 0x2c95, 0x2c97, 0x2c97, - 0x2c99, 0x2c99, 0x2c9b, 0x2c9b, 0x2c9d, 0x2c9d, 0x2c9f, 0x2c9f, - 0x2ca1, 0x2ca1, 0x2ca3, 0x2ca3, 0x2ca5, 0x2ca5, 0x2ca7, 0x2ca7, - 0x2ca9, 0x2ca9, 0x2cab, 0x2cab, 0x2cad, 0x2cad, 0x2caf, 0x2caf, - 0x2cb1, 0x2cb1, 0x2cb3, 0x2cb3, 0x2cb5, 0x2cb5, 0x2cb7, 0x2cb7, - 0x2cb9, 0x2cb9, 0x2cbb, 0x2cbb, 0x2cbd, 0x2cbd, 0x2cbf, 0x2cbf, - }; - -static const unsigned short gNormalizeTable2cc0[] = { - /* U+2cc0 */ - 0x2cc1, 0x2cc1, 0x2cc3, 0x2cc3, 0x2cc5, 0x2cc5, 0x2cc7, 0x2cc7, - 0x2cc9, 0x2cc9, 0x2ccb, 0x2ccb, 0x2ccd, 0x2ccd, 0x2ccf, 0x2ccf, - 0x2cd1, 0x2cd1, 0x2cd3, 0x2cd3, 0x2cd5, 0x2cd5, 0x2cd7, 0x2cd7, - 0x2cd9, 0x2cd9, 0x2cdb, 0x2cdb, 0x2cdd, 0x2cdd, 0x2cdf, 0x2cdf, - 0x2ce1, 0x2ce1, 0x2ce3, 0x2ce3, 0x2ce4, 0x2ce5, 0x2ce6, 0x2ce7, - 0x2ce8, 0x2ce9, 0x2cea, 0x2cec, 0x2cec, 0x2cee, 0x2cee, 0x2cef, - 0x2cf0, 0x2cf1, 0x2cf2, 0x2cf3, 0x2cf4, 0x2cf5, 0x2cf6, 0x2cf7, - 0x2cf8, 0x2cf9, 0x2cfa, 0x2cfb, 0x2cfc, 0x2cfd, 0x2cfe, 0x2cff, - }; - -static const unsigned short gNormalizeTable2d40[] = { - /* U+2d40 */ - 0x2d40, 0x2d41, 0x2d42, 0x2d43, 0x2d44, 0x2d45, 0x2d46, 0x2d47, - 0x2d48, 0x2d49, 0x2d4a, 0x2d4b, 0x2d4c, 0x2d4d, 0x2d4e, 0x2d4f, - 0x2d50, 0x2d51, 0x2d52, 0x2d53, 0x2d54, 0x2d55, 0x2d56, 0x2d57, - 0x2d58, 0x2d59, 0x2d5a, 0x2d5b, 0x2d5c, 0x2d5d, 0x2d5e, 0x2d5f, - 0x2d60, 0x2d61, 0x2d62, 0x2d63, 0x2d64, 0x2d65, 0x2d66, 0x2d67, - 0x2d68, 0x2d69, 0x2d6a, 0x2d6b, 0x2d6c, 0x2d6d, 0x2d6e, 0x2d61, - 0x2d70, 0x2d71, 0x2d72, 0x2d73, 0x2d74, 0x2d75, 0x2d76, 0x2d77, - 0x2d78, 0x2d79, 0x2d7a, 0x2d7b, 0x2d7c, 0x2d7d, 0x2d7e, 0x2d7f, - }; - -static const unsigned short gNormalizeTable2e80[] = { - /* U+2e80 */ - 0x2e80, 0x2e81, 0x2e82, 0x2e83, 0x2e84, 0x2e85, 0x2e86, 0x2e87, - 0x2e88, 0x2e89, 0x2e8a, 0x2e8b, 0x2e8c, 0x2e8d, 0x2e8e, 0x2e8f, - 0x2e90, 0x2e91, 0x2e92, 0x2e93, 0x2e94, 0x2e95, 0x2e96, 0x2e97, - 0x2e98, 0x2e99, 0x2e9a, 0x2e9b, 0x2e9c, 0x2e9d, 0x2e9e, 0x6bcd, - 0x2ea0, 0x2ea1, 0x2ea2, 0x2ea3, 0x2ea4, 0x2ea5, 0x2ea6, 0x2ea7, - 0x2ea8, 0x2ea9, 0x2eaa, 0x2eab, 0x2eac, 0x2ead, 0x2eae, 0x2eaf, - 0x2eb0, 0x2eb1, 0x2eb2, 0x2eb3, 0x2eb4, 0x2eb5, 0x2eb6, 0x2eb7, - 0x2eb8, 0x2eb9, 0x2eba, 0x2ebb, 0x2ebc, 0x2ebd, 0x2ebe, 0x2ebf, - }; - -static const unsigned short gNormalizeTable2ec0[] = { - /* U+2ec0 */ - 0x2ec0, 0x2ec1, 0x2ec2, 0x2ec3, 0x2ec4, 0x2ec5, 0x2ec6, 0x2ec7, - 0x2ec8, 0x2ec9, 0x2eca, 0x2ecb, 0x2ecc, 0x2ecd, 0x2ece, 0x2ecf, - 0x2ed0, 0x2ed1, 0x2ed2, 0x2ed3, 0x2ed4, 0x2ed5, 0x2ed6, 0x2ed7, - 0x2ed8, 0x2ed9, 0x2eda, 0x2edb, 0x2edc, 0x2edd, 0x2ede, 0x2edf, - 0x2ee0, 0x2ee1, 0x2ee2, 0x2ee3, 0x2ee4, 0x2ee5, 0x2ee6, 0x2ee7, - 0x2ee8, 0x2ee9, 0x2eea, 0x2eeb, 0x2eec, 0x2eed, 0x2eee, 0x2eef, - 0x2ef0, 0x2ef1, 0x2ef2, 0x9f9f, 0x2ef4, 0x2ef5, 0x2ef6, 0x2ef7, - 0x2ef8, 0x2ef9, 0x2efa, 0x2efb, 0x2efc, 0x2efd, 0x2efe, 0x2eff, - }; - -static const unsigned short gNormalizeTable2f00[] = { - /* U+2f00 */ - 0x4e00, 0x4e28, 0x4e36, 0x4e3f, 0x4e59, 0x4e85, 0x4e8c, 0x4ea0, - 0x4eba, 0x513f, 0x5165, 0x516b, 0x5182, 0x5196, 0x51ab, 0x51e0, - 0x51f5, 0x5200, 0x529b, 0x52f9, 0x5315, 0x531a, 0x5338, 0x5341, - 0x535c, 0x5369, 0x5382, 0x53b6, 0x53c8, 0x53e3, 0x56d7, 0x571f, - 0x58eb, 0x5902, 0x590a, 0x5915, 0x5927, 0x5973, 0x5b50, 0x5b80, - 0x5bf8, 0x5c0f, 0x5c22, 0x5c38, 0x5c6e, 0x5c71, 0x5ddb, 0x5de5, - 0x5df1, 0x5dfe, 0x5e72, 0x5e7a, 0x5e7f, 0x5ef4, 0x5efe, 0x5f0b, - 0x5f13, 0x5f50, 0x5f61, 0x5f73, 0x5fc3, 0x6208, 0x6236, 0x624b, - }; - -static const unsigned short gNormalizeTable2f40[] = { - /* U+2f40 */ - 0x652f, 0x6534, 0x6587, 0x6597, 0x65a4, 0x65b9, 0x65e0, 0x65e5, - 0x66f0, 0x6708, 0x6728, 0x6b20, 0x6b62, 0x6b79, 0x6bb3, 0x6bcb, - 0x6bd4, 0x6bdb, 0x6c0f, 0x6c14, 0x6c34, 0x706b, 0x722a, 0x7236, - 0x723b, 0x723f, 0x7247, 0x7259, 0x725b, 0x72ac, 0x7384, 0x7389, - 0x74dc, 0x74e6, 0x7518, 0x751f, 0x7528, 0x7530, 0x758b, 0x7592, - 0x7676, 0x767d, 0x76ae, 0x76bf, 0x76ee, 0x77db, 0x77e2, 0x77f3, - 0x793a, 0x79b8, 0x79be, 0x7a74, 0x7acb, 0x7af9, 0x7c73, 0x7cf8, - 0x7f36, 0x7f51, 0x7f8a, 0x7fbd, 0x8001, 0x800c, 0x8012, 0x8033, - }; - -static const unsigned short gNormalizeTable2f80[] = { - /* U+2f80 */ - 0x807f, 0x8089, 0x81e3, 0x81ea, 0x81f3, 0x81fc, 0x820c, 0x821b, - 0x821f, 0x826e, 0x8272, 0x8278, 0x864d, 0x866b, 0x8840, 0x884c, - 0x8863, 0x897e, 0x898b, 0x89d2, 0x8a00, 0x8c37, 0x8c46, 0x8c55, - 0x8c78, 0x8c9d, 0x8d64, 0x8d70, 0x8db3, 0x8eab, 0x8eca, 0x8f9b, - 0x8fb0, 0x8fb5, 0x9091, 0x9149, 0x91c6, 0x91cc, 0x91d1, 0x9577, - 0x9580, 0x961c, 0x96b6, 0x96b9, 0x96e8, 0x9751, 0x975e, 0x9762, - 0x9769, 0x97cb, 0x97ed, 0x97f3, 0x9801, 0x98a8, 0x98db, 0x98df, - 0x9996, 0x9999, 0x99ac, 0x9aa8, 0x9ad8, 0x9adf, 0x9b25, 0x9b2f, - }; - -static const unsigned short gNormalizeTable2fc0[] = { - /* U+2fc0 */ - 0x9b32, 0x9b3c, 0x9b5a, 0x9ce5, 0x9e75, 0x9e7f, 0x9ea5, 0x9ebb, - 0x9ec3, 0x9ecd, 0x9ed1, 0x9ef9, 0x9efd, 0x9f0e, 0x9f13, 0x9f20, - 0x9f3b, 0x9f4a, 0x9f52, 0x9f8d, 0x9f9c, 0x9fa0, 0x2fd6, 0x2fd7, - 0x2fd8, 0x2fd9, 0x2fda, 0x2fdb, 0x2fdc, 0x2fdd, 0x2fde, 0x2fdf, - 0x2fe0, 0x2fe1, 0x2fe2, 0x2fe3, 0x2fe4, 0x2fe5, 0x2fe6, 0x2fe7, - 0x2fe8, 0x2fe9, 0x2fea, 0x2feb, 0x2fec, 0x2fed, 0x2fee, 0x2fef, - 0x2ff0, 0x2ff1, 0x2ff2, 0x2ff3, 0x2ff4, 0x2ff5, 0x2ff6, 0x2ff7, - 0x2ff8, 0x2ff9, 0x2ffa, 0x2ffb, 0x2ffc, 0x2ffd, 0x2ffe, 0x2fff, - }; - -static const unsigned short gNormalizeTable3000[] = { - /* U+3000 */ - 0x0020, 0x3001, 0x3002, 0x3003, 0x3004, 0x3005, 0x3006, 0x3007, - 0x3008, 0x3009, 0x300a, 0x300b, 0x300c, 0x300d, 0x300e, 0x300f, - 0x3010, 0x3011, 0x3012, 0x3013, 0x3014, 0x3015, 0x3016, 0x3017, - 0x3018, 0x3019, 0x301a, 0x301b, 0x301c, 0x301d, 0x301e, 0x301f, - 0x3020, 0x3021, 0x3022, 0x3023, 0x3024, 0x3025, 0x3026, 0x3027, - 0x3028, 0x3029, 0x302a, 0x302b, 0x302c, 0x302d, 0x302e, 0x302f, - 0x3030, 0x3031, 0x3032, 0x3033, 0x3034, 0x3035, 0x3012, 0x3037, - 0x5341, 0x5344, 0x5345, 0x303b, 0x303c, 0x303d, 0x303e, 0x303f, - }; - -static const unsigned short gNormalizeTable3040[] = { - /* U+3040 */ - 0x3040, 0x3041, 0x3042, 0x3043, 0x3044, 0x3045, 0x3046, 0x3047, - 0x3048, 0x3049, 0x304a, 0x304b, 0x304b, 0x304d, 0x304d, 0x304f, - 0x304f, 0x3051, 0x3051, 0x3053, 0x3053, 0x3055, 0x3055, 0x3057, - 0x3057, 0x3059, 0x3059, 0x305b, 0x305b, 0x305d, 0x305d, 0x305f, - 0x305f, 0x3061, 0x3061, 0x3063, 0x3064, 0x3064, 0x3066, 0x3066, - 0x3068, 0x3068, 0x306a, 0x306b, 0x306c, 0x306d, 0x306e, 0x306f, - 0x306f, 0x306f, 0x3072, 0x3072, 0x3072, 0x3075, 0x3075, 0x3075, - 0x3078, 0x3078, 0x3078, 0x307b, 0x307b, 0x307b, 0x307e, 0x307f, - }; - -static const unsigned short gNormalizeTable3080[] = { - /* U+3080 */ - 0x3080, 0x3081, 0x3082, 0x3083, 0x3084, 0x3085, 0x3086, 0x3087, - 0x3088, 0x3089, 0x308a, 0x308b, 0x308c, 0x308d, 0x308e, 0x308f, - 0x3090, 0x3091, 0x3092, 0x3093, 0x3046, 0x3095, 0x3096, 0x3097, - 0x3098, 0x3099, 0x309a, 0x0020, 0x0020, 0x309d, 0x309d, 0x3088, - 0x30a0, 0x30a1, 0x30a2, 0x30a3, 0x30a4, 0x30a5, 0x30a6, 0x30a7, - 0x30a8, 0x30a9, 0x30aa, 0x30ab, 0x30ab, 0x30ad, 0x30ad, 0x30af, - 0x30af, 0x30b1, 0x30b1, 0x30b3, 0x30b3, 0x30b5, 0x30b5, 0x30b7, - 0x30b7, 0x30b9, 0x30b9, 0x30bb, 0x30bb, 0x30bd, 0x30bd, 0x30bf, - }; - -static const unsigned short gNormalizeTable30c0[] = { - /* U+30c0 */ - 0x30bf, 0x30c1, 0x30c1, 0x30c3, 0x30c4, 0x30c4, 0x30c6, 0x30c6, - 0x30c8, 0x30c8, 0x30ca, 0x30cb, 0x30cc, 0x30cd, 0x30ce, 0x30cf, - 0x30cf, 0x30cf, 0x30d2, 0x30d2, 0x30d2, 0x30d5, 0x30d5, 0x30d5, - 0x30d8, 0x30d8, 0x30d8, 0x30db, 0x30db, 0x30db, 0x30de, 0x30df, - 0x30e0, 0x30e1, 0x30e2, 0x30e3, 0x30e4, 0x30e5, 0x30e6, 0x30e7, - 0x30e8, 0x30e9, 0x30ea, 0x30eb, 0x30ec, 0x30ed, 0x30ee, 0x30ef, - 0x30f0, 0x30f1, 0x30f2, 0x30f3, 0x30a6, 0x30f5, 0x30f6, 0x30ef, - 0x30f0, 0x30f1, 0x30f2, 0x30fb, 0x30fc, 0x30fd, 0x30fd, 0x30b3, - }; - -static const unsigned short gNormalizeTable3100[] = { - /* U+3100 */ - 0x3100, 0x3101, 0x3102, 0x3103, 0x3104, 0x3105, 0x3106, 0x3107, - 0x3108, 0x3109, 0x310a, 0x310b, 0x310c, 0x310d, 0x310e, 0x310f, - 0x3110, 0x3111, 0x3112, 0x3113, 0x3114, 0x3115, 0x3116, 0x3117, - 0x3118, 0x3119, 0x311a, 0x311b, 0x311c, 0x311d, 0x311e, 0x311f, - 0x3120, 0x3121, 0x3122, 0x3123, 0x3124, 0x3125, 0x3126, 0x3127, - 0x3128, 0x3129, 0x312a, 0x312b, 0x312c, 0x312d, 0x312e, 0x312f, - 0x3130, 0x1100, 0x1101, 0x11aa, 0x1102, 0x11ac, 0x11ad, 0x1103, - 0x1104, 0x1105, 0x11b0, 0x11b1, 0x11b2, 0x11b3, 0x11b4, 0x11b5, - }; - -static const unsigned short gNormalizeTable3140[] = { - /* U+3140 */ - 0x111a, 0x1106, 0x1107, 0x1108, 0x1121, 0x1109, 0x110a, 0x110b, - 0x110c, 0x110d, 0x110e, 0x110f, 0x1110, 0x1111, 0x1112, 0x1161, - 0x1162, 0x1163, 0x1164, 0x1165, 0x1166, 0x1167, 0x1168, 0x1169, - 0x116a, 0x116b, 0x116c, 0x116d, 0x116e, 0x116f, 0x1170, 0x1171, - 0x1172, 0x1173, 0x1174, 0x1175, 0x0020, 0x1114, 0x1115, 0x11c7, - 0x11c8, 0x11cc, 0x11ce, 0x11d3, 0x11d7, 0x11d9, 0x111c, 0x11dd, - 0x11df, 0x111d, 0x111e, 0x1120, 0x1122, 0x1123, 0x1127, 0x1129, - 0x112b, 0x112c, 0x112d, 0x112e, 0x112f, 0x1132, 0x1136, 0x1140, - }; - -static const unsigned short gNormalizeTable3180[] = { - /* U+3180 */ - 0x1147, 0x114c, 0x11f1, 0x11f2, 0x1157, 0x1158, 0x1159, 0x1184, - 0x1185, 0x1188, 0x1191, 0x1192, 0x1194, 0x119e, 0x11a1, 0x318f, - 0x3190, 0x3191, 0x4e00, 0x4e8c, 0x4e09, 0x56db, 0x4e0a, 0x4e2d, - 0x4e0b, 0x7532, 0x4e59, 0x4e19, 0x4e01, 0x5929, 0x5730, 0x4eba, - 0x31a0, 0x31a1, 0x31a2, 0x31a3, 0x31a4, 0x31a5, 0x31a6, 0x31a7, - 0x31a8, 0x31a9, 0x31aa, 0x31ab, 0x31ac, 0x31ad, 0x31ae, 0x31af, - 0x31b0, 0x31b1, 0x31b2, 0x31b3, 0x31b4, 0x31b5, 0x31b6, 0x31b7, - 0x31b8, 0x31b9, 0x31ba, 0x31bb, 0x31bc, 0x31bd, 0x31be, 0x31bf, - }; - -static const unsigned short gNormalizeTable3200[] = { - /* U+3200 */ - 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, - 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, - 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, - 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x321f, - 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, - 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, - 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, - 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, - }; - -static const unsigned short gNormalizeTable3240[] = { - /* U+3240 */ - 0x0028, 0x0028, 0x0028, 0x0028, 0x554f, 0x5e7c, 0x6587, 0x7b8f, - 0x3248, 0x3249, 0x324a, 0x324b, 0x324c, 0x324d, 0x324e, 0x324f, - 0x0070, 0x0032, 0x0032, 0x0032, 0x0032, 0x0032, 0x0032, 0x0032, - 0x0032, 0x0032, 0x0033, 0x0033, 0x0033, 0x0033, 0x0033, 0x0033, - 0x1100, 0x1102, 0x1103, 0x1105, 0x1106, 0x1107, 0x1109, 0x110b, - 0x110c, 0x110e, 0x110f, 0x1110, 0x1111, 0x1112, 0x1100, 0x1102, - 0x1103, 0x1105, 0x1106, 0x1107, 0x1109, 0x110b, 0x110c, 0x110e, - 0x110f, 0x1110, 0x1111, 0x1112, 0x110e, 0x110c, 0x110b, 0x327f, - }; - -static const unsigned short gNormalizeTable3280[] = { - /* U+3280 */ - 0x4e00, 0x4e8c, 0x4e09, 0x56db, 0x4e94, 0x516d, 0x4e03, 0x516b, - 0x4e5d, 0x5341, 0x6708, 0x706b, 0x6c34, 0x6728, 0x91d1, 0x571f, - 0x65e5, 0x682a, 0x6709, 0x793e, 0x540d, 0x7279, 0x8ca1, 0x795d, - 0x52b4, 0x79d8, 0x7537, 0x5973, 0x9069, 0x512a, 0x5370, 0x6ce8, - 0x9805, 0x4f11, 0x5199, 0x6b63, 0x4e0a, 0x4e2d, 0x4e0b, 0x5de6, - 0x53f3, 0x533b, 0x5b97, 0x5b66, 0x76e3, 0x4f01, 0x8cc7, 0x5354, - 0x591c, 0x0033, 0x0033, 0x0033, 0x0033, 0x0034, 0x0034, 0x0034, - 0x0034, 0x0034, 0x0034, 0x0034, 0x0034, 0x0034, 0x0034, 0x0035, - }; - -static const unsigned short gNormalizeTable32c0[] = { - /* U+32c0 */ - 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, - 0x0039, 0x0031, 0x0031, 0x0031, 0x0068, 0x0065, 0x0065, 0x006c, - 0x30a2, 0x30a4, 0x30a6, 0x30a8, 0x30aa, 0x30ab, 0x30ad, 0x30af, - 0x30b1, 0x30b3, 0x30b5, 0x30b7, 0x30b9, 0x30bb, 0x30bd, 0x30bf, - 0x30c1, 0x30c4, 0x30c6, 0x30c8, 0x30ca, 0x30cb, 0x30cc, 0x30cd, - 0x30ce, 0x30cf, 0x30d2, 0x30d5, 0x30d8, 0x30db, 0x30de, 0x30df, - 0x30e0, 0x30e1, 0x30e2, 0x30e4, 0x30e6, 0x30e8, 0x30e9, 0x30ea, - 0x30eb, 0x30ec, 0x30ed, 0x30ef, 0x30f0, 0x30f1, 0x30f2, 0x32ff, - }; - -static const unsigned short gNormalizeTable3300[] = { - /* U+3300 */ - 0x30a2, 0x30a2, 0x30a2, 0x30a2, 0x30a4, 0x30a4, 0x30a6, 0x30a8, - 0x30a8, 0x30aa, 0x30aa, 0x30ab, 0x30ab, 0x30ab, 0x30ab, 0x30ab, - 0x30ad, 0x30ad, 0x30ad, 0x30ad, 0x30ad, 0x30ad, 0x30ad, 0x30ad, - 0x30af, 0x30af, 0x30af, 0x30af, 0x30b1, 0x30b3, 0x30b3, 0x30b5, - 0x30b5, 0x30b7, 0x30bb, 0x30bb, 0x30bf, 0x30c6, 0x30c8, 0x30c8, - 0x30ca, 0x30ce, 0x30cf, 0x30cf, 0x30cf, 0x30cf, 0x30d2, 0x30d2, - 0x30d2, 0x30d2, 0x30d5, 0x30d5, 0x30d5, 0x30d5, 0x30d8, 0x30d8, - 0x30d8, 0x30d8, 0x30d8, 0x30d8, 0x30d8, 0x30db, 0x30db, 0x30db, - }; - -static const unsigned short gNormalizeTable3340[] = { - /* U+3340 */ - 0x30db, 0x30db, 0x30db, 0x30de, 0x30de, 0x30de, 0x30de, 0x30de, - 0x30df, 0x30df, 0x30df, 0x30e1, 0x30e1, 0x30e1, 0x30e4, 0x30e4, - 0x30e6, 0x30ea, 0x30ea, 0x30eb, 0x30eb, 0x30ec, 0x30ec, 0x30ef, - 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, - 0x0038, 0x0039, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, - 0x0031, 0x0031, 0x0031, 0x0031, 0x0032, 0x0032, 0x0032, 0x0032, - 0x0032, 0x0068, 0x0064, 0x0061, 0x0062, 0x006f, 0x0070, 0x0064, - 0x0064, 0x0064, 0x0069, 0x5e73, 0x662d, 0x5927, 0x660e, 0x682a, - }; - -static const unsigned short gNormalizeTable3380[] = { - /* U+3380 */ - 0x0070, 0x006e, 0x03bc, 0x006d, 0x006b, 0x006b, 0x006d, 0x0067, - 0x0063, 0x006b, 0x0070, 0x006e, 0x03bc, 0x03bc, 0x006d, 0x006b, - 0x0068, 0x006b, 0x006d, 0x0067, 0x0074, 0x03bc, 0x006d, 0x0064, - 0x006b, 0x0066, 0x006e, 0x03bc, 0x006d, 0x0063, 0x006b, 0x006d, - 0x0063, 0x006d, 0x006b, 0x006d, 0x0063, 0x006d, 0x006b, 0x006d, - 0x006d, 0x0070, 0x006b, 0x006d, 0x0067, 0x0072, 0x0072, 0x0072, - 0x0070, 0x006e, 0x03bc, 0x006d, 0x0070, 0x006e, 0x03bc, 0x006d, - 0x006b, 0x006d, 0x0070, 0x006e, 0x03bc, 0x006d, 0x006b, 0x006d, - }; - -static const unsigned short gNormalizeTable33c0[] = { - /* U+33c0 */ - 0x006b, 0x006d, 0x0061, 0x0062, 0x0063, 0x0063, 0x0063, 0x0063, - 0x0064, 0x0067, 0x0068, 0x0068, 0x0069, 0x006b, 0x006b, 0x006b, - 0x006c, 0x006c, 0x006c, 0x006c, 0x006d, 0x006d, 0x006d, 0x0070, - 0x0070, 0x0070, 0x0070, 0x0073, 0x0073, 0x0077, 0x0076, 0x0061, - 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, - 0x0039, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, - 0x0031, 0x0031, 0x0031, 0x0032, 0x0032, 0x0032, 0x0032, 0x0032, - 0x0032, 0x0032, 0x0032, 0x0032, 0x0032, 0x0033, 0x0033, 0x0067, - }; - -static const unsigned short gNormalizeTablea640[] = { - /* U+a640 */ - 0xa641, 0xa641, 0xa643, 0xa643, 0xa645, 0xa645, 0xa647, 0xa647, - 0xa649, 0xa649, 0xa64b, 0xa64b, 0xa64d, 0xa64d, 0xa64f, 0xa64f, - 0xa651, 0xa651, 0xa653, 0xa653, 0xa655, 0xa655, 0xa657, 0xa657, - 0xa659, 0xa659, 0xa65b, 0xa65b, 0xa65d, 0xa65d, 0xa65f, 0xa65f, - 0xa660, 0xa661, 0xa663, 0xa663, 0xa665, 0xa665, 0xa667, 0xa667, - 0xa669, 0xa669, 0xa66b, 0xa66b, 0xa66d, 0xa66d, 0xa66e, 0xa66f, - 0xa670, 0xa671, 0xa672, 0xa673, 0xa674, 0xa675, 0xa676, 0xa677, - 0xa678, 0xa679, 0xa67a, 0xa67b, 0xa67c, 0xa67d, 0xa67e, 0xa67f, - }; - -static const unsigned short gNormalizeTablea680[] = { - /* U+a680 */ - 0xa681, 0xa681, 0xa683, 0xa683, 0xa685, 0xa685, 0xa687, 0xa687, - 0xa689, 0xa689, 0xa68b, 0xa68b, 0xa68d, 0xa68d, 0xa68f, 0xa68f, - 0xa691, 0xa691, 0xa693, 0xa693, 0xa695, 0xa695, 0xa697, 0xa697, - 0xa698, 0xa699, 0xa69a, 0xa69b, 0xa69c, 0xa69d, 0xa69e, 0xa69f, - 0xa6a0, 0xa6a1, 0xa6a2, 0xa6a3, 0xa6a4, 0xa6a5, 0xa6a6, 0xa6a7, - 0xa6a8, 0xa6a9, 0xa6aa, 0xa6ab, 0xa6ac, 0xa6ad, 0xa6ae, 0xa6af, - 0xa6b0, 0xa6b1, 0xa6b2, 0xa6b3, 0xa6b4, 0xa6b5, 0xa6b6, 0xa6b7, - 0xa6b8, 0xa6b9, 0xa6ba, 0xa6bb, 0xa6bc, 0xa6bd, 0xa6be, 0xa6bf, - }; - -static const unsigned short gNormalizeTablea700[] = { - /* U+a700 */ - 0xa700, 0xa701, 0xa702, 0xa703, 0xa704, 0xa705, 0xa706, 0xa707, - 0xa708, 0xa709, 0xa70a, 0xa70b, 0xa70c, 0xa70d, 0xa70e, 0xa70f, - 0xa710, 0xa711, 0xa712, 0xa713, 0xa714, 0xa715, 0xa716, 0xa717, - 0xa718, 0xa719, 0xa71a, 0xa71b, 0xa71c, 0xa71d, 0xa71e, 0xa71f, - 0xa720, 0xa721, 0xa723, 0xa723, 0xa725, 0xa725, 0xa727, 0xa727, - 0xa729, 0xa729, 0xa72b, 0xa72b, 0xa72d, 0xa72d, 0xa72f, 0xa72f, - 0xa730, 0xa731, 0xa733, 0xa733, 0xa735, 0xa735, 0xa737, 0xa737, - 0xa739, 0xa739, 0xa73b, 0xa73b, 0xa73d, 0xa73d, 0xa73f, 0xa73f, - }; - -static const unsigned short gNormalizeTablea740[] = { - /* U+a740 */ - 0xa741, 0xa741, 0xa743, 0xa743, 0xa745, 0xa745, 0xa747, 0xa747, - 0xa749, 0xa749, 0xa74b, 0xa74b, 0xa74d, 0xa74d, 0xa74f, 0xa74f, - 0xa751, 0xa751, 0xa753, 0xa753, 0xa755, 0xa755, 0xa757, 0xa757, - 0xa759, 0xa759, 0xa75b, 0xa75b, 0xa75d, 0xa75d, 0xa75f, 0xa75f, - 0xa761, 0xa761, 0xa763, 0xa763, 0xa765, 0xa765, 0xa767, 0xa767, - 0xa769, 0xa769, 0xa76b, 0xa76b, 0xa76d, 0xa76d, 0xa76f, 0xa76f, - 0xa76f, 0xa771, 0xa772, 0xa773, 0xa774, 0xa775, 0xa776, 0xa777, - 0xa778, 0xa77a, 0xa77a, 0xa77c, 0xa77c, 0x1d79, 0xa77f, 0xa77f, - }; - -static const unsigned short gNormalizeTablea780[] = { - /* U+a780 */ - 0xa781, 0xa781, 0xa783, 0xa783, 0xa785, 0xa785, 0xa787, 0xa787, - 0xa788, 0xa789, 0xa78a, 0xa78c, 0xa78c, 0xa78d, 0xa78e, 0xa78f, - 0xa790, 0xa791, 0xa792, 0xa793, 0xa794, 0xa795, 0xa796, 0xa797, - 0xa798, 0xa799, 0xa79a, 0xa79b, 0xa79c, 0xa79d, 0xa79e, 0xa79f, - 0xa7a0, 0xa7a1, 0xa7a2, 0xa7a3, 0xa7a4, 0xa7a5, 0xa7a6, 0xa7a7, - 0xa7a8, 0xa7a9, 0xa7aa, 0xa7ab, 0xa7ac, 0xa7ad, 0xa7ae, 0xa7af, - 0xa7b0, 0xa7b1, 0xa7b2, 0xa7b3, 0xa7b4, 0xa7b5, 0xa7b6, 0xa7b7, - 0xa7b8, 0xa7b9, 0xa7ba, 0xa7bb, 0xa7bc, 0xa7bd, 0xa7be, 0xa7bf, - }; - -static const unsigned short gNormalizeTablef900[] = { - /* U+f900 */ - 0x8c48, 0x66f4, 0x8eca, 0x8cc8, 0x6ed1, 0x4e32, 0x53e5, 0x9f9c, - 0x9f9c, 0x5951, 0x91d1, 0x5587, 0x5948, 0x61f6, 0x7669, 0x7f85, - 0x863f, 0x87ba, 0x88f8, 0x908f, 0x6a02, 0x6d1b, 0x70d9, 0x73de, - 0x843d, 0x916a, 0x99f1, 0x4e82, 0x5375, 0x6b04, 0x721b, 0x862d, - 0x9e1e, 0x5d50, 0x6feb, 0x85cd, 0x8964, 0x62c9, 0x81d8, 0x881f, - 0x5eca, 0x6717, 0x6d6a, 0x72fc, 0x90ce, 0x4f86, 0x51b7, 0x52de, - 0x64c4, 0x6ad3, 0x7210, 0x76e7, 0x8001, 0x8606, 0x865c, 0x8def, - 0x9732, 0x9b6f, 0x9dfa, 0x788c, 0x797f, 0x7da0, 0x83c9, 0x9304, - }; - -static const unsigned short gNormalizeTablef940[] = { - /* U+f940 */ - 0x9e7f, 0x8ad6, 0x58df, 0x5f04, 0x7c60, 0x807e, 0x7262, 0x78ca, - 0x8cc2, 0x96f7, 0x58d8, 0x5c62, 0x6a13, 0x6dda, 0x6f0f, 0x7d2f, - 0x7e37, 0x964b, 0x52d2, 0x808b, 0x51dc, 0x51cc, 0x7a1c, 0x7dbe, - 0x83f1, 0x9675, 0x8b80, 0x62cf, 0x6a02, 0x8afe, 0x4e39, 0x5be7, - 0x6012, 0x7387, 0x7570, 0x5317, 0x78fb, 0x4fbf, 0x5fa9, 0x4e0d, - 0x6ccc, 0x6578, 0x7d22, 0x53c3, 0x585e, 0x7701, 0x8449, 0x8aaa, - 0x6bba, 0x8fb0, 0x6c88, 0x62fe, 0x82e5, 0x63a0, 0x7565, 0x4eae, - 0x5169, 0x51c9, 0x6881, 0x7ce7, 0x826f, 0x8ad2, 0x91cf, 0x52f5, - }; - -static const unsigned short gNormalizeTablef980[] = { - /* U+f980 */ - 0x5442, 0x5973, 0x5eec, 0x65c5, 0x6ffe, 0x792a, 0x95ad, 0x9a6a, - 0x9e97, 0x9ece, 0x529b, 0x66c6, 0x6b77, 0x8f62, 0x5e74, 0x6190, - 0x6200, 0x649a, 0x6f23, 0x7149, 0x7489, 0x79ca, 0x7df4, 0x806f, - 0x8f26, 0x84ee, 0x9023, 0x934a, 0x5217, 0x52a3, 0x54bd, 0x70c8, - 0x88c2, 0x8aaa, 0x5ec9, 0x5ff5, 0x637b, 0x6bae, 0x7c3e, 0x7375, - 0x4ee4, 0x56f9, 0x5be7, 0x5dba, 0x601c, 0x73b2, 0x7469, 0x7f9a, - 0x8046, 0x9234, 0x96f6, 0x9748, 0x9818, 0x4f8b, 0x79ae, 0x91b4, - 0x96b8, 0x60e1, 0x4e86, 0x50da, 0x5bee, 0x5c3f, 0x6599, 0x6a02, - }; - -static const unsigned short gNormalizeTablef9c0[] = { - /* U+f9c0 */ - 0x71ce, 0x7642, 0x84fc, 0x907c, 0x9f8d, 0x6688, 0x962e, 0x5289, - 0x677b, 0x67f3, 0x6d41, 0x6e9c, 0x7409, 0x7559, 0x786b, 0x7d10, - 0x985e, 0x516d, 0x622e, 0x9678, 0x502b, 0x5d19, 0x6dea, 0x8f2a, - 0x5f8b, 0x6144, 0x6817, 0x7387, 0x9686, 0x5229, 0x540f, 0x5c65, - 0x6613, 0x674e, 0x68a8, 0x6ce5, 0x7406, 0x75e2, 0x7f79, 0x88cf, - 0x88e1, 0x91cc, 0x96e2, 0x533f, 0x6eba, 0x541d, 0x71d0, 0x7498, - 0x85fa, 0x96a3, 0x9c57, 0x9e9f, 0x6797, 0x6dcb, 0x81e8, 0x7acb, - 0x7b20, 0x7c92, 0x72c0, 0x7099, 0x8b58, 0x4ec0, 0x8336, 0x523a, - }; - -static const unsigned short gNormalizeTablefa00[] = { - /* U+fa00 */ - 0x5207, 0x5ea6, 0x62d3, 0x7cd6, 0x5b85, 0x6d1e, 0x66b4, 0x8f3b, - 0x884c, 0x964d, 0x898b, 0x5ed3, 0x5140, 0x55c0, 0xfa0e, 0xfa0f, - 0x585a, 0xfa11, 0x6674, 0xfa13, 0xfa14, 0x51de, 0x732a, 0x76ca, - 0x793c, 0x795e, 0x7965, 0x798f, 0x9756, 0x7cbe, 0x7fbd, 0xfa1f, - 0x8612, 0xfa21, 0x8af8, 0xfa23, 0xfa24, 0x9038, 0x90fd, 0xfa27, - 0xfa28, 0xfa29, 0x98ef, 0x98fc, 0x9928, 0x9db4, 0xfa2e, 0xfa2f, - 0x4fae, 0x50e7, 0x514d, 0x52c9, 0x52e4, 0x5351, 0x559d, 0x5606, - 0x5668, 0x5840, 0x58a8, 0x5c64, 0x5c6e, 0x6094, 0x6168, 0x618e, - }; - -static const unsigned short gNormalizeTablefa40[] = { - /* U+fa40 */ - 0x61f2, 0x654f, 0x65e2, 0x6691, 0x6885, 0x6d77, 0x6e1a, 0x6f22, - 0x716e, 0x722b, 0x7422, 0x7891, 0x793e, 0x7949, 0x7948, 0x7950, - 0x7956, 0x795d, 0x798d, 0x798e, 0x7a40, 0x7a81, 0x7bc0, 0x7df4, - 0x7e09, 0x7e41, 0x7f72, 0x8005, 0x81ed, 0x8279, 0x8279, 0x8457, - 0x8910, 0x8996, 0x8b01, 0x8b39, 0x8cd3, 0x8d08, 0x8fb6, 0x9038, - 0x96e3, 0x97ff, 0x983b, 0x6075, 0xfa6c, 0x8218, 0xfa6e, 0xfa6f, - 0x4e26, 0x51b5, 0x5168, 0x4f80, 0x5145, 0x5180, 0x52c7, 0x52fa, - 0x559d, 0x5555, 0x5599, 0x55e2, 0x585a, 0x58b3, 0x5944, 0x5954, - }; - -static const unsigned short gNormalizeTablefa80[] = { - /* U+fa80 */ - 0x5a62, 0x5b28, 0x5ed2, 0x5ed9, 0x5f69, 0x5fad, 0x60d8, 0x614e, - 0x6108, 0x618e, 0x6160, 0x61f2, 0x6234, 0x63c4, 0x641c, 0x6452, - 0x6556, 0x6674, 0x6717, 0x671b, 0x6756, 0x6b79, 0x6bba, 0x6d41, - 0x6edb, 0x6ecb, 0x6f22, 0x701e, 0x716e, 0x77a7, 0x7235, 0x72af, - 0x732a, 0x7471, 0x7506, 0x753b, 0x761d, 0x761f, 0x76ca, 0x76db, - 0x76f4, 0x774a, 0x7740, 0x78cc, 0x7ab1, 0x7bc0, 0x7c7b, 0x7d5b, - 0x7df4, 0x7f3e, 0x8005, 0x8352, 0x83ef, 0x8779, 0x8941, 0x8986, - 0x8996, 0x8abf, 0x8af8, 0x8acb, 0x8b01, 0x8afe, 0x8aed, 0x8b39, - }; - -static const unsigned short gNormalizeTablefac0[] = { - /* U+fac0 */ - 0x8b8a, 0x8d08, 0x8f38, 0x9072, 0x9199, 0x9276, 0x967c, 0x96e3, - 0x9756, 0x97db, 0x97ff, 0x980b, 0x983b, 0x9b12, 0x9f9c, 0xfacf, - 0xfad0, 0xfad1, 0x3b9d, 0x4018, 0x4039, 0xfad5, 0xfad6, 0xfad7, - 0x9f43, 0x9f8e, 0xfada, 0xfadb, 0xfadc, 0xfadd, 0xfade, 0xfadf, - 0xfae0, 0xfae1, 0xfae2, 0xfae3, 0xfae4, 0xfae5, 0xfae6, 0xfae7, - 0xfae8, 0xfae9, 0xfaea, 0xfaeb, 0xfaec, 0xfaed, 0xfaee, 0xfaef, - 0xfaf0, 0xfaf1, 0xfaf2, 0xfaf3, 0xfaf4, 0xfaf5, 0xfaf6, 0xfaf7, - 0xfaf8, 0xfaf9, 0xfafa, 0xfafb, 0xfafc, 0xfafd, 0xfafe, 0xfaff, - }; - -static const unsigned short gNormalizeTablefb00[] = { - /* U+fb00 */ - 0x0066, 0x0066, 0x0066, 0x0066, 0x0066, 0x0073, 0x0073, 0xfb07, - 0xfb08, 0xfb09, 0xfb0a, 0xfb0b, 0xfb0c, 0xfb0d, 0xfb0e, 0xfb0f, - 0xfb10, 0xfb11, 0xfb12, 0x0574, 0x0574, 0x0574, 0x057e, 0x0574, - 0xfb18, 0xfb19, 0xfb1a, 0xfb1b, 0xfb1c, 0x05d9, 0xfb1e, 0x05f2, - 0x05e2, 0x05d0, 0x05d3, 0x05d4, 0x05db, 0x05dc, 0x05dd, 0x05e8, - 0x05ea, 0x002b, 0x05e9, 0x05e9, 0x05e9, 0x05e9, 0x05d0, 0x05d0, - 0x05d0, 0x05d1, 0x05d2, 0x05d3, 0x05d4, 0x05d5, 0x05d6, 0xfb37, - 0x05d8, 0x05d9, 0x05da, 0x05db, 0x05dc, 0xfb3d, 0x05de, 0xfb3f, - }; - -static const unsigned short gNormalizeTablefb40[] = { - /* U+fb40 */ - 0x05e0, 0x05e1, 0xfb42, 0x05e3, 0x05e4, 0xfb45, 0x05e6, 0x05e7, - 0x05e8, 0x05e9, 0x05ea, 0x05d5, 0x05d1, 0x05db, 0x05e4, 0x05d0, - 0x0671, 0x0671, 0x067b, 0x067b, 0x067b, 0x067b, 0x067e, 0x067e, - 0x067e, 0x067e, 0x0680, 0x0680, 0x0680, 0x0680, 0x067a, 0x067a, - 0x067a, 0x067a, 0x067f, 0x067f, 0x067f, 0x067f, 0x0679, 0x0679, - 0x0679, 0x0679, 0x06a4, 0x06a4, 0x06a4, 0x06a4, 0x06a6, 0x06a6, - 0x06a6, 0x06a6, 0x0684, 0x0684, 0x0684, 0x0684, 0x0683, 0x0683, - 0x0683, 0x0683, 0x0686, 0x0686, 0x0686, 0x0686, 0x0687, 0x0687, - }; - -static const unsigned short gNormalizeTablefb80[] = { - /* U+fb80 */ - 0x0687, 0x0687, 0x068d, 0x068d, 0x068c, 0x068c, 0x068e, 0x068e, - 0x0688, 0x0688, 0x0698, 0x0698, 0x0691, 0x0691, 0x06a9, 0x06a9, - 0x06a9, 0x06a9, 0x06af, 0x06af, 0x06af, 0x06af, 0x06b3, 0x06b3, - 0x06b3, 0x06b3, 0x06b1, 0x06b1, 0x06b1, 0x06b1, 0x06ba, 0x06ba, - 0x06bb, 0x06bb, 0x06bb, 0x06bb, 0x06d5, 0x06d5, 0x06c1, 0x06c1, - 0x06c1, 0x06c1, 0x06be, 0x06be, 0x06be, 0x06be, 0x06d2, 0x06d2, - 0x06d2, 0x06d2, 0xfbb2, 0xfbb3, 0xfbb4, 0xfbb5, 0xfbb6, 0xfbb7, - 0xfbb8, 0xfbb9, 0xfbba, 0xfbbb, 0xfbbc, 0xfbbd, 0xfbbe, 0xfbbf, - }; - -static const unsigned short gNormalizeTablefbc0[] = { - /* U+fbc0 */ - 0xfbc0, 0xfbc1, 0xfbc2, 0xfbc3, 0xfbc4, 0xfbc5, 0xfbc6, 0xfbc7, - 0xfbc8, 0xfbc9, 0xfbca, 0xfbcb, 0xfbcc, 0xfbcd, 0xfbce, 0xfbcf, - 0xfbd0, 0xfbd1, 0xfbd2, 0x06ad, 0x06ad, 0x06ad, 0x06ad, 0x06c7, - 0x06c7, 0x06c6, 0x06c6, 0x06c8, 0x06c8, 0x06c7, 0x06cb, 0x06cb, - 0x06c5, 0x06c5, 0x06c9, 0x06c9, 0x06d0, 0x06d0, 0x06d0, 0x06d0, - 0x0649, 0x0649, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, - 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, - 0x064a, 0x064a, 0x064a, 0x064a, 0x06cc, 0x06cc, 0x06cc, 0x06cc, - }; - -static const unsigned short gNormalizeTablefc00[] = { - /* U+fc00 */ - 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x0628, 0x0628, 0x0628, - 0x0628, 0x0628, 0x0628, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, - 0x062a, 0x062b, 0x062b, 0x062b, 0x062b, 0x062c, 0x062c, 0x062d, - 0x062d, 0x062e, 0x062e, 0x062e, 0x0633, 0x0633, 0x0633, 0x0633, - 0x0635, 0x0635, 0x0636, 0x0636, 0x0636, 0x0636, 0x0637, 0x0637, - 0x0638, 0x0639, 0x0639, 0x063a, 0x063a, 0x0641, 0x0641, 0x0641, - 0x0641, 0x0641, 0x0641, 0x0642, 0x0642, 0x0642, 0x0642, 0x0643, - 0x0643, 0x0643, 0x0643, 0x0643, 0x0643, 0x0643, 0x0643, 0x0644, - }; - -static const unsigned short gNormalizeTablefc40[] = { - /* U+fc40 */ - 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0x0645, 0x0645, 0x0645, - 0x0645, 0x0645, 0x0645, 0x0646, 0x0646, 0x0646, 0x0646, 0x0646, - 0x0646, 0x0647, 0x0647, 0x0647, 0x0647, 0x064a, 0x064a, 0x064a, - 0x064a, 0x064a, 0x064a, 0x0630, 0x0631, 0x0649, 0x0020, 0x0020, - 0x0020, 0x0020, 0x0020, 0x0020, 0x064a, 0x064a, 0x064a, 0x064a, - 0x064a, 0x064a, 0x0628, 0x0628, 0x0628, 0x0628, 0x0628, 0x0628, - 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062b, 0x062b, - 0x062b, 0x062b, 0x062b, 0x062b, 0x0641, 0x0641, 0x0642, 0x0642, - }; - -static const unsigned short gNormalizeTablefc80[] = { - /* U+fc80 */ - 0x0643, 0x0643, 0x0643, 0x0643, 0x0643, 0x0644, 0x0644, 0x0644, - 0x0645, 0x0645, 0x0646, 0x0646, 0x0646, 0x0646, 0x0646, 0x0646, - 0x0649, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, - 0x064a, 0x064a, 0x064a, 0x064a, 0x0628, 0x0628, 0x0628, 0x0628, - 0x0628, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062b, 0x062c, - 0x062c, 0x062d, 0x062d, 0x062e, 0x062e, 0x0633, 0x0633, 0x0633, - 0x0633, 0x0635, 0x0635, 0x0635, 0x0636, 0x0636, 0x0636, 0x0636, - 0x0637, 0x0638, 0x0639, 0x0639, 0x063a, 0x063a, 0x0641, 0x0641, - }; - -static const unsigned short gNormalizeTablefcc0[] = { - /* U+fcc0 */ - 0x0641, 0x0641, 0x0642, 0x0642, 0x0643, 0x0643, 0x0643, 0x0643, - 0x0643, 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0x0645, 0x0645, - 0x0645, 0x0645, 0x0646, 0x0646, 0x0646, 0x0646, 0x0646, 0x0647, - 0x0647, 0x0647, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, - 0x064a, 0x0628, 0x0628, 0x062a, 0x062a, 0x062b, 0x062b, 0x0633, - 0x0633, 0x0634, 0x0634, 0x0643, 0x0643, 0x0644, 0x0646, 0x0646, - 0x064a, 0x064a, 0x0640, 0x0640, 0x0640, 0x0637, 0x0637, 0x0639, - 0x0639, 0x063a, 0x063a, 0x0633, 0x0633, 0x0634, 0x0634, 0x062d, - }; - -static const unsigned short gNormalizeTablefd00[] = { - /* U+fd00 */ - 0x062d, 0x062c, 0x062c, 0x062e, 0x062e, 0x0635, 0x0635, 0x0636, - 0x0636, 0x0634, 0x0634, 0x0634, 0x0634, 0x0634, 0x0633, 0x0635, - 0x0636, 0x0637, 0x0637, 0x0639, 0x0639, 0x063a, 0x063a, 0x0633, - 0x0633, 0x0634, 0x0634, 0x062d, 0x062d, 0x062c, 0x062c, 0x062e, - 0x062e, 0x0635, 0x0635, 0x0636, 0x0636, 0x0634, 0x0634, 0x0634, - 0x0634, 0x0634, 0x0633, 0x0635, 0x0636, 0x0634, 0x0634, 0x0634, - 0x0634, 0x0633, 0x0634, 0x0637, 0x0633, 0x0633, 0x0633, 0x0634, - 0x0634, 0x0634, 0x0637, 0x0638, 0x0627, 0x0627, 0xfd3e, 0xfd3f, - }; - -static const unsigned short gNormalizeTablefd40[] = { - /* U+fd40 */ - 0xfd40, 0xfd41, 0xfd42, 0xfd43, 0xfd44, 0xfd45, 0xfd46, 0xfd47, - 0xfd48, 0xfd49, 0xfd4a, 0xfd4b, 0xfd4c, 0xfd4d, 0xfd4e, 0xfd4f, - 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, - 0x062c, 0x062c, 0x062d, 0x062d, 0x0633, 0x0633, 0x0633, 0x0633, - 0x0633, 0x0633, 0x0633, 0x0633, 0x0635, 0x0635, 0x0635, 0x0634, - 0x0634, 0x0634, 0x0634, 0x0634, 0x0634, 0x0634, 0x0636, 0x0636, - 0x0636, 0x0637, 0x0637, 0x0637, 0x0637, 0x0639, 0x0639, 0x0639, - 0x0639, 0x063a, 0x063a, 0x063a, 0x0641, 0x0641, 0x0642, 0x0642, - }; - -static const unsigned short gNormalizeTablefd80[] = { - /* U+fd80 */ - 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, - 0x0644, 0x0645, 0x0645, 0x0645, 0x0645, 0x0645, 0x0645, 0x0645, - 0xfd90, 0xfd91, 0x0645, 0x0647, 0x0647, 0x0646, 0x0646, 0x0646, - 0x0646, 0x0646, 0x0646, 0x0646, 0x064a, 0x064a, 0x0628, 0x062a, - 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062c, 0x062c, 0x062c, - 0x0633, 0x0635, 0x0634, 0x0636, 0x0644, 0x0644, 0x064a, 0x064a, - 0x064a, 0x0645, 0x0642, 0x0646, 0x0642, 0x0644, 0x0639, 0x0643, - 0x0646, 0x0645, 0x0644, 0x0643, 0x0644, 0x0646, 0x062c, 0x062d, - }; - -static const unsigned short gNormalizeTablefdc0[] = { - /* U+fdc0 */ - 0x0645, 0x0641, 0x0628, 0x0643, 0x0639, 0x0635, 0x0633, 0x0646, - 0xfdc8, 0xfdc9, 0xfdca, 0xfdcb, 0xfdcc, 0xfdcd, 0xfdce, 0xfdcf, - 0xfdd0, 0xfdd1, 0xfdd2, 0xfdd3, 0xfdd4, 0xfdd5, 0xfdd6, 0xfdd7, - 0xfdd8, 0xfdd9, 0xfdda, 0xfddb, 0xfddc, 0xfddd, 0xfdde, 0xfddf, - 0xfde0, 0xfde1, 0xfde2, 0xfde3, 0xfde4, 0xfde5, 0xfde6, 0xfde7, - 0xfde8, 0xfde9, 0xfdea, 0xfdeb, 0xfdec, 0xfded, 0xfdee, 0xfdef, - 0x0635, 0x0642, 0x0627, 0x0627, 0x0645, 0x0635, 0x0631, 0x0639, - 0x0648, 0x0635, 0x0635, 0x062c, 0x0631, 0xfdfd, 0xfdfe, 0xfdff, - }; - -static const unsigned short gNormalizeTablefe00[] = { - /* U+fe00 */ - 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, - 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, - 0x002c, 0x3001, 0x3002, 0x003a, 0x003b, 0x0021, 0x003f, 0x3016, - 0x3017, 0x002e, 0xfe1a, 0xfe1b, 0xfe1c, 0xfe1d, 0xfe1e, 0xfe1f, - 0xfe20, 0xfe21, 0xfe22, 0xfe23, 0xfe24, 0xfe25, 0xfe26, 0xfe27, - 0xfe28, 0xfe29, 0xfe2a, 0xfe2b, 0xfe2c, 0xfe2d, 0xfe2e, 0xfe2f, - 0x002e, 0x2014, 0x2013, 0x005f, 0x005f, 0x0028, 0x0029, 0x007b, - 0x007d, 0x3014, 0x3015, 0x3010, 0x3011, 0x300a, 0x300b, 0x3008, - }; - -static const unsigned short gNormalizeTablefe40[] = { - /* U+fe40 */ - 0x3009, 0x300c, 0x300d, 0x300e, 0x300f, 0xfe45, 0xfe46, 0x005b, - 0x005d, 0x0020, 0x0020, 0x0020, 0x0020, 0x005f, 0x005f, 0x005f, - 0x002c, 0x3001, 0x002e, 0xfe53, 0x003b, 0x003a, 0x003f, 0x0021, - 0x2014, 0x0028, 0x0029, 0x007b, 0x007d, 0x3014, 0x3015, 0x0023, - 0x0026, 0x002a, 0x002b, 0x002d, 0x003c, 0x003e, 0x003d, 0xfe67, - 0x005c, 0x0024, 0x0025, 0x0040, 0xfe6c, 0xfe6d, 0xfe6e, 0xfe6f, - 0x0020, 0x0640, 0x0020, 0xfe73, 0x0020, 0xfe75, 0x0020, 0x0640, - 0x0020, 0x0640, 0x0020, 0x0640, 0x0020, 0x0640, 0x0020, 0x0640, - }; - -static const unsigned short gNormalizeTablefe80[] = { - /* U+fe80 */ - 0x0621, 0x0627, 0x0627, 0x0627, 0x0627, 0x0648, 0x0648, 0x0627, - 0x0627, 0x064a, 0x064a, 0x064a, 0x064a, 0x0627, 0x0627, 0x0628, - 0x0628, 0x0628, 0x0628, 0x0629, 0x0629, 0x062a, 0x062a, 0x062a, - 0x062a, 0x062b, 0x062b, 0x062b, 0x062b, 0x062c, 0x062c, 0x062c, - 0x062c, 0x062d, 0x062d, 0x062d, 0x062d, 0x062e, 0x062e, 0x062e, - 0x062e, 0x062f, 0x062f, 0x0630, 0x0630, 0x0631, 0x0631, 0x0632, - 0x0632, 0x0633, 0x0633, 0x0633, 0x0633, 0x0634, 0x0634, 0x0634, - 0x0634, 0x0635, 0x0635, 0x0635, 0x0635, 0x0636, 0x0636, 0x0636, - }; - -static const unsigned short gNormalizeTablefec0[] = { - /* U+fec0 */ - 0x0636, 0x0637, 0x0637, 0x0637, 0x0637, 0x0638, 0x0638, 0x0638, - 0x0638, 0x0639, 0x0639, 0x0639, 0x0639, 0x063a, 0x063a, 0x063a, - 0x063a, 0x0641, 0x0641, 0x0641, 0x0641, 0x0642, 0x0642, 0x0642, - 0x0642, 0x0643, 0x0643, 0x0643, 0x0643, 0x0644, 0x0644, 0x0644, - 0x0644, 0x0645, 0x0645, 0x0645, 0x0645, 0x0646, 0x0646, 0x0646, - 0x0646, 0x0647, 0x0647, 0x0647, 0x0647, 0x0648, 0x0648, 0x0649, - 0x0649, 0x064a, 0x064a, 0x064a, 0x064a, 0x0644, 0x0644, 0x0644, - 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0xfefd, 0xfefe, 0x0020, - }; - -static const unsigned short gNormalizeTableff00[] = { - /* U+ff00 */ - 0xff00, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, - 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, - 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, - 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, - 0x0040, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, - 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, - 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, - 0x0078, 0x0079, 0x007a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, - }; - -static const unsigned short gNormalizeTableff40[] = { - /* U+ff40 */ - 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, - 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, - 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, - 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x2985, - 0x2986, 0x3002, 0x300c, 0x300d, 0x3001, 0x30fb, 0x30f2, 0x30a1, - 0x30a3, 0x30a5, 0x30a7, 0x30a9, 0x30e3, 0x30e5, 0x30e7, 0x30c3, - 0x30fc, 0x30a2, 0x30a4, 0x30a6, 0x30a8, 0x30aa, 0x30ab, 0x30ad, - 0x30af, 0x30b1, 0x30b3, 0x30b5, 0x30b7, 0x30b9, 0x30bb, 0x30bd, - }; - -static const unsigned short gNormalizeTableff80[] = { - /* U+ff80 */ - 0x30bf, 0x30c1, 0x30c4, 0x30c6, 0x30c8, 0x30ca, 0x30cb, 0x30cc, - 0x30cd, 0x30ce, 0x30cf, 0x30d2, 0x30d5, 0x30d8, 0x30db, 0x30de, - 0x30df, 0x30e0, 0x30e1, 0x30e2, 0x30e4, 0x30e6, 0x30e8, 0x30e9, - 0x30ea, 0x30eb, 0x30ec, 0x30ed, 0x30ef, 0x30f3, 0x3099, 0x309a, - 0x0020, 0x1100, 0x1101, 0x11aa, 0x1102, 0x11ac, 0x11ad, 0x1103, - 0x1104, 0x1105, 0x11b0, 0x11b1, 0x11b2, 0x11b3, 0x11b4, 0x11b5, - 0x111a, 0x1106, 0x1107, 0x1108, 0x1121, 0x1109, 0x110a, 0x110b, - 0x110c, 0x110d, 0x110e, 0x110f, 0x1110, 0x1111, 0x1112, 0xffbf, - }; - -static const unsigned short gNormalizeTableffc0[] = { - /* U+ffc0 */ - 0xffc0, 0xffc1, 0x1161, 0x1162, 0x1163, 0x1164, 0x1165, 0x1166, - 0xffc8, 0xffc9, 0x1167, 0x1168, 0x1169, 0x116a, 0x116b, 0x116c, - 0xffd0, 0xffd1, 0x116d, 0x116e, 0x116f, 0x1170, 0x1171, 0x1172, - 0xffd8, 0xffd9, 0x1173, 0x1174, 0x1175, 0xffdd, 0xffde, 0xffdf, - 0x00a2, 0x00a3, 0x00ac, 0x0020, 0x00a6, 0x00a5, 0x20a9, 0xffe7, - 0x2502, 0x2190, 0x2191, 0x2192, 0x2193, 0x25a0, 0x25cb, 0xffef, - 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, - 0x0020, 0xfff9, 0xfffa, 0xfffb, 0xfffc, 0xfffd, 0xfffe, 0xffff, - }; - -static const unsigned short* gNormalizeTable[] = { - 0, gNormalizeTable0040, gNormalizeTable0080, gNormalizeTable00c0, - gNormalizeTable0100, gNormalizeTable0140, gNormalizeTable0180, gNormalizeTable01c0, - gNormalizeTable0200, gNormalizeTable0240, gNormalizeTable0280, gNormalizeTable02c0, - 0, gNormalizeTable0340, gNormalizeTable0380, gNormalizeTable03c0, - gNormalizeTable0400, gNormalizeTable0440, gNormalizeTable0480, gNormalizeTable04c0, - gNormalizeTable0500, gNormalizeTable0540, gNormalizeTable0580, 0, - gNormalizeTable0600, gNormalizeTable0640, 0, gNormalizeTable06c0, - 0, 0, 0, 0, - 0, 0, 0, 0, - gNormalizeTable0900, gNormalizeTable0940, 0, gNormalizeTable09c0, - gNormalizeTable0a00, gNormalizeTable0a40, 0, 0, - 0, gNormalizeTable0b40, gNormalizeTable0b80, gNormalizeTable0bc0, - 0, gNormalizeTable0c40, 0, gNormalizeTable0cc0, - 0, gNormalizeTable0d40, 0, gNormalizeTable0dc0, - gNormalizeTable0e00, 0, gNormalizeTable0e80, gNormalizeTable0ec0, - gNormalizeTable0f00, gNormalizeTable0f40, gNormalizeTable0f80, 0, - gNormalizeTable1000, 0, gNormalizeTable1080, gNormalizeTable10c0, - 0, gNormalizeTable1140, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, gNormalizeTable1780, 0, - gNormalizeTable1800, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - gNormalizeTable1b00, gNormalizeTable1b40, 0, 0, - 0, 0, 0, 0, - gNormalizeTable1d00, gNormalizeTable1d40, gNormalizeTable1d80, 0, - gNormalizeTable1e00, gNormalizeTable1e40, gNormalizeTable1e80, gNormalizeTable1ec0, - gNormalizeTable1f00, gNormalizeTable1f40, gNormalizeTable1f80, gNormalizeTable1fc0, - gNormalizeTable2000, gNormalizeTable2040, gNormalizeTable2080, 0, - gNormalizeTable2100, gNormalizeTable2140, gNormalizeTable2180, gNormalizeTable21c0, - gNormalizeTable2200, gNormalizeTable2240, gNormalizeTable2280, gNormalizeTable22c0, - gNormalizeTable2300, 0, 0, 0, - 0, gNormalizeTable2440, gNormalizeTable2480, gNormalizeTable24c0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - gNormalizeTable2a00, gNormalizeTable2a40, 0, gNormalizeTable2ac0, - 0, 0, 0, 0, - gNormalizeTable2c00, gNormalizeTable2c40, gNormalizeTable2c80, gNormalizeTable2cc0, - 0, gNormalizeTable2d40, 0, 0, - 0, 0, gNormalizeTable2e80, gNormalizeTable2ec0, - gNormalizeTable2f00, gNormalizeTable2f40, gNormalizeTable2f80, gNormalizeTable2fc0, - gNormalizeTable3000, gNormalizeTable3040, gNormalizeTable3080, gNormalizeTable30c0, - gNormalizeTable3100, gNormalizeTable3140, gNormalizeTable3180, 0, - gNormalizeTable3200, gNormalizeTable3240, gNormalizeTable3280, gNormalizeTable32c0, - gNormalizeTable3300, gNormalizeTable3340, gNormalizeTable3380, gNormalizeTable33c0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, gNormalizeTablea640, gNormalizeTablea680, 0, - gNormalizeTablea700, gNormalizeTablea740, gNormalizeTablea780, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0, - gNormalizeTablef900, gNormalizeTablef940, gNormalizeTablef980, gNormalizeTablef9c0, - gNormalizeTablefa00, gNormalizeTablefa40, gNormalizeTablefa80, gNormalizeTablefac0, - gNormalizeTablefb00, gNormalizeTablefb40, gNormalizeTablefb80, gNormalizeTablefbc0, - gNormalizeTablefc00, gNormalizeTablefc40, gNormalizeTablefc80, gNormalizeTablefcc0, - gNormalizeTablefd00, gNormalizeTablefd40, gNormalizeTablefd80, gNormalizeTablefdc0, - gNormalizeTablefe00, gNormalizeTablefe40, gNormalizeTablefe80, gNormalizeTablefec0, - gNormalizeTableff00, gNormalizeTableff40, gNormalizeTableff80, gNormalizeTableffc0, + /* U+0040 */ + 0x0040, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, }; -unsigned int normalize_character(const unsigned int c) -{ - if (c >= 0x10000 || !gNormalizeTable[c >> 6]) - return c; +static const unsigned short gNormalizeTable0080[] = { + /* U+0080 */ + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x0020, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, + 0x0020, 0x00a9, 0x0061, 0x00ab, 0x00ac, 0x0020, 0x00ae, 0x0020, + 0x00b0, 0x00b1, 0x0032, 0x0033, 0x0020, 0x03bc, 0x00b6, 0x00b7, + 0x0020, 0x0031, 0x006f, 0x00bb, 0x0031, 0x0031, 0x0033, 0x00bf, +}; + +static const unsigned short gNormalizeTable00c0[] = { + /* U+00c0 */ + 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x00e6, 0x0063, + 0x0065, 0x0065, 0x0065, 0x0065, 0x0069, 0x0069, 0x0069, 0x0069, + 0x00f0, 0x006e, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x00d7, + 0x00f8, 0x0075, 0x0075, 0x0075, 0x0075, 0x0079, 0x00fe, 0x0073, + 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x00e6, 0x0063, + 0x0065, 0x0065, 0x0065, 0x0065, 0x0069, 0x0069, 0x0069, 0x0069, + 0x00f0, 0x006e, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x00f7, + 0x00f8, 0x0075, 0x0075, 0x0075, 0x0075, 0x0079, 0x00fe, 0x0079, +}; + +static const unsigned short gNormalizeTable0100[] = { + /* U+0100 */ + 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0063, 0x0063, + 0x0063, 0x0063, 0x0063, 0x0063, 0x0063, 0x0063, 0x0064, 0x0064, + 0x0111, 0x0111, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, + 0x0065, 0x0065, 0x0065, 0x0065, 0x0067, 0x0067, 0x0067, 0x0067, + 0x0067, 0x0067, 0x0067, 0x0067, 0x0068, 0x0068, 0x0127, 0x0127, + 0x0069, 0x0069, 0x0069, 0x0069, 0x0069, 0x0069, 0x0069, 0x0069, + 0x0069, 0x0131, 0x0069, 0x0069, 0x006a, 0x006a, 0x006b, 0x006b, + 0x0138, 0x006c, 0x006c, 0x006c, 0x006c, 0x006c, 0x006c, 0x006c, +}; + +static const unsigned short gNormalizeTable0140[] = { + /* U+0140 */ + 0x006c, 0x0142, 0x0142, 0x006e, 0x006e, 0x006e, 0x006e, 0x006e, + 0x006e, 0x02bc, 0x014b, 0x014b, 0x006f, 0x006f, 0x006f, 0x006f, + 0x006f, 0x006f, 0x0153, 0x0153, 0x0072, 0x0072, 0x0072, 0x0072, + 0x0072, 0x0072, 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, + 0x0073, 0x0073, 0x0074, 0x0074, 0x0074, 0x0074, 0x0167, 0x0167, + 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, + 0x0075, 0x0075, 0x0075, 0x0075, 0x0077, 0x0077, 0x0079, 0x0079, + 0x0079, 0x007a, 0x007a, 0x007a, 0x007a, 0x007a, 0x007a, 0x0073, +}; + +static const unsigned short gNormalizeTable0180[] = { + /* U+0180 */ + 0x0180, 0x0253, 0x0183, 0x0183, 0x0185, 0x0185, 0x0254, 0x0188, + 0x0188, 0x0256, 0x0257, 0x018c, 0x018c, 0x018d, 0x01dd, 0x0259, + 0x025b, 0x0192, 0x0192, 0x0260, 0x0263, 0x0195, 0x0269, 0x0268, + 0x0199, 0x0199, 0x019a, 0x019b, 0x026f, 0x0272, 0x019e, 0x0275, + 0x006f, 0x006f, 0x01a3, 0x01a3, 0x01a5, 0x01a5, 0x0280, 0x01a8, + 0x01a8, 0x0283, 0x01aa, 0x01ab, 0x01ad, 0x01ad, 0x0288, 0x0075, + 0x0075, 0x028a, 0x028b, 0x01b4, 0x01b4, 0x01b6, 0x01b6, 0x0292, + 0x01b9, 0x01b9, 0x01ba, 0x01bb, 0x01bd, 0x01bd, 0x01be, 0x01bf, +}; + +static const unsigned short gNormalizeTable01c0[] = { + /* U+01c0 */ + 0x01c0, 0x01c1, 0x01c2, 0x01c3, 0x0064, 0x0064, 0x0064, 0x006c, + 0x006c, 0x006c, 0x006e, 0x006e, 0x006e, 0x0061, 0x0061, 0x0069, + 0x0069, 0x006f, 0x006f, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, + 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x01dd, 0x0061, 0x0061, + 0x0061, 0x0061, 0x00e6, 0x00e6, 0x01e5, 0x01e5, 0x0067, 0x0067, + 0x006b, 0x006b, 0x006f, 0x006f, 0x006f, 0x006f, 0x0292, 0x0292, + 0x006a, 0x0064, 0x0064, 0x0064, 0x0067, 0x0067, 0x0195, 0x01bf, + 0x006e, 0x006e, 0x0061, 0x0061, 0x00e6, 0x00e6, 0x00f8, 0x00f8, +}; + +static const unsigned short gNormalizeTable0200[] = { + /* U+0200 */ + 0x0061, 0x0061, 0x0061, 0x0061, 0x0065, 0x0065, 0x0065, 0x0065, + 0x0069, 0x0069, 0x0069, 0x0069, 0x006f, 0x006f, 0x006f, 0x006f, + 0x0072, 0x0072, 0x0072, 0x0072, 0x0075, 0x0075, 0x0075, 0x0075, + 0x0073, 0x0073, 0x0074, 0x0074, 0x021d, 0x021d, 0x0068, 0x0068, + 0x019e, 0x0221, 0x0223, 0x0223, 0x0225, 0x0225, 0x0061, 0x0061, + 0x0065, 0x0065, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, + 0x006f, 0x006f, 0x0079, 0x0079, 0x0234, 0x0235, 0x0236, 0x0237, + 0x0238, 0x0239, 0x2c65, 0x023c, 0x023c, 0x019a, 0x2c66, 0x023f, +}; + +static const unsigned short gNormalizeTable0240[] = { + /* U+0240 */ + 0x0240, 0x0242, 0x0242, 0x0180, 0x0289, 0x028c, 0x0247, 0x0247, + 0x0249, 0x0249, 0x024b, 0x024b, 0x024d, 0x024d, 0x024f, 0x024f, + 0x0250, 0x0251, 0x0252, 0x0253, 0x0254, 0x0255, 0x0256, 0x0257, + 0x0258, 0x0259, 0x025a, 0x025b, 0x025c, 0x025d, 0x025e, 0x025f, + 0x0260, 0x0261, 0x0262, 0x0263, 0x0264, 0x0265, 0x0266, 0x0267, + 0x0268, 0x0269, 0x026a, 0x026b, 0x026c, 0x026d, 0x026e, 0x026f, + 0x0270, 0x0271, 0x0272, 0x0273, 0x0274, 0x0275, 0x0276, 0x0277, + 0x0278, 0x0279, 0x027a, 0x027b, 0x027c, 0x027d, 0x027e, 0x027f, +}; + +static const unsigned short gNormalizeTable0280[] = { + /* U+0280 */ + 0x0280, 0x0281, 0x0282, 0x0283, 0x0284, 0x0285, 0x0286, 0x0287, + 0x0288, 0x0289, 0x028a, 0x028b, 0x028c, 0x028d, 0x028e, 0x028f, + 0x0290, 0x0291, 0x0292, 0x0293, 0x0294, 0x0295, 0x0296, 0x0297, + 0x0298, 0x0299, 0x029a, 0x029b, 0x029c, 0x029d, 0x029e, 0x029f, + 0x02a0, 0x02a1, 0x02a2, 0x02a3, 0x02a4, 0x02a5, 0x02a6, 0x02a7, + 0x02a8, 0x02a9, 0x02aa, 0x02ab, 0x02ac, 0x02ad, 0x02ae, 0x02af, + 0x0068, 0x0266, 0x006a, 0x0072, 0x0279, 0x027b, 0x0281, 0x0077, + 0x0079, 0x02b9, 0x02ba, 0x02bb, 0x02bc, 0x02bd, 0x02be, 0x02bf, +}; + +static const unsigned short gNormalizeTable02c0[] = { + /* U+02c0 */ + 0x02c0, 0x02c1, 0x02c2, 0x02c3, 0x02c4, 0x02c5, 0x02c6, 0x02c7, + 0x02c8, 0x02c9, 0x02ca, 0x02cb, 0x02cc, 0x02cd, 0x02ce, 0x02cf, + 0x02d0, 0x02d1, 0x02d2, 0x02d3, 0x02d4, 0x02d5, 0x02d6, 0x02d7, + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x02de, 0x02df, + 0x0263, 0x006c, 0x0073, 0x0078, 0x0295, 0x02e5, 0x02e6, 0x02e7, + 0x02e8, 0x02e9, 0x02ea, 0x02eb, 0x02ec, 0x02ed, 0x02ee, 0x02ef, + 0x02f0, 0x02f1, 0x02f2, 0x02f3, 0x02f4, 0x02f5, 0x02f6, 0x02f7, + 0x02f8, 0x02f9, 0x02fa, 0x02fb, 0x02fc, 0x02fd, 0x02fe, 0x02ff, +}; + +static const unsigned short gNormalizeTable0340[] = { + /* U+0340 */ + 0x0300, 0x0301, 0x0342, 0x0313, 0x0308, 0x03b9, 0x0346, 0x0347, + 0x0348, 0x0349, 0x034a, 0x034b, 0x034c, 0x034d, 0x034e, 0x0020, + 0x0350, 0x0351, 0x0352, 0x0353, 0x0354, 0x0355, 0x0356, 0x0357, + 0x0358, 0x0359, 0x035a, 0x035b, 0x035c, 0x035d, 0x035e, 0x035f, + 0x0360, 0x0361, 0x0362, 0x0363, 0x0364, 0x0365, 0x0366, 0x0367, + 0x0368, 0x0369, 0x036a, 0x036b, 0x036c, 0x036d, 0x036e, 0x036f, + 0x0371, 0x0371, 0x0373, 0x0373, 0x02b9, 0x0375, 0x0377, 0x0377, + 0x0378, 0x0379, 0x0020, 0x037b, 0x037c, 0x037d, 0x003b, 0x037f, +}; + +static const unsigned short gNormalizeTable0380[] = { + /* U+0380 */ + 0x0380, 0x0381, 0x0382, 0x0383, 0x0020, 0x0020, 0x03b1, 0x00b7, + 0x03b5, 0x03b7, 0x03b9, 0x038b, 0x03bf, 0x038d, 0x03c5, 0x03c9, + 0x03b9, 0x03b1, 0x03b2, 0x03b3, 0x03b4, 0x03b5, 0x03b6, 0x03b7, + 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd, 0x03be, 0x03bf, + 0x03c0, 0x03c1, 0x03a2, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7, + 0x03c8, 0x03c9, 0x03b9, 0x03c5, 0x03b1, 0x03b5, 0x03b7, 0x03b9, + 0x03c5, 0x03b1, 0x03b2, 0x03b3, 0x03b4, 0x03b5, 0x03b6, 0x03b7, + 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd, 0x03be, 0x03bf, +}; + +static const unsigned short gNormalizeTable03c0[] = { + /* U+03c0 */ + 0x03c0, 0x03c1, 0x03c3, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7, + 0x03c8, 0x03c9, 0x03b9, 0x03c5, 0x03bf, 0x03c5, 0x03c9, 0x03d7, + 0x03b2, 0x03b8, 0x03c5, 0x03c5, 0x03c5, 0x03c6, 0x03c0, 0x03d7, + 0x03d9, 0x03d9, 0x03db, 0x03db, 0x03dd, 0x03dd, 0x03df, 0x03df, + 0x03e1, 0x03e1, 0x03e3, 0x03e3, 0x03e5, 0x03e5, 0x03e7, 0x03e7, + 0x03e9, 0x03e9, 0x03eb, 0x03eb, 0x03ed, 0x03ed, 0x03ef, 0x03ef, + 0x03ba, 0x03c1, 0x03c3, 0x03f3, 0x03b8, 0x03b5, 0x03f6, 0x03f8, + 0x03f8, 0x03c3, 0x03fb, 0x03fb, 0x03fc, 0x037b, 0x037c, 0x037d, +}; + +static const unsigned short gNormalizeTable0400[] = { + /* U+0400 */ + 0x0435, 0x0435, 0x0452, 0x0433, 0x0454, 0x0455, 0x0456, 0x0456, + 0x0458, 0x0459, 0x045a, 0x045b, 0x043a, 0x0438, 0x0443, 0x045f, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0438, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0438, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, +}; + +static const unsigned short gNormalizeTable0440[] = { + /* U+0440 */ + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f, + 0x0435, 0x0435, 0x0452, 0x0433, 0x0454, 0x0455, 0x0456, 0x0456, + 0x0458, 0x0459, 0x045a, 0x045b, 0x043a, 0x0438, 0x0443, 0x045f, + 0x0461, 0x0461, 0x0463, 0x0463, 0x0465, 0x0465, 0x0467, 0x0467, + 0x0469, 0x0469, 0x046b, 0x046b, 0x046d, 0x046d, 0x046f, 0x046f, + 0x0471, 0x0471, 0x0473, 0x0473, 0x0475, 0x0475, 0x0475, 0x0475, + 0x0479, 0x0479, 0x047b, 0x047b, 0x047d, 0x047d, 0x047f, 0x047f, +}; + +static const unsigned short gNormalizeTable0480[] = { + /* U+0480 */ + 0x0481, 0x0481, 0x0482, 0x0483, 0x0484, 0x0485, 0x0486, 0x0487, + 0x0488, 0x0489, 0x048b, 0x048b, 0x048d, 0x048d, 0x048f, 0x048f, + 0x0491, 0x0491, 0x0493, 0x0493, 0x0495, 0x0495, 0x0497, 0x0497, + 0x0499, 0x0499, 0x049b, 0x049b, 0x049d, 0x049d, 0x049f, 0x049f, + 0x04a1, 0x04a1, 0x04a3, 0x04a3, 0x04a5, 0x04a5, 0x04a7, 0x04a7, + 0x04a9, 0x04a9, 0x04ab, 0x04ab, 0x04ad, 0x04ad, 0x04af, 0x04af, + 0x04b1, 0x04b1, 0x04b3, 0x04b3, 0x04b5, 0x04b5, 0x04b7, 0x04b7, + 0x04b9, 0x04b9, 0x04bb, 0x04bb, 0x04bd, 0x04bd, 0x04bf, 0x04bf, +}; + +static const unsigned short gNormalizeTable04c0[] = { + /* U+04c0 */ + 0x04cf, 0x0436, 0x0436, 0x04c4, 0x04c4, 0x04c6, 0x04c6, 0x04c8, + 0x04c8, 0x04ca, 0x04ca, 0x04cc, 0x04cc, 0x04ce, 0x04ce, 0x04cf, + 0x0430, 0x0430, 0x0430, 0x0430, 0x04d5, 0x04d5, 0x0435, 0x0435, + 0x04d9, 0x04d9, 0x04d9, 0x04d9, 0x0436, 0x0436, 0x0437, 0x0437, + 0x04e1, 0x04e1, 0x0438, 0x0438, 0x0438, 0x0438, 0x043e, 0x043e, + 0x04e9, 0x04e9, 0x04e9, 0x04e9, 0x044d, 0x044d, 0x0443, 0x0443, + 0x0443, 0x0443, 0x0443, 0x0443, 0x0447, 0x0447, 0x04f7, 0x04f7, + 0x044b, 0x044b, 0x04fb, 0x04fb, 0x04fd, 0x04fd, 0x04ff, 0x04ff, +}; + +static const unsigned short gNormalizeTable0500[] = { + /* U+0500 */ + 0x0501, 0x0501, 0x0503, 0x0503, 0x0505, 0x0505, 0x0507, 0x0507, + 0x0509, 0x0509, 0x050b, 0x050b, 0x050d, 0x050d, 0x050f, 0x050f, + 0x0511, 0x0511, 0x0513, 0x0513, 0x0515, 0x0515, 0x0517, 0x0517, + 0x0519, 0x0519, 0x051b, 0x051b, 0x051d, 0x051d, 0x051f, 0x051f, + 0x0521, 0x0521, 0x0523, 0x0523, 0x0525, 0x0525, 0x0526, 0x0527, + 0x0528, 0x0529, 0x052a, 0x052b, 0x052c, 0x052d, 0x052e, 0x052f, + 0x0530, 0x0561, 0x0562, 0x0563, 0x0564, 0x0565, 0x0566, 0x0567, + 0x0568, 0x0569, 0x056a, 0x056b, 0x056c, 0x056d, 0x056e, 0x056f, +}; + +static const unsigned short gNormalizeTable0540[] = { + /* U+0540 */ + 0x0570, 0x0571, 0x0572, 0x0573, 0x0574, 0x0575, 0x0576, 0x0577, + 0x0578, 0x0579, 0x057a, 0x057b, 0x057c, 0x057d, 0x057e, 0x057f, + 0x0580, 0x0581, 0x0582, 0x0583, 0x0584, 0x0585, 0x0586, 0x0557, + 0x0558, 0x0559, 0x055a, 0x055b, 0x055c, 0x055d, 0x055e, 0x055f, + 0x0560, 0x0561, 0x0562, 0x0563, 0x0564, 0x0565, 0x0566, 0x0567, + 0x0568, 0x0569, 0x056a, 0x056b, 0x056c, 0x056d, 0x056e, 0x056f, + 0x0570, 0x0571, 0x0572, 0x0573, 0x0574, 0x0575, 0x0576, 0x0577, + 0x0578, 0x0579, 0x057a, 0x057b, 0x057c, 0x057d, 0x057e, 0x057f, +}; + +static const unsigned short gNormalizeTable0580[] = { + /* U+0580 */ + 0x0580, 0x0581, 0x0582, 0x0583, 0x0584, 0x0585, 0x0586, 0x0565, + 0x0588, 0x0589, 0x058a, 0x058b, 0x058c, 0x058d, 0x058e, 0x058f, + 0x0590, 0x0591, 0x0592, 0x0593, 0x0594, 0x0595, 0x0596, 0x0597, + 0x0598, 0x0599, 0x059a, 0x059b, 0x059c, 0x059d, 0x059e, 0x059f, + 0x05a0, 0x05a1, 0x05a2, 0x05a3, 0x05a4, 0x05a5, 0x05a6, 0x05a7, + 0x05a8, 0x05a9, 0x05aa, 0x05ab, 0x05ac, 0x05ad, 0x05ae, 0x05af, + 0x05b0, 0x05b1, 0x05b2, 0x05b3, 0x05b4, 0x05b5, 0x05b6, 0x05b7, + 0x05b8, 0x05b9, 0x05ba, 0x05bb, 0x05bc, 0x05bd, 0x05be, 0x05bf, +}; + +static const unsigned short gNormalizeTable0600[] = { + /* U+0600 */ + 0x0600, 0x0601, 0x0602, 0x0603, 0x0604, 0x0605, 0x0606, 0x0607, + 0x0608, 0x0609, 0x060a, 0x060b, 0x060c, 0x060d, 0x060e, 0x060f, + 0x0610, 0x0611, 0x0612, 0x0613, 0x0614, 0x0615, 0x0616, 0x0617, + 0x0618, 0x0619, 0x061a, 0x061b, 0x061c, 0x061d, 0x061e, 0x061f, + 0x0620, 0x0621, 0x0627, 0x0627, 0x0648, 0x0627, 0x064a, 0x0627, + 0x0628, 0x0629, 0x062a, 0x062b, 0x062c, 0x062d, 0x062e, 0x062f, + 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x0637, + 0x0638, 0x0639, 0x063a, 0x063b, 0x063c, 0x063d, 0x063e, 0x063f, +}; + +static const unsigned short gNormalizeTable0640[] = { + /* U+0640 */ + 0x0640, 0x0641, 0x0642, 0x0643, 0x0644, 0x0645, 0x0646, 0x0647, + 0x0648, 0x0649, 0x064a, 0x064b, 0x064c, 0x064d, 0x064e, 0x064f, + 0x0650, 0x0651, 0x0652, 0x0653, 0x0654, 0x0655, 0x0656, 0x0657, + 0x0658, 0x0659, 0x065a, 0x065b, 0x065c, 0x065d, 0x065e, 0x065f, + 0x0660, 0x0661, 0x0662, 0x0663, 0x0664, 0x0665, 0x0666, 0x0667, + 0x0668, 0x0669, 0x066a, 0x066b, 0x066c, 0x066d, 0x066e, 0x066f, + 0x0670, 0x0671, 0x0672, 0x0673, 0x0674, 0x0627, 0x0648, 0x06c7, + 0x064a, 0x0679, 0x067a, 0x067b, 0x067c, 0x067d, 0x067e, 0x067f, +}; + +static const unsigned short gNormalizeTable06c0[] = { + /* U+06c0 */ + 0x06d5, 0x06c1, 0x06c1, 0x06c3, 0x06c4, 0x06c5, 0x06c6, 0x06c7, + 0x06c8, 0x06c9, 0x06ca, 0x06cb, 0x06cc, 0x06cd, 0x06ce, 0x06cf, + 0x06d0, 0x06d1, 0x06d2, 0x06d2, 0x06d4, 0x06d5, 0x06d6, 0x06d7, + 0x06d8, 0x06d9, 0x06da, 0x06db, 0x06dc, 0x06dd, 0x06de, 0x06df, + 0x06e0, 0x06e1, 0x06e2, 0x06e3, 0x06e4, 0x06e5, 0x06e6, 0x06e7, + 0x06e8, 0x06e9, 0x06ea, 0x06eb, 0x06ec, 0x06ed, 0x06ee, 0x06ef, + 0x06f0, 0x06f1, 0x06f2, 0x06f3, 0x06f4, 0x06f5, 0x06f6, 0x06f7, + 0x06f8, 0x06f9, 0x06fa, 0x06fb, 0x06fc, 0x06fd, 0x06fe, 0x06ff, +}; + +static const unsigned short gNormalizeTable0900[] = { + /* U+0900 */ + 0x0900, 0x0901, 0x0902, 0x0903, 0x0904, 0x0905, 0x0906, 0x0907, + 0x0908, 0x0909, 0x090a, 0x090b, 0x090c, 0x090d, 0x090e, 0x090f, + 0x0910, 0x0911, 0x0912, 0x0913, 0x0914, 0x0915, 0x0916, 0x0917, + 0x0918, 0x0919, 0x091a, 0x091b, 0x091c, 0x091d, 0x091e, 0x091f, + 0x0920, 0x0921, 0x0922, 0x0923, 0x0924, 0x0925, 0x0926, 0x0927, + 0x0928, 0x0928, 0x092a, 0x092b, 0x092c, 0x092d, 0x092e, 0x092f, + 0x0930, 0x0930, 0x0932, 0x0933, 0x0933, 0x0935, 0x0936, 0x0937, + 0x0938, 0x0939, 0x093a, 0x093b, 0x093c, 0x093d, 0x093e, 0x093f, +}; + +static const unsigned short gNormalizeTable0940[] = { + /* U+0940 */ + 0x0940, 0x0941, 0x0942, 0x0943, 0x0944, 0x0945, 0x0946, 0x0947, + 0x0948, 0x0949, 0x094a, 0x094b, 0x094c, 0x094d, 0x094e, 0x094f, + 0x0950, 0x0951, 0x0952, 0x0953, 0x0954, 0x0955, 0x0956, 0x0957, + 0x0915, 0x0916, 0x0917, 0x091c, 0x0921, 0x0922, 0x092b, 0x092f, + 0x0960, 0x0961, 0x0962, 0x0963, 0x0964, 0x0965, 0x0966, 0x0967, + 0x0968, 0x0969, 0x096a, 0x096b, 0x096c, 0x096d, 0x096e, 0x096f, + 0x0970, 0x0971, 0x0972, 0x0973, 0x0974, 0x0975, 0x0976, 0x0977, + 0x0978, 0x0979, 0x097a, 0x097b, 0x097c, 0x097d, 0x097e, 0x097f, +}; + +static const unsigned short gNormalizeTable09c0[] = { + /* U+09c0 */ + 0x09c0, 0x09c1, 0x09c2, 0x09c3, 0x09c4, 0x09c5, 0x09c6, 0x09c7, + 0x09c8, 0x09c9, 0x09ca, 0x09c7, 0x09c7, 0x09cd, 0x09ce, 0x09cf, + 0x09d0, 0x09d1, 0x09d2, 0x09d3, 0x09d4, 0x09d5, 0x09d6, 0x09d7, + 0x09d8, 0x09d9, 0x09da, 0x09db, 0x09a1, 0x09a2, 0x09de, 0x09af, + 0x09e0, 0x09e1, 0x09e2, 0x09e3, 0x09e4, 0x09e5, 0x09e6, 0x09e7, + 0x09e8, 0x09e9, 0x09ea, 0x09eb, 0x09ec, 0x09ed, 0x09ee, 0x09ef, + 0x09f0, 0x09f1, 0x09f2, 0x09f3, 0x09f4, 0x09f5, 0x09f6, 0x09f7, + 0x09f8, 0x09f9, 0x09fa, 0x09fb, 0x09fc, 0x09fd, 0x09fe, 0x09ff, +}; + +static const unsigned short gNormalizeTable0a00[] = { + /* U+0a00 */ + 0x0a00, 0x0a01, 0x0a02, 0x0a03, 0x0a04, 0x0a05, 0x0a06, 0x0a07, + 0x0a08, 0x0a09, 0x0a0a, 0x0a0b, 0x0a0c, 0x0a0d, 0x0a0e, 0x0a0f, + 0x0a10, 0x0a11, 0x0a12, 0x0a13, 0x0a14, 0x0a15, 0x0a16, 0x0a17, + 0x0a18, 0x0a19, 0x0a1a, 0x0a1b, 0x0a1c, 0x0a1d, 0x0a1e, 0x0a1f, + 0x0a20, 0x0a21, 0x0a22, 0x0a23, 0x0a24, 0x0a25, 0x0a26, 0x0a27, + 0x0a28, 0x0a29, 0x0a2a, 0x0a2b, 0x0a2c, 0x0a2d, 0x0a2e, 0x0a2f, + 0x0a30, 0x0a31, 0x0a32, 0x0a32, 0x0a34, 0x0a35, 0x0a38, 0x0a37, + 0x0a38, 0x0a39, 0x0a3a, 0x0a3b, 0x0a3c, 0x0a3d, 0x0a3e, 0x0a3f, +}; + +static const unsigned short gNormalizeTable0a40[] = { + /* U+0a40 */ + 0x0a40, 0x0a41, 0x0a42, 0x0a43, 0x0a44, 0x0a45, 0x0a46, 0x0a47, + 0x0a48, 0x0a49, 0x0a4a, 0x0a4b, 0x0a4c, 0x0a4d, 0x0a4e, 0x0a4f, + 0x0a50, 0x0a51, 0x0a52, 0x0a53, 0x0a54, 0x0a55, 0x0a56, 0x0a57, + 0x0a58, 0x0a16, 0x0a17, 0x0a1c, 0x0a5c, 0x0a5d, 0x0a2b, 0x0a5f, + 0x0a60, 0x0a61, 0x0a62, 0x0a63, 0x0a64, 0x0a65, 0x0a66, 0x0a67, + 0x0a68, 0x0a69, 0x0a6a, 0x0a6b, 0x0a6c, 0x0a6d, 0x0a6e, 0x0a6f, + 0x0a70, 0x0a71, 0x0a72, 0x0a73, 0x0a74, 0x0a75, 0x0a76, 0x0a77, + 0x0a78, 0x0a79, 0x0a7a, 0x0a7b, 0x0a7c, 0x0a7d, 0x0a7e, 0x0a7f, +}; + +static const unsigned short gNormalizeTable0b40[] = { + /* U+0b40 */ + 0x0b40, 0x0b41, 0x0b42, 0x0b43, 0x0b44, 0x0b45, 0x0b46, 0x0b47, + 0x0b47, 0x0b49, 0x0b4a, 0x0b47, 0x0b47, 0x0b4d, 0x0b4e, 0x0b4f, + 0x0b50, 0x0b51, 0x0b52, 0x0b53, 0x0b54, 0x0b55, 0x0b56, 0x0b57, + 0x0b58, 0x0b59, 0x0b5a, 0x0b5b, 0x0b21, 0x0b22, 0x0b5e, 0x0b5f, + 0x0b60, 0x0b61, 0x0b62, 0x0b63, 0x0b64, 0x0b65, 0x0b66, 0x0b67, + 0x0b68, 0x0b69, 0x0b6a, 0x0b6b, 0x0b6c, 0x0b6d, 0x0b6e, 0x0b6f, + 0x0b70, 0x0b71, 0x0b72, 0x0b73, 0x0b74, 0x0b75, 0x0b76, 0x0b77, + 0x0b78, 0x0b79, 0x0b7a, 0x0b7b, 0x0b7c, 0x0b7d, 0x0b7e, 0x0b7f, +}; + +static const unsigned short gNormalizeTable0b80[] = { + /* U+0b80 */ + 0x0b80, 0x0b81, 0x0b82, 0x0b83, 0x0b84, 0x0b85, 0x0b86, 0x0b87, + 0x0b88, 0x0b89, 0x0b8a, 0x0b8b, 0x0b8c, 0x0b8d, 0x0b8e, 0x0b8f, + 0x0b90, 0x0b91, 0x0b92, 0x0b93, 0x0b92, 0x0b95, 0x0b96, 0x0b97, + 0x0b98, 0x0b99, 0x0b9a, 0x0b9b, 0x0b9c, 0x0b9d, 0x0b9e, 0x0b9f, + 0x0ba0, 0x0ba1, 0x0ba2, 0x0ba3, 0x0ba4, 0x0ba5, 0x0ba6, 0x0ba7, + 0x0ba8, 0x0ba9, 0x0baa, 0x0bab, 0x0bac, 0x0bad, 0x0bae, 0x0baf, + 0x0bb0, 0x0bb1, 0x0bb2, 0x0bb3, 0x0bb4, 0x0bb5, 0x0bb6, 0x0bb7, + 0x0bb8, 0x0bb9, 0x0bba, 0x0bbb, 0x0bbc, 0x0bbd, 0x0bbe, 0x0bbf, +}; + +static const unsigned short gNormalizeTable0bc0[] = { + /* U+0bc0 */ + 0x0bc0, 0x0bc1, 0x0bc2, 0x0bc3, 0x0bc4, 0x0bc5, 0x0bc6, 0x0bc7, + 0x0bc8, 0x0bc9, 0x0bc6, 0x0bc7, 0x0bc6, 0x0bcd, 0x0bce, 0x0bcf, + 0x0bd0, 0x0bd1, 0x0bd2, 0x0bd3, 0x0bd4, 0x0bd5, 0x0bd6, 0x0bd7, + 0x0bd8, 0x0bd9, 0x0bda, 0x0bdb, 0x0bdc, 0x0bdd, 0x0bde, 0x0bdf, + 0x0be0, 0x0be1, 0x0be2, 0x0be3, 0x0be4, 0x0be5, 0x0be6, 0x0be7, + 0x0be8, 0x0be9, 0x0bea, 0x0beb, 0x0bec, 0x0bed, 0x0bee, 0x0bef, + 0x0bf0, 0x0bf1, 0x0bf2, 0x0bf3, 0x0bf4, 0x0bf5, 0x0bf6, 0x0bf7, + 0x0bf8, 0x0bf9, 0x0bfa, 0x0bfb, 0x0bfc, 0x0bfd, 0x0bfe, 0x0bff, +}; + +static const unsigned short gNormalizeTable0c40[] = { + /* U+0c40 */ + 0x0c40, 0x0c41, 0x0c42, 0x0c43, 0x0c44, 0x0c45, 0x0c46, 0x0c47, + 0x0c46, 0x0c49, 0x0c4a, 0x0c4b, 0x0c4c, 0x0c4d, 0x0c4e, 0x0c4f, + 0x0c50, 0x0c51, 0x0c52, 0x0c53, 0x0c54, 0x0c55, 0x0c56, 0x0c57, + 0x0c58, 0x0c59, 0x0c5a, 0x0c5b, 0x0c5c, 0x0c5d, 0x0c5e, 0x0c5f, + 0x0c60, 0x0c61, 0x0c62, 0x0c63, 0x0c64, 0x0c65, 0x0c66, 0x0c67, + 0x0c68, 0x0c69, 0x0c6a, 0x0c6b, 0x0c6c, 0x0c6d, 0x0c6e, 0x0c6f, + 0x0c70, 0x0c71, 0x0c72, 0x0c73, 0x0c74, 0x0c75, 0x0c76, 0x0c77, + 0x0c78, 0x0c79, 0x0c7a, 0x0c7b, 0x0c7c, 0x0c7d, 0x0c7e, 0x0c7f, +}; + +static const unsigned short gNormalizeTable0cc0[] = { + /* U+0cc0 */ + 0x0cbf, 0x0cc1, 0x0cc2, 0x0cc3, 0x0cc4, 0x0cc5, 0x0cc6, 0x0cc6, + 0x0cc6, 0x0cc9, 0x0cc6, 0x0cc6, 0x0ccc, 0x0ccd, 0x0cce, 0x0ccf, + 0x0cd0, 0x0cd1, 0x0cd2, 0x0cd3, 0x0cd4, 0x0cd5, 0x0cd6, 0x0cd7, + 0x0cd8, 0x0cd9, 0x0cda, 0x0cdb, 0x0cdc, 0x0cdd, 0x0cde, 0x0cdf, + 0x0ce0, 0x0ce1, 0x0ce2, 0x0ce3, 0x0ce4, 0x0ce5, 0x0ce6, 0x0ce7, + 0x0ce8, 0x0ce9, 0x0cea, 0x0ceb, 0x0cec, 0x0ced, 0x0cee, 0x0cef, + 0x0cf0, 0x0cf1, 0x0cf2, 0x0cf3, 0x0cf4, 0x0cf5, 0x0cf6, 0x0cf7, + 0x0cf8, 0x0cf9, 0x0cfa, 0x0cfb, 0x0cfc, 0x0cfd, 0x0cfe, 0x0cff, +}; + +static const unsigned short gNormalizeTable0d40[] = { + /* U+0d40 */ + 0x0d40, 0x0d41, 0x0d42, 0x0d43, 0x0d44, 0x0d45, 0x0d46, 0x0d47, + 0x0d48, 0x0d49, 0x0d46, 0x0d47, 0x0d46, 0x0d4d, 0x0d4e, 0x0d4f, + 0x0d50, 0x0d51, 0x0d52, 0x0d53, 0x0d54, 0x0d55, 0x0d56, 0x0d57, + 0x0d58, 0x0d59, 0x0d5a, 0x0d5b, 0x0d5c, 0x0d5d, 0x0d5e, 0x0d5f, + 0x0d60, 0x0d61, 0x0d62, 0x0d63, 0x0d64, 0x0d65, 0x0d66, 0x0d67, + 0x0d68, 0x0d69, 0x0d6a, 0x0d6b, 0x0d6c, 0x0d6d, 0x0d6e, 0x0d6f, + 0x0d70, 0x0d71, 0x0d72, 0x0d73, 0x0d74, 0x0d75, 0x0d76, 0x0d77, + 0x0d78, 0x0d79, 0x0d7a, 0x0d7b, 0x0d7c, 0x0d7d, 0x0d7e, 0x0d7f, +}; + +static const unsigned short gNormalizeTable0dc0[] = { + /* U+0dc0 */ + 0x0dc0, 0x0dc1, 0x0dc2, 0x0dc3, 0x0dc4, 0x0dc5, 0x0dc6, 0x0dc7, + 0x0dc8, 0x0dc9, 0x0dca, 0x0dcb, 0x0dcc, 0x0dcd, 0x0dce, 0x0dcf, + 0x0dd0, 0x0dd1, 0x0dd2, 0x0dd3, 0x0dd4, 0x0dd5, 0x0dd6, 0x0dd7, + 0x0dd8, 0x0dd9, 0x0dd9, 0x0ddb, 0x0dd9, 0x0dd9, 0x0dd9, 0x0ddf, + 0x0de0, 0x0de1, 0x0de2, 0x0de3, 0x0de4, 0x0de5, 0x0de6, 0x0de7, + 0x0de8, 0x0de9, 0x0dea, 0x0deb, 0x0dec, 0x0ded, 0x0dee, 0x0def, + 0x0df0, 0x0df1, 0x0df2, 0x0df3, 0x0df4, 0x0df5, 0x0df6, 0x0df7, + 0x0df8, 0x0df9, 0x0dfa, 0x0dfb, 0x0dfc, 0x0dfd, 0x0dfe, 0x0dff, +}; + +static const unsigned short gNormalizeTable0e00[] = { + /* U+0e00 */ + 0x0e00, 0x0e01, 0x0e02, 0x0e03, 0x0e04, 0x0e05, 0x0e06, 0x0e07, + 0x0e08, 0x0e09, 0x0e0a, 0x0e0b, 0x0e0c, 0x0e0d, 0x0e0e, 0x0e0f, + 0x0e10, 0x0e11, 0x0e12, 0x0e13, 0x0e14, 0x0e15, 0x0e16, 0x0e17, + 0x0e18, 0x0e19, 0x0e1a, 0x0e1b, 0x0e1c, 0x0e1d, 0x0e1e, 0x0e1f, + 0x0e20, 0x0e21, 0x0e22, 0x0e23, 0x0e24, 0x0e25, 0x0e26, 0x0e27, + 0x0e28, 0x0e29, 0x0e2a, 0x0e2b, 0x0e2c, 0x0e2d, 0x0e2e, 0x0e2f, + 0x0e30, 0x0e31, 0x0e32, 0x0e4d, 0x0e34, 0x0e35, 0x0e36, 0x0e37, + 0x0e38, 0x0e39, 0x0e3a, 0x0e3b, 0x0e3c, 0x0e3d, 0x0e3e, 0x0e3f, +}; + +static const unsigned short gNormalizeTable0e80[] = { + /* U+0e80 */ + 0x0e80, 0x0e81, 0x0e82, 0x0e83, 0x0e84, 0x0e85, 0x0e86, 0x0e87, + 0x0e88, 0x0e89, 0x0e8a, 0x0e8b, 0x0e8c, 0x0e8d, 0x0e8e, 0x0e8f, + 0x0e90, 0x0e91, 0x0e92, 0x0e93, 0x0e94, 0x0e95, 0x0e96, 0x0e97, + 0x0e98, 0x0e99, 0x0e9a, 0x0e9b, 0x0e9c, 0x0e9d, 0x0e9e, 0x0e9f, + 0x0ea0, 0x0ea1, 0x0ea2, 0x0ea3, 0x0ea4, 0x0ea5, 0x0ea6, 0x0ea7, + 0x0ea8, 0x0ea9, 0x0eaa, 0x0eab, 0x0eac, 0x0ead, 0x0eae, 0x0eaf, + 0x0eb0, 0x0eb1, 0x0eb2, 0x0ecd, 0x0eb4, 0x0eb5, 0x0eb6, 0x0eb7, + 0x0eb8, 0x0eb9, 0x0eba, 0x0ebb, 0x0ebc, 0x0ebd, 0x0ebe, 0x0ebf, +}; + +static const unsigned short gNormalizeTable0ec0[] = { + /* U+0ec0 */ + 0x0ec0, 0x0ec1, 0x0ec2, 0x0ec3, 0x0ec4, 0x0ec5, 0x0ec6, 0x0ec7, + 0x0ec8, 0x0ec9, 0x0eca, 0x0ecb, 0x0ecc, 0x0ecd, 0x0ece, 0x0ecf, + 0x0ed0, 0x0ed1, 0x0ed2, 0x0ed3, 0x0ed4, 0x0ed5, 0x0ed6, 0x0ed7, + 0x0ed8, 0x0ed9, 0x0eda, 0x0edb, 0x0eab, 0x0eab, 0x0ede, 0x0edf, + 0x0ee0, 0x0ee1, 0x0ee2, 0x0ee3, 0x0ee4, 0x0ee5, 0x0ee6, 0x0ee7, + 0x0ee8, 0x0ee9, 0x0eea, 0x0eeb, 0x0eec, 0x0eed, 0x0eee, 0x0eef, + 0x0ef0, 0x0ef1, 0x0ef2, 0x0ef3, 0x0ef4, 0x0ef5, 0x0ef6, 0x0ef7, + 0x0ef8, 0x0ef9, 0x0efa, 0x0efb, 0x0efc, 0x0efd, 0x0efe, 0x0eff, +}; + +static const unsigned short gNormalizeTable0f00[] = { + /* U+0f00 */ + 0x0f00, 0x0f01, 0x0f02, 0x0f03, 0x0f04, 0x0f05, 0x0f06, 0x0f07, + 0x0f08, 0x0f09, 0x0f0a, 0x0f0b, 0x0f0b, 0x0f0d, 0x0f0e, 0x0f0f, + 0x0f10, 0x0f11, 0x0f12, 0x0f13, 0x0f14, 0x0f15, 0x0f16, 0x0f17, + 0x0f18, 0x0f19, 0x0f1a, 0x0f1b, 0x0f1c, 0x0f1d, 0x0f1e, 0x0f1f, + 0x0f20, 0x0f21, 0x0f22, 0x0f23, 0x0f24, 0x0f25, 0x0f26, 0x0f27, + 0x0f28, 0x0f29, 0x0f2a, 0x0f2b, 0x0f2c, 0x0f2d, 0x0f2e, 0x0f2f, + 0x0f30, 0x0f31, 0x0f32, 0x0f33, 0x0f34, 0x0f35, 0x0f36, 0x0f37, + 0x0f38, 0x0f39, 0x0f3a, 0x0f3b, 0x0f3c, 0x0f3d, 0x0f3e, 0x0f3f, +}; + +static const unsigned short gNormalizeTable0f40[] = { + /* U+0f40 */ + 0x0f40, 0x0f41, 0x0f42, 0x0f42, 0x0f44, 0x0f45, 0x0f46, 0x0f47, + 0x0f48, 0x0f49, 0x0f4a, 0x0f4b, 0x0f4c, 0x0f4c, 0x0f4e, 0x0f4f, + 0x0f50, 0x0f51, 0x0f51, 0x0f53, 0x0f54, 0x0f55, 0x0f56, 0x0f56, + 0x0f58, 0x0f59, 0x0f5a, 0x0f5b, 0x0f5b, 0x0f5d, 0x0f5e, 0x0f5f, + 0x0f60, 0x0f61, 0x0f62, 0x0f63, 0x0f64, 0x0f65, 0x0f66, 0x0f67, + 0x0f68, 0x0f40, 0x0f6a, 0x0f6b, 0x0f6c, 0x0f6d, 0x0f6e, 0x0f6f, + 0x0f70, 0x0f71, 0x0f72, 0x0f71, 0x0f74, 0x0f71, 0x0fb2, 0x0fb2, + 0x0fb3, 0x0fb3, 0x0f7a, 0x0f7b, 0x0f7c, 0x0f7d, 0x0f7e, 0x0f7f, +}; + +static const unsigned short gNormalizeTable0f80[] = { + /* U+0f80 */ + 0x0f80, 0x0f71, 0x0f82, 0x0f83, 0x0f84, 0x0f85, 0x0f86, 0x0f87, + 0x0f88, 0x0f89, 0x0f8a, 0x0f8b, 0x0f8c, 0x0f8d, 0x0f8e, 0x0f8f, + 0x0f90, 0x0f91, 0x0f92, 0x0f92, 0x0f94, 0x0f95, 0x0f96, 0x0f97, + 0x0f98, 0x0f99, 0x0f9a, 0x0f9b, 0x0f9c, 0x0f9c, 0x0f9e, 0x0f9f, + 0x0fa0, 0x0fa1, 0x0fa1, 0x0fa3, 0x0fa4, 0x0fa5, 0x0fa6, 0x0fa6, + 0x0fa8, 0x0fa9, 0x0faa, 0x0fab, 0x0fab, 0x0fad, 0x0fae, 0x0faf, + 0x0fb0, 0x0fb1, 0x0fb2, 0x0fb3, 0x0fb4, 0x0fb5, 0x0fb6, 0x0fb7, + 0x0fb8, 0x0f90, 0x0fba, 0x0fbb, 0x0fbc, 0x0fbd, 0x0fbe, 0x0fbf, +}; + +static const unsigned short gNormalizeTable1000[] = { + /* U+1000 */ + 0x1000, 0x1001, 0x1002, 0x1003, 0x1004, 0x1005, 0x1006, 0x1007, + 0x1008, 0x1009, 0x100a, 0x100b, 0x100c, 0x100d, 0x100e, 0x100f, + 0x1010, 0x1011, 0x1012, 0x1013, 0x1014, 0x1015, 0x1016, 0x1017, + 0x1018, 0x1019, 0x101a, 0x101b, 0x101c, 0x101d, 0x101e, 0x101f, + 0x1020, 0x1021, 0x1022, 0x1023, 0x1024, 0x1025, 0x1025, 0x1027, + 0x1028, 0x1029, 0x102a, 0x102b, 0x102c, 0x102d, 0x102e, 0x102f, + 0x1030, 0x1031, 0x1032, 0x1033, 0x1034, 0x1035, 0x1036, 0x1037, + 0x1038, 0x1039, 0x103a, 0x103b, 0x103c, 0x103d, 0x103e, 0x103f, +}; + +static const unsigned short gNormalizeTable1080[] = { + /* U+1080 */ + 0x1080, 0x1081, 0x1082, 0x1083, 0x1084, 0x1085, 0x1086, 0x1087, + 0x1088, 0x1089, 0x108a, 0x108b, 0x108c, 0x108d, 0x108e, 0x108f, + 0x1090, 0x1091, 0x1092, 0x1093, 0x1094, 0x1095, 0x1096, 0x1097, + 0x1098, 0x1099, 0x109a, 0x109b, 0x109c, 0x109d, 0x109e, 0x109f, + 0x2d00, 0x2d01, 0x2d02, 0x2d03, 0x2d04, 0x2d05, 0x2d06, 0x2d07, + 0x2d08, 0x2d09, 0x2d0a, 0x2d0b, 0x2d0c, 0x2d0d, 0x2d0e, 0x2d0f, + 0x2d10, 0x2d11, 0x2d12, 0x2d13, 0x2d14, 0x2d15, 0x2d16, 0x2d17, + 0x2d18, 0x2d19, 0x2d1a, 0x2d1b, 0x2d1c, 0x2d1d, 0x2d1e, 0x2d1f, +}; + +static const unsigned short gNormalizeTable10c0[] = { + /* U+10c0 */ + 0x2d20, 0x2d21, 0x2d22, 0x2d23, 0x2d24, 0x2d25, 0x10c6, 0x10c7, + 0x10c8, 0x10c9, 0x10ca, 0x10cb, 0x10cc, 0x10cd, 0x10ce, 0x10cf, + 0x10d0, 0x10d1, 0x10d2, 0x10d3, 0x10d4, 0x10d5, 0x10d6, 0x10d7, + 0x10d8, 0x10d9, 0x10da, 0x10db, 0x10dc, 0x10dd, 0x10de, 0x10df, + 0x10e0, 0x10e1, 0x10e2, 0x10e3, 0x10e4, 0x10e5, 0x10e6, 0x10e7, + 0x10e8, 0x10e9, 0x10ea, 0x10eb, 0x10ec, 0x10ed, 0x10ee, 0x10ef, + 0x10f0, 0x10f1, 0x10f2, 0x10f3, 0x10f4, 0x10f5, 0x10f6, 0x10f7, + 0x10f8, 0x10f9, 0x10fa, 0x10fb, 0x10dc, 0x10fd, 0x10fe, 0x10ff, +}; + +static const unsigned short gNormalizeTable1140[] = { + /* U+1140 */ + 0x1140, 0x1141, 0x1142, 0x1143, 0x1144, 0x1145, 0x1146, 0x1147, + 0x1148, 0x1149, 0x114a, 0x114b, 0x114c, 0x114d, 0x114e, 0x114f, + 0x1150, 0x1151, 0x1152, 0x1153, 0x1154, 0x1155, 0x1156, 0x1157, + 0x1158, 0x1159, 0x115a, 0x115b, 0x115c, 0x115d, 0x115e, 0x0020, + 0x0020, 0x1161, 0x1162, 0x1163, 0x1164, 0x1165, 0x1166, 0x1167, + 0x1168, 0x1169, 0x116a, 0x116b, 0x116c, 0x116d, 0x116e, 0x116f, + 0x1170, 0x1171, 0x1172, 0x1173, 0x1174, 0x1175, 0x1176, 0x1177, + 0x1178, 0x1179, 0x117a, 0x117b, 0x117c, 0x117d, 0x117e, 0x117f, +}; + +static const unsigned short gNormalizeTable1780[] = { + /* U+1780 */ + 0x1780, 0x1781, 0x1782, 0x1783, 0x1784, 0x1785, 0x1786, 0x1787, + 0x1788, 0x1789, 0x178a, 0x178b, 0x178c, 0x178d, 0x178e, 0x178f, + 0x1790, 0x1791, 0x1792, 0x1793, 0x1794, 0x1795, 0x1796, 0x1797, + 0x1798, 0x1799, 0x179a, 0x179b, 0x179c, 0x179d, 0x179e, 0x179f, + 0x17a0, 0x17a1, 0x17a2, 0x17a3, 0x17a4, 0x17a5, 0x17a6, 0x17a7, + 0x17a8, 0x17a9, 0x17aa, 0x17ab, 0x17ac, 0x17ad, 0x17ae, 0x17af, + 0x17b0, 0x17b1, 0x17b2, 0x17b3, 0x0020, 0x0020, 0x17b6, 0x17b7, + 0x17b8, 0x17b9, 0x17ba, 0x17bb, 0x17bc, 0x17bd, 0x17be, 0x17bf, +}; + +static const unsigned short gNormalizeTable1800[] = { + /* U+1800 */ + 0x1800, 0x1801, 0x1802, 0x1803, 0x1804, 0x1805, 0x1806, 0x1807, + 0x1808, 0x1809, 0x180a, 0x0020, 0x0020, 0x0020, 0x180e, 0x180f, + 0x1810, 0x1811, 0x1812, 0x1813, 0x1814, 0x1815, 0x1816, 0x1817, + 0x1818, 0x1819, 0x181a, 0x181b, 0x181c, 0x181d, 0x181e, 0x181f, + 0x1820, 0x1821, 0x1822, 0x1823, 0x1824, 0x1825, 0x1826, 0x1827, + 0x1828, 0x1829, 0x182a, 0x182b, 0x182c, 0x182d, 0x182e, 0x182f, + 0x1830, 0x1831, 0x1832, 0x1833, 0x1834, 0x1835, 0x1836, 0x1837, + 0x1838, 0x1839, 0x183a, 0x183b, 0x183c, 0x183d, 0x183e, 0x183f, +}; + +static const unsigned short gNormalizeTable1b00[] = { + /* U+1b00 */ + 0x1b00, 0x1b01, 0x1b02, 0x1b03, 0x1b04, 0x1b05, 0x1b05, 0x1b07, + 0x1b07, 0x1b09, 0x1b09, 0x1b0b, 0x1b0b, 0x1b0d, 0x1b0d, 0x1b0f, + 0x1b10, 0x1b11, 0x1b11, 0x1b13, 0x1b14, 0x1b15, 0x1b16, 0x1b17, + 0x1b18, 0x1b19, 0x1b1a, 0x1b1b, 0x1b1c, 0x1b1d, 0x1b1e, 0x1b1f, + 0x1b20, 0x1b21, 0x1b22, 0x1b23, 0x1b24, 0x1b25, 0x1b26, 0x1b27, + 0x1b28, 0x1b29, 0x1b2a, 0x1b2b, 0x1b2c, 0x1b2d, 0x1b2e, 0x1b2f, + 0x1b30, 0x1b31, 0x1b32, 0x1b33, 0x1b34, 0x1b35, 0x1b36, 0x1b37, + 0x1b38, 0x1b39, 0x1b3a, 0x1b3a, 0x1b3c, 0x1b3c, 0x1b3e, 0x1b3f, +}; + +static const unsigned short gNormalizeTable1b40[] = { + /* U+1b40 */ + 0x1b3e, 0x1b3f, 0x1b42, 0x1b42, 0x1b44, 0x1b45, 0x1b46, 0x1b47, + 0x1b48, 0x1b49, 0x1b4a, 0x1b4b, 0x1b4c, 0x1b4d, 0x1b4e, 0x1b4f, + 0x1b50, 0x1b51, 0x1b52, 0x1b53, 0x1b54, 0x1b55, 0x1b56, 0x1b57, + 0x1b58, 0x1b59, 0x1b5a, 0x1b5b, 0x1b5c, 0x1b5d, 0x1b5e, 0x1b5f, + 0x1b60, 0x1b61, 0x1b62, 0x1b63, 0x1b64, 0x1b65, 0x1b66, 0x1b67, + 0x1b68, 0x1b69, 0x1b6a, 0x1b6b, 0x1b6c, 0x1b6d, 0x1b6e, 0x1b6f, + 0x1b70, 0x1b71, 0x1b72, 0x1b73, 0x1b74, 0x1b75, 0x1b76, 0x1b77, + 0x1b78, 0x1b79, 0x1b7a, 0x1b7b, 0x1b7c, 0x1b7d, 0x1b7e, 0x1b7f, +}; + +static const unsigned short gNormalizeTable1d00[] = { + /* U+1d00 */ + 0x1d00, 0x1d01, 0x1d02, 0x1d03, 0x1d04, 0x1d05, 0x1d06, 0x1d07, + 0x1d08, 0x1d09, 0x1d0a, 0x1d0b, 0x1d0c, 0x1d0d, 0x1d0e, 0x1d0f, + 0x1d10, 0x1d11, 0x1d12, 0x1d13, 0x1d14, 0x1d15, 0x1d16, 0x1d17, + 0x1d18, 0x1d19, 0x1d1a, 0x1d1b, 0x1d1c, 0x1d1d, 0x1d1e, 0x1d1f, + 0x1d20, 0x1d21, 0x1d22, 0x1d23, 0x1d24, 0x1d25, 0x1d26, 0x1d27, + 0x1d28, 0x1d29, 0x1d2a, 0x1d2b, 0x0061, 0x00e6, 0x0062, 0x1d2f, + 0x0064, 0x0065, 0x01dd, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x1d3b, 0x006f, 0x0223, 0x0070, 0x0072, +}; + +static const unsigned short gNormalizeTable1d40[] = { + /* U+1d40 */ + 0x0074, 0x0075, 0x0077, 0x0061, 0x0250, 0x0251, 0x1d02, 0x0062, + 0x0064, 0x0065, 0x0259, 0x025b, 0x025c, 0x0067, 0x1d4e, 0x006b, + 0x006d, 0x014b, 0x006f, 0x0254, 0x1d16, 0x1d17, 0x0070, 0x0074, + 0x0075, 0x1d1d, 0x026f, 0x0076, 0x1d25, 0x03b2, 0x03b3, 0x03b4, + 0x03c6, 0x03c7, 0x0069, 0x0072, 0x0075, 0x0076, 0x03b2, 0x03b3, + 0x03c1, 0x03c6, 0x03c7, 0x1d6b, 0x1d6c, 0x1d6d, 0x1d6e, 0x1d6f, + 0x1d70, 0x1d71, 0x1d72, 0x1d73, 0x1d74, 0x1d75, 0x1d76, 0x1d77, + 0x043d, 0x1d79, 0x1d7a, 0x1d7b, 0x1d7c, 0x1d7d, 0x1d7e, 0x1d7f, +}; + +static const unsigned short gNormalizeTable1d80[] = { + /* U+1d80 */ + 0x1d80, 0x1d81, 0x1d82, 0x1d83, 0x1d84, 0x1d85, 0x1d86, 0x1d87, + 0x1d88, 0x1d89, 0x1d8a, 0x1d8b, 0x1d8c, 0x1d8d, 0x1d8e, 0x1d8f, + 0x1d90, 0x1d91, 0x1d92, 0x1d93, 0x1d94, 0x1d95, 0x1d96, 0x1d97, + 0x1d98, 0x1d99, 0x1d9a, 0x0252, 0x0063, 0x0255, 0x00f0, 0x025c, + 0x0066, 0x025f, 0x0261, 0x0265, 0x0268, 0x0269, 0x026a, 0x1d7b, + 0x029d, 0x026d, 0x1d85, 0x029f, 0x0271, 0x0270, 0x0272, 0x0273, + 0x0274, 0x0275, 0x0278, 0x0282, 0x0283, 0x01ab, 0x0289, 0x028a, + 0x1d1c, 0x028b, 0x028c, 0x007a, 0x0290, 0x0291, 0x0292, 0x03b8, +}; + +static const unsigned short gNormalizeTable1e00[] = { + /* U+1e00 */ + 0x0061, 0x0061, 0x0062, 0x0062, 0x0062, 0x0062, 0x0062, 0x0062, + 0x0063, 0x0063, 0x0064, 0x0064, 0x0064, 0x0064, 0x0064, 0x0064, + 0x0064, 0x0064, 0x0064, 0x0064, 0x0065, 0x0065, 0x0065, 0x0065, + 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0066, 0x0066, + 0x0067, 0x0067, 0x0068, 0x0068, 0x0068, 0x0068, 0x0068, 0x0068, + 0x0068, 0x0068, 0x0068, 0x0068, 0x0069, 0x0069, 0x0069, 0x0069, + 0x006b, 0x006b, 0x006b, 0x006b, 0x006b, 0x006b, 0x006c, 0x006c, + 0x006c, 0x006c, 0x006c, 0x006c, 0x006c, 0x006c, 0x006d, 0x006d, +}; + +static const unsigned short gNormalizeTable1e40[] = { + /* U+1e40 */ + 0x006d, 0x006d, 0x006d, 0x006d, 0x006e, 0x006e, 0x006e, 0x006e, + 0x006e, 0x006e, 0x006e, 0x006e, 0x006f, 0x006f, 0x006f, 0x006f, + 0x006f, 0x006f, 0x006f, 0x006f, 0x0070, 0x0070, 0x0070, 0x0070, + 0x0072, 0x0072, 0x0072, 0x0072, 0x0072, 0x0072, 0x0072, 0x0072, + 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, 0x0073, + 0x0073, 0x0073, 0x0074, 0x0074, 0x0074, 0x0074, 0x0074, 0x0074, + 0x0074, 0x0074, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, + 0x0075, 0x0075, 0x0075, 0x0075, 0x0076, 0x0076, 0x0076, 0x0076, +}; + +static const unsigned short gNormalizeTable1e80[] = { + /* U+1e80 */ + 0x0077, 0x0077, 0x0077, 0x0077, 0x0077, 0x0077, 0x0077, 0x0077, + 0x0077, 0x0077, 0x0078, 0x0078, 0x0078, 0x0078, 0x0079, 0x0079, + 0x007a, 0x007a, 0x007a, 0x007a, 0x007a, 0x007a, 0x0068, 0x0074, + 0x0077, 0x0079, 0x0061, 0x0073, 0x1e9c, 0x1e9d, 0x0073, 0x1e9f, + 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, + 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, + 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, + 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, +}; + +static const unsigned short gNormalizeTable1ec0[] = { + /* U+1ec0 */ + 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, 0x0065, + 0x0069, 0x0069, 0x0069, 0x0069, 0x006f, 0x006f, 0x006f, 0x006f, + 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, + 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, + 0x006f, 0x006f, 0x006f, 0x006f, 0x0075, 0x0075, 0x0075, 0x0075, + 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, 0x0075, + 0x0075, 0x0075, 0x0079, 0x0079, 0x0079, 0x0079, 0x0079, 0x0079, + 0x0079, 0x0079, 0x1efb, 0x1efb, 0x1efd, 0x1efd, 0x1eff, 0x1eff, +}; + +static const unsigned short gNormalizeTable1f00[] = { + /* U+1f00 */ + 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, + 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, + 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x1f16, 0x1f17, + 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x03b5, 0x1f1e, 0x1f1f, + 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, + 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, + 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, + 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x03b9, +}; + +static const unsigned short gNormalizeTable1f40[] = { + /* U+1f40 */ + 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x1f46, 0x1f47, + 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x03bf, 0x1f4e, 0x1f4f, + 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c5, + 0x1f58, 0x03c5, 0x1f5a, 0x03c5, 0x1f5c, 0x03c5, 0x1f5e, 0x03c5, + 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, + 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, + 0x03b1, 0x03b1, 0x03b5, 0x03b5, 0x03b7, 0x03b7, 0x03b9, 0x03b9, + 0x03bf, 0x03bf, 0x03c5, 0x03c5, 0x03c9, 0x03c9, 0x1f7e, 0x1f7f, +}; + +static const unsigned short gNormalizeTable1f80[] = { + /* U+1f80 */ + 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, + 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, + 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, + 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x03b7, + 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, + 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, 0x03c9, + 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x1fb5, 0x03b1, 0x03b1, + 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x03b1, 0x0020, 0x03b9, 0x0020, +}; + +static const unsigned short gNormalizeTable1fc0[] = { + /* U+1fc0 */ + 0x0020, 0x0020, 0x03b7, 0x03b7, 0x03b7, 0x1fc5, 0x03b7, 0x03b7, + 0x03b5, 0x03b5, 0x03b7, 0x03b7, 0x03b7, 0x0020, 0x0020, 0x0020, + 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x1fd4, 0x1fd5, 0x03b9, 0x03b9, + 0x03b9, 0x03b9, 0x03b9, 0x03b9, 0x1fdc, 0x0020, 0x0020, 0x0020, + 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c1, 0x03c1, 0x03c5, 0x03c5, + 0x03c5, 0x03c5, 0x03c5, 0x03c5, 0x03c1, 0x0020, 0x0020, 0x0060, + 0x1ff0, 0x1ff1, 0x03c9, 0x03c9, 0x03c9, 0x1ff5, 0x03c9, 0x03c9, + 0x03bf, 0x03bf, 0x03c9, 0x03c9, 0x03c9, 0x0020, 0x0020, 0x1fff, +}; + +static const unsigned short gNormalizeTable2000[] = { + /* U+2000 */ + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x2010, 0x2010, 0x2012, 0x2013, 0x2014, 0x2015, 0x2016, 0x0020, + 0x2018, 0x2019, 0x201a, 0x201b, 0x201c, 0x201d, 0x201e, 0x201f, + 0x2020, 0x2021, 0x2022, 0x2023, 0x002e, 0x002e, 0x002e, 0x2027, + 0x2028, 0x2029, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x2030, 0x2031, 0x2032, 0x2032, 0x2032, 0x2035, 0x2035, 0x2035, + 0x2038, 0x2039, 0x203a, 0x203b, 0x0021, 0x203d, 0x0020, 0x203f, +}; + +static const unsigned short gNormalizeTable2040[] = { + /* U+2040 */ + 0x2040, 0x2041, 0x2042, 0x2043, 0x2044, 0x2045, 0x2046, 0x003f, + 0x003f, 0x0021, 0x204a, 0x204b, 0x204c, 0x204d, 0x204e, 0x204f, + 0x2050, 0x2051, 0x2052, 0x2053, 0x2054, 0x2055, 0x2056, 0x2032, + 0x2058, 0x2059, 0x205a, 0x205b, 0x205c, 0x205d, 0x205e, 0x0020, + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x0030, 0x0069, 0x2072, 0x2073, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x002b, 0x2212, 0x003d, 0x0028, 0x0029, 0x006e, +}; + +static const unsigned short gNormalizeTable2080[] = { + /* U+2080 */ + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x002b, 0x2212, 0x003d, 0x0028, 0x0029, 0x208f, + 0x0061, 0x0065, 0x006f, 0x0078, 0x0259, 0x2095, 0x2096, 0x2097, + 0x2098, 0x2099, 0x209a, 0x209b, 0x209c, 0x209d, 0x209e, 0x209f, + 0x20a0, 0x20a1, 0x20a2, 0x20a3, 0x20a4, 0x20a5, 0x20a6, 0x20a7, + 0x0072, 0x20a9, 0x20aa, 0x20ab, 0x20ac, 0x20ad, 0x20ae, 0x20af, + 0x20b0, 0x20b1, 0x20b2, 0x20b3, 0x20b4, 0x20b5, 0x20b6, 0x20b7, + 0x20b8, 0x20b9, 0x20ba, 0x20bb, 0x20bc, 0x20bd, 0x20be, 0x20bf, +}; + +static const unsigned short gNormalizeTable2100[] = { + /* U+2100 */ + 0x0061, 0x0061, 0x0063, 0x00b0, 0x2104, 0x0063, 0x0063, 0x025b, + 0x2108, 0x00b0, 0x0067, 0x0068, 0x0068, 0x0068, 0x0068, 0x0127, + 0x0069, 0x0069, 0x006c, 0x006c, 0x2114, 0x006e, 0x006e, 0x2117, + 0x2118, 0x0070, 0x0071, 0x0072, 0x0072, 0x0072, 0x211e, 0x211f, + 0x0073, 0x0074, 0x0074, 0x2123, 0x007a, 0x2125, 0x03c9, 0x2127, + 0x007a, 0x2129, 0x006b, 0x0061, 0x0062, 0x0063, 0x212e, 0x0065, + 0x0065, 0x0066, 0x214e, 0x006d, 0x006f, 0x05d0, 0x05d1, 0x05d2, + 0x05d3, 0x0069, 0x213a, 0x0066, 0x03c0, 0x03b3, 0x03b3, 0x03c0, +}; + +static const unsigned short gNormalizeTable2140[] = { + /* U+2140 */ + 0x2211, 0x2141, 0x2142, 0x2143, 0x2144, 0x0064, 0x0064, 0x0065, + 0x0069, 0x006a, 0x214a, 0x214b, 0x214c, 0x214d, 0x214e, 0x214f, + 0x0031, 0x0031, 0x0031, 0x0031, 0x0032, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0031, 0x0035, 0x0031, 0x0033, 0x0035, 0x0037, 0x0031, + 0x0069, 0x0069, 0x0069, 0x0069, 0x0076, 0x0076, 0x0076, 0x0076, + 0x0069, 0x0078, 0x0078, 0x0078, 0x006c, 0x0063, 0x0064, 0x006d, + 0x0069, 0x0069, 0x0069, 0x0069, 0x0076, 0x0076, 0x0076, 0x0076, + 0x0069, 0x0078, 0x0078, 0x0078, 0x006c, 0x0063, 0x0064, 0x006d, +}; + +static const unsigned short gNormalizeTable2180[] = { + /* U+2180 */ + 0x2180, 0x2181, 0x2182, 0x2184, 0x2184, 0x2185, 0x2186, 0x2187, + 0x2188, 0x0030, 0x218a, 0x218b, 0x218c, 0x218d, 0x218e, 0x218f, + 0x2190, 0x2191, 0x2192, 0x2193, 0x2194, 0x2195, 0x2196, 0x2197, + 0x2198, 0x2199, 0x2190, 0x2192, 0x219c, 0x219d, 0x219e, 0x219f, + 0x21a0, 0x21a1, 0x21a2, 0x21a3, 0x21a4, 0x21a5, 0x21a6, 0x21a7, + 0x21a8, 0x21a9, 0x21aa, 0x21ab, 0x21ac, 0x21ad, 0x2194, 0x21af, + 0x21b0, 0x21b1, 0x21b2, 0x21b3, 0x21b4, 0x21b5, 0x21b6, 0x21b7, + 0x21b8, 0x21b9, 0x21ba, 0x21bb, 0x21bc, 0x21bd, 0x21be, 0x21bf, +}; + +static const unsigned short gNormalizeTable21c0[] = { + /* U+21c0 */ + 0x21c0, 0x21c1, 0x21c2, 0x21c3, 0x21c4, 0x21c5, 0x21c6, 0x21c7, + 0x21c8, 0x21c9, 0x21ca, 0x21cb, 0x21cc, 0x21d0, 0x21d4, 0x21d2, + 0x21d0, 0x21d1, 0x21d2, 0x21d3, 0x21d4, 0x21d5, 0x21d6, 0x21d7, + 0x21d8, 0x21d9, 0x21da, 0x21db, 0x21dc, 0x21dd, 0x21de, 0x21df, + 0x21e0, 0x21e1, 0x21e2, 0x21e3, 0x21e4, 0x21e5, 0x21e6, 0x21e7, + 0x21e8, 0x21e9, 0x21ea, 0x21eb, 0x21ec, 0x21ed, 0x21ee, 0x21ef, + 0x21f0, 0x21f1, 0x21f2, 0x21f3, 0x21f4, 0x21f5, 0x21f6, 0x21f7, + 0x21f8, 0x21f9, 0x21fa, 0x21fb, 0x21fc, 0x21fd, 0x21fe, 0x21ff, +}; + +static const unsigned short gNormalizeTable2200[] = { + /* U+2200 */ + 0x2200, 0x2201, 0x2202, 0x2203, 0x2203, 0x2205, 0x2206, 0x2207, + 0x2208, 0x2208, 0x220a, 0x220b, 0x220b, 0x220d, 0x220e, 0x220f, + 0x2210, 0x2211, 0x2212, 0x2213, 0x2214, 0x2215, 0x2216, 0x2217, + 0x2218, 0x2219, 0x221a, 0x221b, 0x221c, 0x221d, 0x221e, 0x221f, + 0x2220, 0x2221, 0x2222, 0x2223, 0x2223, 0x2225, 0x2225, 0x2227, + 0x2228, 0x2229, 0x222a, 0x222b, 0x222b, 0x222b, 0x222e, 0x222e, + 0x222e, 0x2231, 0x2232, 0x2233, 0x2234, 0x2235, 0x2236, 0x2237, + 0x2238, 0x2239, 0x223a, 0x223b, 0x223c, 0x223d, 0x223e, 0x223f, +}; + +static const unsigned short gNormalizeTable2240[] = { + /* U+2240 */ + 0x2240, 0x223c, 0x2242, 0x2243, 0x2243, 0x2245, 0x2246, 0x2245, + 0x2248, 0x2248, 0x224a, 0x224b, 0x224c, 0x224d, 0x224e, 0x224f, + 0x2250, 0x2251, 0x2252, 0x2253, 0x2254, 0x2255, 0x2256, 0x2257, + 0x2258, 0x2259, 0x225a, 0x225b, 0x225c, 0x225d, 0x225e, 0x225f, + 0x003d, 0x2261, 0x2261, 0x2263, 0x2264, 0x2265, 0x2266, 0x2267, + 0x2268, 0x2269, 0x226a, 0x226b, 0x226c, 0x224d, 0x003c, 0x003e, + 0x2264, 0x2265, 0x2272, 0x2273, 0x2272, 0x2273, 0x2276, 0x2277, + 0x2276, 0x2277, 0x227a, 0x227b, 0x227c, 0x227d, 0x227e, 0x227f, +}; + +static const unsigned short gNormalizeTable2280[] = { + /* U+2280 */ + 0x227a, 0x227b, 0x2282, 0x2283, 0x2282, 0x2283, 0x2286, 0x2287, + 0x2286, 0x2287, 0x228a, 0x228b, 0x228c, 0x228d, 0x228e, 0x228f, + 0x2290, 0x2291, 0x2292, 0x2293, 0x2294, 0x2295, 0x2296, 0x2297, + 0x2298, 0x2299, 0x229a, 0x229b, 0x229c, 0x229d, 0x229e, 0x229f, + 0x22a0, 0x22a1, 0x22a2, 0x22a3, 0x22a4, 0x22a5, 0x22a6, 0x22a7, + 0x22a8, 0x22a9, 0x22aa, 0x22ab, 0x22a2, 0x22a8, 0x22a9, 0x22ab, + 0x22b0, 0x22b1, 0x22b2, 0x22b3, 0x22b4, 0x22b5, 0x22b6, 0x22b7, + 0x22b8, 0x22b9, 0x22ba, 0x22bb, 0x22bc, 0x22bd, 0x22be, 0x22bf, +}; + +static const unsigned short gNormalizeTable22c0[] = { + /* U+22c0 */ + 0x22c0, 0x22c1, 0x22c2, 0x22c3, 0x22c4, 0x22c5, 0x22c6, 0x22c7, + 0x22c8, 0x22c9, 0x22ca, 0x22cb, 0x22cc, 0x22cd, 0x22ce, 0x22cf, + 0x22d0, 0x22d1, 0x22d2, 0x22d3, 0x22d4, 0x22d5, 0x22d6, 0x22d7, + 0x22d8, 0x22d9, 0x22da, 0x22db, 0x22dc, 0x22dd, 0x22de, 0x22df, + 0x227c, 0x227d, 0x2291, 0x2292, 0x22e4, 0x22e5, 0x22e6, 0x22e7, + 0x22e8, 0x22e9, 0x22b2, 0x22b3, 0x22b4, 0x22b5, 0x22ee, 0x22ef, + 0x22f0, 0x22f1, 0x22f2, 0x22f3, 0x22f4, 0x22f5, 0x22f6, 0x22f7, + 0x22f8, 0x22f9, 0x22fa, 0x22fb, 0x22fc, 0x22fd, 0x22fe, 0x22ff, +}; + +static const unsigned short gNormalizeTable2300[] = { + /* U+2300 */ + 0x2300, 0x2301, 0x2302, 0x2303, 0x2304, 0x2305, 0x2306, 0x2307, + 0x2308, 0x2309, 0x230a, 0x230b, 0x230c, 0x230d, 0x230e, 0x230f, + 0x2310, 0x2311, 0x2312, 0x2313, 0x2314, 0x2315, 0x2316, 0x2317, + 0x2318, 0x2319, 0x231a, 0x231b, 0x231c, 0x231d, 0x231e, 0x231f, + 0x2320, 0x2321, 0x2322, 0x2323, 0x2324, 0x2325, 0x2326, 0x2327, + 0x2328, 0x3008, 0x3009, 0x232b, 0x232c, 0x232d, 0x232e, 0x232f, + 0x2330, 0x2331, 0x2332, 0x2333, 0x2334, 0x2335, 0x2336, 0x2337, + 0x2338, 0x2339, 0x233a, 0x233b, 0x233c, 0x233d, 0x233e, 0x233f, +}; + +static const unsigned short gNormalizeTable2440[] = { + /* U+2440 */ + 0x2440, 0x2441, 0x2442, 0x2443, 0x2444, 0x2445, 0x2446, 0x2447, + 0x2448, 0x2449, 0x244a, 0x244b, 0x244c, 0x244d, 0x244e, 0x244f, + 0x2450, 0x2451, 0x2452, 0x2453, 0x2454, 0x2455, 0x2456, 0x2457, + 0x2458, 0x2459, 0x245a, 0x245b, 0x245c, 0x245d, 0x245e, 0x245f, + 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, + 0x0039, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, + 0x0031, 0x0031, 0x0031, 0x0032, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, +}; + +static const unsigned short gNormalizeTable2480[] = { + /* U+2480 */ + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, + 0x0039, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, + 0x0031, 0x0031, 0x0031, 0x0032, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0061, 0x0062, + 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, +}; + +static const unsigned short gNormalizeTable24c0[] = { + /* U+24c0 */ + 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, 0x0070, 0x0071, 0x0072, + 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, + 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, + 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, 0x0070, + 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, + 0x0079, 0x007a, 0x0030, 0x24eb, 0x24ec, 0x24ed, 0x24ee, 0x24ef, + 0x24f0, 0x24f1, 0x24f2, 0x24f3, 0x24f4, 0x24f5, 0x24f6, 0x24f7, + 0x24f8, 0x24f9, 0x24fa, 0x24fb, 0x24fc, 0x24fd, 0x24fe, 0x24ff, +}; + +static const unsigned short gNormalizeTable2a00[] = { + /* U+2a00 */ + 0x2a00, 0x2a01, 0x2a02, 0x2a03, 0x2a04, 0x2a05, 0x2a06, 0x2a07, + 0x2a08, 0x2a09, 0x2a0a, 0x2a0b, 0x222b, 0x2a0d, 0x2a0e, 0x2a0f, + 0x2a10, 0x2a11, 0x2a12, 0x2a13, 0x2a14, 0x2a15, 0x2a16, 0x2a17, + 0x2a18, 0x2a19, 0x2a1a, 0x2a1b, 0x2a1c, 0x2a1d, 0x2a1e, 0x2a1f, + 0x2a20, 0x2a21, 0x2a22, 0x2a23, 0x2a24, 0x2a25, 0x2a26, 0x2a27, + 0x2a28, 0x2a29, 0x2a2a, 0x2a2b, 0x2a2c, 0x2a2d, 0x2a2e, 0x2a2f, + 0x2a30, 0x2a31, 0x2a32, 0x2a33, 0x2a34, 0x2a35, 0x2a36, 0x2a37, + 0x2a38, 0x2a39, 0x2a3a, 0x2a3b, 0x2a3c, 0x2a3d, 0x2a3e, 0x2a3f, +}; + +static const unsigned short gNormalizeTable2a40[] = { + /* U+2a40 */ + 0x2a40, 0x2a41, 0x2a42, 0x2a43, 0x2a44, 0x2a45, 0x2a46, 0x2a47, + 0x2a48, 0x2a49, 0x2a4a, 0x2a4b, 0x2a4c, 0x2a4d, 0x2a4e, 0x2a4f, + 0x2a50, 0x2a51, 0x2a52, 0x2a53, 0x2a54, 0x2a55, 0x2a56, 0x2a57, + 0x2a58, 0x2a59, 0x2a5a, 0x2a5b, 0x2a5c, 0x2a5d, 0x2a5e, 0x2a5f, + 0x2a60, 0x2a61, 0x2a62, 0x2a63, 0x2a64, 0x2a65, 0x2a66, 0x2a67, + 0x2a68, 0x2a69, 0x2a6a, 0x2a6b, 0x2a6c, 0x2a6d, 0x2a6e, 0x2a6f, + 0x2a70, 0x2a71, 0x2a72, 0x2a73, 0x003a, 0x003d, 0x003d, 0x2a77, + 0x2a78, 0x2a79, 0x2a7a, 0x2a7b, 0x2a7c, 0x2a7d, 0x2a7e, 0x2a7f, +}; + +static const unsigned short gNormalizeTable2ac0[] = { + /* U+2ac0 */ + 0x2ac0, 0x2ac1, 0x2ac2, 0x2ac3, 0x2ac4, 0x2ac5, 0x2ac6, 0x2ac7, + 0x2ac8, 0x2ac9, 0x2aca, 0x2acb, 0x2acc, 0x2acd, 0x2ace, 0x2acf, + 0x2ad0, 0x2ad1, 0x2ad2, 0x2ad3, 0x2ad4, 0x2ad5, 0x2ad6, 0x2ad7, + 0x2ad8, 0x2ad9, 0x2ada, 0x2adb, 0x2add, 0x2add, 0x2ade, 0x2adf, + 0x2ae0, 0x2ae1, 0x2ae2, 0x2ae3, 0x2ae4, 0x2ae5, 0x2ae6, 0x2ae7, + 0x2ae8, 0x2ae9, 0x2aea, 0x2aeb, 0x2aec, 0x2aed, 0x2aee, 0x2aef, + 0x2af0, 0x2af1, 0x2af2, 0x2af3, 0x2af4, 0x2af5, 0x2af6, 0x2af7, + 0x2af8, 0x2af9, 0x2afa, 0x2afb, 0x2afc, 0x2afd, 0x2afe, 0x2aff, +}; + +static const unsigned short gNormalizeTable2c00[] = { + /* U+2c00 */ + 0x2c30, 0x2c31, 0x2c32, 0x2c33, 0x2c34, 0x2c35, 0x2c36, 0x2c37, + 0x2c38, 0x2c39, 0x2c3a, 0x2c3b, 0x2c3c, 0x2c3d, 0x2c3e, 0x2c3f, + 0x2c40, 0x2c41, 0x2c42, 0x2c43, 0x2c44, 0x2c45, 0x2c46, 0x2c47, + 0x2c48, 0x2c49, 0x2c4a, 0x2c4b, 0x2c4c, 0x2c4d, 0x2c4e, 0x2c4f, + 0x2c50, 0x2c51, 0x2c52, 0x2c53, 0x2c54, 0x2c55, 0x2c56, 0x2c57, + 0x2c58, 0x2c59, 0x2c5a, 0x2c5b, 0x2c5c, 0x2c5d, 0x2c5e, 0x2c2f, + 0x2c30, 0x2c31, 0x2c32, 0x2c33, 0x2c34, 0x2c35, 0x2c36, 0x2c37, + 0x2c38, 0x2c39, 0x2c3a, 0x2c3b, 0x2c3c, 0x2c3d, 0x2c3e, 0x2c3f, +}; + +static const unsigned short gNormalizeTable2c40[] = { + /* U+2c40 */ + 0x2c40, 0x2c41, 0x2c42, 0x2c43, 0x2c44, 0x2c45, 0x2c46, 0x2c47, + 0x2c48, 0x2c49, 0x2c4a, 0x2c4b, 0x2c4c, 0x2c4d, 0x2c4e, 0x2c4f, + 0x2c50, 0x2c51, 0x2c52, 0x2c53, 0x2c54, 0x2c55, 0x2c56, 0x2c57, + 0x2c58, 0x2c59, 0x2c5a, 0x2c5b, 0x2c5c, 0x2c5d, 0x2c5e, 0x2c5f, + 0x2c61, 0x2c61, 0x026b, 0x1d7d, 0x027d, 0x2c65, 0x2c66, 0x2c68, + 0x2c68, 0x2c6a, 0x2c6a, 0x2c6c, 0x2c6c, 0x0251, 0x0271, 0x0250, + 0x0252, 0x2c71, 0x2c73, 0x2c73, 0x2c74, 0x2c76, 0x2c76, 0x2c77, + 0x2c78, 0x2c79, 0x2c7a, 0x2c7b, 0x006a, 0x0076, 0x023f, 0x0240, +}; + +static const unsigned short gNormalizeTable2c80[] = { + /* U+2c80 */ + 0x2c81, 0x2c81, 0x2c83, 0x2c83, 0x2c85, 0x2c85, 0x2c87, 0x2c87, + 0x2c89, 0x2c89, 0x2c8b, 0x2c8b, 0x2c8d, 0x2c8d, 0x2c8f, 0x2c8f, + 0x2c91, 0x2c91, 0x2c93, 0x2c93, 0x2c95, 0x2c95, 0x2c97, 0x2c97, + 0x2c99, 0x2c99, 0x2c9b, 0x2c9b, 0x2c9d, 0x2c9d, 0x2c9f, 0x2c9f, + 0x2ca1, 0x2ca1, 0x2ca3, 0x2ca3, 0x2ca5, 0x2ca5, 0x2ca7, 0x2ca7, + 0x2ca9, 0x2ca9, 0x2cab, 0x2cab, 0x2cad, 0x2cad, 0x2caf, 0x2caf, + 0x2cb1, 0x2cb1, 0x2cb3, 0x2cb3, 0x2cb5, 0x2cb5, 0x2cb7, 0x2cb7, + 0x2cb9, 0x2cb9, 0x2cbb, 0x2cbb, 0x2cbd, 0x2cbd, 0x2cbf, 0x2cbf, +}; + +static const unsigned short gNormalizeTable2cc0[] = { + /* U+2cc0 */ + 0x2cc1, 0x2cc1, 0x2cc3, 0x2cc3, 0x2cc5, 0x2cc5, 0x2cc7, 0x2cc7, + 0x2cc9, 0x2cc9, 0x2ccb, 0x2ccb, 0x2ccd, 0x2ccd, 0x2ccf, 0x2ccf, + 0x2cd1, 0x2cd1, 0x2cd3, 0x2cd3, 0x2cd5, 0x2cd5, 0x2cd7, 0x2cd7, + 0x2cd9, 0x2cd9, 0x2cdb, 0x2cdb, 0x2cdd, 0x2cdd, 0x2cdf, 0x2cdf, + 0x2ce1, 0x2ce1, 0x2ce3, 0x2ce3, 0x2ce4, 0x2ce5, 0x2ce6, 0x2ce7, + 0x2ce8, 0x2ce9, 0x2cea, 0x2cec, 0x2cec, 0x2cee, 0x2cee, 0x2cef, + 0x2cf0, 0x2cf1, 0x2cf2, 0x2cf3, 0x2cf4, 0x2cf5, 0x2cf6, 0x2cf7, + 0x2cf8, 0x2cf9, 0x2cfa, 0x2cfb, 0x2cfc, 0x2cfd, 0x2cfe, 0x2cff, +}; + +static const unsigned short gNormalizeTable2d40[] = { + /* U+2d40 */ + 0x2d40, 0x2d41, 0x2d42, 0x2d43, 0x2d44, 0x2d45, 0x2d46, 0x2d47, + 0x2d48, 0x2d49, 0x2d4a, 0x2d4b, 0x2d4c, 0x2d4d, 0x2d4e, 0x2d4f, + 0x2d50, 0x2d51, 0x2d52, 0x2d53, 0x2d54, 0x2d55, 0x2d56, 0x2d57, + 0x2d58, 0x2d59, 0x2d5a, 0x2d5b, 0x2d5c, 0x2d5d, 0x2d5e, 0x2d5f, + 0x2d60, 0x2d61, 0x2d62, 0x2d63, 0x2d64, 0x2d65, 0x2d66, 0x2d67, + 0x2d68, 0x2d69, 0x2d6a, 0x2d6b, 0x2d6c, 0x2d6d, 0x2d6e, 0x2d61, + 0x2d70, 0x2d71, 0x2d72, 0x2d73, 0x2d74, 0x2d75, 0x2d76, 0x2d77, + 0x2d78, 0x2d79, 0x2d7a, 0x2d7b, 0x2d7c, 0x2d7d, 0x2d7e, 0x2d7f, +}; + +static const unsigned short gNormalizeTable2e80[] = { + /* U+2e80 */ + 0x2e80, 0x2e81, 0x2e82, 0x2e83, 0x2e84, 0x2e85, 0x2e86, 0x2e87, + 0x2e88, 0x2e89, 0x2e8a, 0x2e8b, 0x2e8c, 0x2e8d, 0x2e8e, 0x2e8f, + 0x2e90, 0x2e91, 0x2e92, 0x2e93, 0x2e94, 0x2e95, 0x2e96, 0x2e97, + 0x2e98, 0x2e99, 0x2e9a, 0x2e9b, 0x2e9c, 0x2e9d, 0x2e9e, 0x6bcd, + 0x2ea0, 0x2ea1, 0x2ea2, 0x2ea3, 0x2ea4, 0x2ea5, 0x2ea6, 0x2ea7, + 0x2ea8, 0x2ea9, 0x2eaa, 0x2eab, 0x2eac, 0x2ead, 0x2eae, 0x2eaf, + 0x2eb0, 0x2eb1, 0x2eb2, 0x2eb3, 0x2eb4, 0x2eb5, 0x2eb6, 0x2eb7, + 0x2eb8, 0x2eb9, 0x2eba, 0x2ebb, 0x2ebc, 0x2ebd, 0x2ebe, 0x2ebf, +}; + +static const unsigned short gNormalizeTable2ec0[] = { + /* U+2ec0 */ + 0x2ec0, 0x2ec1, 0x2ec2, 0x2ec3, 0x2ec4, 0x2ec5, 0x2ec6, 0x2ec7, + 0x2ec8, 0x2ec9, 0x2eca, 0x2ecb, 0x2ecc, 0x2ecd, 0x2ece, 0x2ecf, + 0x2ed0, 0x2ed1, 0x2ed2, 0x2ed3, 0x2ed4, 0x2ed5, 0x2ed6, 0x2ed7, + 0x2ed8, 0x2ed9, 0x2eda, 0x2edb, 0x2edc, 0x2edd, 0x2ede, 0x2edf, + 0x2ee0, 0x2ee1, 0x2ee2, 0x2ee3, 0x2ee4, 0x2ee5, 0x2ee6, 0x2ee7, + 0x2ee8, 0x2ee9, 0x2eea, 0x2eeb, 0x2eec, 0x2eed, 0x2eee, 0x2eef, + 0x2ef0, 0x2ef1, 0x2ef2, 0x9f9f, 0x2ef4, 0x2ef5, 0x2ef6, 0x2ef7, + 0x2ef8, 0x2ef9, 0x2efa, 0x2efb, 0x2efc, 0x2efd, 0x2efe, 0x2eff, +}; + +static const unsigned short gNormalizeTable2f00[] = { + /* U+2f00 */ + 0x4e00, 0x4e28, 0x4e36, 0x4e3f, 0x4e59, 0x4e85, 0x4e8c, 0x4ea0, + 0x4eba, 0x513f, 0x5165, 0x516b, 0x5182, 0x5196, 0x51ab, 0x51e0, + 0x51f5, 0x5200, 0x529b, 0x52f9, 0x5315, 0x531a, 0x5338, 0x5341, + 0x535c, 0x5369, 0x5382, 0x53b6, 0x53c8, 0x53e3, 0x56d7, 0x571f, + 0x58eb, 0x5902, 0x590a, 0x5915, 0x5927, 0x5973, 0x5b50, 0x5b80, + 0x5bf8, 0x5c0f, 0x5c22, 0x5c38, 0x5c6e, 0x5c71, 0x5ddb, 0x5de5, + 0x5df1, 0x5dfe, 0x5e72, 0x5e7a, 0x5e7f, 0x5ef4, 0x5efe, 0x5f0b, + 0x5f13, 0x5f50, 0x5f61, 0x5f73, 0x5fc3, 0x6208, 0x6236, 0x624b, +}; + +static const unsigned short gNormalizeTable2f40[] = { + /* U+2f40 */ + 0x652f, 0x6534, 0x6587, 0x6597, 0x65a4, 0x65b9, 0x65e0, 0x65e5, + 0x66f0, 0x6708, 0x6728, 0x6b20, 0x6b62, 0x6b79, 0x6bb3, 0x6bcb, + 0x6bd4, 0x6bdb, 0x6c0f, 0x6c14, 0x6c34, 0x706b, 0x722a, 0x7236, + 0x723b, 0x723f, 0x7247, 0x7259, 0x725b, 0x72ac, 0x7384, 0x7389, + 0x74dc, 0x74e6, 0x7518, 0x751f, 0x7528, 0x7530, 0x758b, 0x7592, + 0x7676, 0x767d, 0x76ae, 0x76bf, 0x76ee, 0x77db, 0x77e2, 0x77f3, + 0x793a, 0x79b8, 0x79be, 0x7a74, 0x7acb, 0x7af9, 0x7c73, 0x7cf8, + 0x7f36, 0x7f51, 0x7f8a, 0x7fbd, 0x8001, 0x800c, 0x8012, 0x8033, +}; + +static const unsigned short gNormalizeTable2f80[] = { + /* U+2f80 */ + 0x807f, 0x8089, 0x81e3, 0x81ea, 0x81f3, 0x81fc, 0x820c, 0x821b, + 0x821f, 0x826e, 0x8272, 0x8278, 0x864d, 0x866b, 0x8840, 0x884c, + 0x8863, 0x897e, 0x898b, 0x89d2, 0x8a00, 0x8c37, 0x8c46, 0x8c55, + 0x8c78, 0x8c9d, 0x8d64, 0x8d70, 0x8db3, 0x8eab, 0x8eca, 0x8f9b, + 0x8fb0, 0x8fb5, 0x9091, 0x9149, 0x91c6, 0x91cc, 0x91d1, 0x9577, + 0x9580, 0x961c, 0x96b6, 0x96b9, 0x96e8, 0x9751, 0x975e, 0x9762, + 0x9769, 0x97cb, 0x97ed, 0x97f3, 0x9801, 0x98a8, 0x98db, 0x98df, + 0x9996, 0x9999, 0x99ac, 0x9aa8, 0x9ad8, 0x9adf, 0x9b25, 0x9b2f, +}; + +static const unsigned short gNormalizeTable2fc0[] = { + /* U+2fc0 */ + 0x9b32, 0x9b3c, 0x9b5a, 0x9ce5, 0x9e75, 0x9e7f, 0x9ea5, 0x9ebb, + 0x9ec3, 0x9ecd, 0x9ed1, 0x9ef9, 0x9efd, 0x9f0e, 0x9f13, 0x9f20, + 0x9f3b, 0x9f4a, 0x9f52, 0x9f8d, 0x9f9c, 0x9fa0, 0x2fd6, 0x2fd7, + 0x2fd8, 0x2fd9, 0x2fda, 0x2fdb, 0x2fdc, 0x2fdd, 0x2fde, 0x2fdf, + 0x2fe0, 0x2fe1, 0x2fe2, 0x2fe3, 0x2fe4, 0x2fe5, 0x2fe6, 0x2fe7, + 0x2fe8, 0x2fe9, 0x2fea, 0x2feb, 0x2fec, 0x2fed, 0x2fee, 0x2fef, + 0x2ff0, 0x2ff1, 0x2ff2, 0x2ff3, 0x2ff4, 0x2ff5, 0x2ff6, 0x2ff7, + 0x2ff8, 0x2ff9, 0x2ffa, 0x2ffb, 0x2ffc, 0x2ffd, 0x2ffe, 0x2fff, +}; + +static const unsigned short gNormalizeTable3000[] = { + /* U+3000 */ + 0x0020, 0x3001, 0x3002, 0x3003, 0x3004, 0x3005, 0x3006, 0x3007, + 0x3008, 0x3009, 0x300a, 0x300b, 0x300c, 0x300d, 0x300e, 0x300f, + 0x3010, 0x3011, 0x3012, 0x3013, 0x3014, 0x3015, 0x3016, 0x3017, + 0x3018, 0x3019, 0x301a, 0x301b, 0x301c, 0x301d, 0x301e, 0x301f, + 0x3020, 0x3021, 0x3022, 0x3023, 0x3024, 0x3025, 0x3026, 0x3027, + 0x3028, 0x3029, 0x302a, 0x302b, 0x302c, 0x302d, 0x302e, 0x302f, + 0x3030, 0x3031, 0x3032, 0x3033, 0x3034, 0x3035, 0x3012, 0x3037, + 0x5341, 0x5344, 0x5345, 0x303b, 0x303c, 0x303d, 0x303e, 0x303f, +}; + +static const unsigned short gNormalizeTable3040[] = { + /* U+3040 */ + 0x3040, 0x3041, 0x3042, 0x3043, 0x3044, 0x3045, 0x3046, 0x3047, + 0x3048, 0x3049, 0x304a, 0x304b, 0x304b, 0x304d, 0x304d, 0x304f, + 0x304f, 0x3051, 0x3051, 0x3053, 0x3053, 0x3055, 0x3055, 0x3057, + 0x3057, 0x3059, 0x3059, 0x305b, 0x305b, 0x305d, 0x305d, 0x305f, + 0x305f, 0x3061, 0x3061, 0x3063, 0x3064, 0x3064, 0x3066, 0x3066, + 0x3068, 0x3068, 0x306a, 0x306b, 0x306c, 0x306d, 0x306e, 0x306f, + 0x306f, 0x306f, 0x3072, 0x3072, 0x3072, 0x3075, 0x3075, 0x3075, + 0x3078, 0x3078, 0x3078, 0x307b, 0x307b, 0x307b, 0x307e, 0x307f, +}; + +static const unsigned short gNormalizeTable3080[] = { + /* U+3080 */ + 0x3080, 0x3081, 0x3082, 0x3083, 0x3084, 0x3085, 0x3086, 0x3087, + 0x3088, 0x3089, 0x308a, 0x308b, 0x308c, 0x308d, 0x308e, 0x308f, + 0x3090, 0x3091, 0x3092, 0x3093, 0x3046, 0x3095, 0x3096, 0x3097, + 0x3098, 0x3099, 0x309a, 0x0020, 0x0020, 0x309d, 0x309d, 0x3088, + 0x30a0, 0x30a1, 0x30a2, 0x30a3, 0x30a4, 0x30a5, 0x30a6, 0x30a7, + 0x30a8, 0x30a9, 0x30aa, 0x30ab, 0x30ab, 0x30ad, 0x30ad, 0x30af, + 0x30af, 0x30b1, 0x30b1, 0x30b3, 0x30b3, 0x30b5, 0x30b5, 0x30b7, + 0x30b7, 0x30b9, 0x30b9, 0x30bb, 0x30bb, 0x30bd, 0x30bd, 0x30bf, +}; + +static const unsigned short gNormalizeTable30c0[] = { + /* U+30c0 */ + 0x30bf, 0x30c1, 0x30c1, 0x30c3, 0x30c4, 0x30c4, 0x30c6, 0x30c6, + 0x30c8, 0x30c8, 0x30ca, 0x30cb, 0x30cc, 0x30cd, 0x30ce, 0x30cf, + 0x30cf, 0x30cf, 0x30d2, 0x30d2, 0x30d2, 0x30d5, 0x30d5, 0x30d5, + 0x30d8, 0x30d8, 0x30d8, 0x30db, 0x30db, 0x30db, 0x30de, 0x30df, + 0x30e0, 0x30e1, 0x30e2, 0x30e3, 0x30e4, 0x30e5, 0x30e6, 0x30e7, + 0x30e8, 0x30e9, 0x30ea, 0x30eb, 0x30ec, 0x30ed, 0x30ee, 0x30ef, + 0x30f0, 0x30f1, 0x30f2, 0x30f3, 0x30a6, 0x30f5, 0x30f6, 0x30ef, + 0x30f0, 0x30f1, 0x30f2, 0x30fb, 0x30fc, 0x30fd, 0x30fd, 0x30b3, +}; + +static const unsigned short gNormalizeTable3100[] = { + /* U+3100 */ + 0x3100, 0x3101, 0x3102, 0x3103, 0x3104, 0x3105, 0x3106, 0x3107, + 0x3108, 0x3109, 0x310a, 0x310b, 0x310c, 0x310d, 0x310e, 0x310f, + 0x3110, 0x3111, 0x3112, 0x3113, 0x3114, 0x3115, 0x3116, 0x3117, + 0x3118, 0x3119, 0x311a, 0x311b, 0x311c, 0x311d, 0x311e, 0x311f, + 0x3120, 0x3121, 0x3122, 0x3123, 0x3124, 0x3125, 0x3126, 0x3127, + 0x3128, 0x3129, 0x312a, 0x312b, 0x312c, 0x312d, 0x312e, 0x312f, + 0x3130, 0x1100, 0x1101, 0x11aa, 0x1102, 0x11ac, 0x11ad, 0x1103, + 0x1104, 0x1105, 0x11b0, 0x11b1, 0x11b2, 0x11b3, 0x11b4, 0x11b5, +}; + +static const unsigned short gNormalizeTable3140[] = { + /* U+3140 */ + 0x111a, 0x1106, 0x1107, 0x1108, 0x1121, 0x1109, 0x110a, 0x110b, + 0x110c, 0x110d, 0x110e, 0x110f, 0x1110, 0x1111, 0x1112, 0x1161, + 0x1162, 0x1163, 0x1164, 0x1165, 0x1166, 0x1167, 0x1168, 0x1169, + 0x116a, 0x116b, 0x116c, 0x116d, 0x116e, 0x116f, 0x1170, 0x1171, + 0x1172, 0x1173, 0x1174, 0x1175, 0x0020, 0x1114, 0x1115, 0x11c7, + 0x11c8, 0x11cc, 0x11ce, 0x11d3, 0x11d7, 0x11d9, 0x111c, 0x11dd, + 0x11df, 0x111d, 0x111e, 0x1120, 0x1122, 0x1123, 0x1127, 0x1129, + 0x112b, 0x112c, 0x112d, 0x112e, 0x112f, 0x1132, 0x1136, 0x1140, +}; + +static const unsigned short gNormalizeTable3180[] = { + /* U+3180 */ + 0x1147, 0x114c, 0x11f1, 0x11f2, 0x1157, 0x1158, 0x1159, 0x1184, + 0x1185, 0x1188, 0x1191, 0x1192, 0x1194, 0x119e, 0x11a1, 0x318f, + 0x3190, 0x3191, 0x4e00, 0x4e8c, 0x4e09, 0x56db, 0x4e0a, 0x4e2d, + 0x4e0b, 0x7532, 0x4e59, 0x4e19, 0x4e01, 0x5929, 0x5730, 0x4eba, + 0x31a0, 0x31a1, 0x31a2, 0x31a3, 0x31a4, 0x31a5, 0x31a6, 0x31a7, + 0x31a8, 0x31a9, 0x31aa, 0x31ab, 0x31ac, 0x31ad, 0x31ae, 0x31af, + 0x31b0, 0x31b1, 0x31b2, 0x31b3, 0x31b4, 0x31b5, 0x31b6, 0x31b7, + 0x31b8, 0x31b9, 0x31ba, 0x31bb, 0x31bc, 0x31bd, 0x31be, 0x31bf, +}; + +static const unsigned short gNormalizeTable3200[] = { + /* U+3200 */ + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x321f, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, + 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, 0x0028, +}; + +static const unsigned short gNormalizeTable3240[] = { + /* U+3240 */ + 0x0028, 0x0028, 0x0028, 0x0028, 0x554f, 0x5e7c, 0x6587, 0x7b8f, + 0x3248, 0x3249, 0x324a, 0x324b, 0x324c, 0x324d, 0x324e, 0x324f, + 0x0070, 0x0032, 0x0032, 0x0032, 0x0032, 0x0032, 0x0032, 0x0032, + 0x0032, 0x0032, 0x0033, 0x0033, 0x0033, 0x0033, 0x0033, 0x0033, + 0x1100, 0x1102, 0x1103, 0x1105, 0x1106, 0x1107, 0x1109, 0x110b, + 0x110c, 0x110e, 0x110f, 0x1110, 0x1111, 0x1112, 0x1100, 0x1102, + 0x1103, 0x1105, 0x1106, 0x1107, 0x1109, 0x110b, 0x110c, 0x110e, + 0x110f, 0x1110, 0x1111, 0x1112, 0x110e, 0x110c, 0x110b, 0x327f, +}; + +static const unsigned short gNormalizeTable3280[] = { + /* U+3280 */ + 0x4e00, 0x4e8c, 0x4e09, 0x56db, 0x4e94, 0x516d, 0x4e03, 0x516b, + 0x4e5d, 0x5341, 0x6708, 0x706b, 0x6c34, 0x6728, 0x91d1, 0x571f, + 0x65e5, 0x682a, 0x6709, 0x793e, 0x540d, 0x7279, 0x8ca1, 0x795d, + 0x52b4, 0x79d8, 0x7537, 0x5973, 0x9069, 0x512a, 0x5370, 0x6ce8, + 0x9805, 0x4f11, 0x5199, 0x6b63, 0x4e0a, 0x4e2d, 0x4e0b, 0x5de6, + 0x53f3, 0x533b, 0x5b97, 0x5b66, 0x76e3, 0x4f01, 0x8cc7, 0x5354, + 0x591c, 0x0033, 0x0033, 0x0033, 0x0033, 0x0034, 0x0034, 0x0034, + 0x0034, 0x0034, 0x0034, 0x0034, 0x0034, 0x0034, 0x0034, 0x0035, +}; + +static const unsigned short gNormalizeTable32c0[] = { + /* U+32c0 */ + 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, + 0x0039, 0x0031, 0x0031, 0x0031, 0x0068, 0x0065, 0x0065, 0x006c, + 0x30a2, 0x30a4, 0x30a6, 0x30a8, 0x30aa, 0x30ab, 0x30ad, 0x30af, + 0x30b1, 0x30b3, 0x30b5, 0x30b7, 0x30b9, 0x30bb, 0x30bd, 0x30bf, + 0x30c1, 0x30c4, 0x30c6, 0x30c8, 0x30ca, 0x30cb, 0x30cc, 0x30cd, + 0x30ce, 0x30cf, 0x30d2, 0x30d5, 0x30d8, 0x30db, 0x30de, 0x30df, + 0x30e0, 0x30e1, 0x30e2, 0x30e4, 0x30e6, 0x30e8, 0x30e9, 0x30ea, + 0x30eb, 0x30ec, 0x30ed, 0x30ef, 0x30f0, 0x30f1, 0x30f2, 0x32ff, +}; + +static const unsigned short gNormalizeTable3300[] = { + /* U+3300 */ + 0x30a2, 0x30a2, 0x30a2, 0x30a2, 0x30a4, 0x30a4, 0x30a6, 0x30a8, + 0x30a8, 0x30aa, 0x30aa, 0x30ab, 0x30ab, 0x30ab, 0x30ab, 0x30ab, + 0x30ad, 0x30ad, 0x30ad, 0x30ad, 0x30ad, 0x30ad, 0x30ad, 0x30ad, + 0x30af, 0x30af, 0x30af, 0x30af, 0x30b1, 0x30b3, 0x30b3, 0x30b5, + 0x30b5, 0x30b7, 0x30bb, 0x30bb, 0x30bf, 0x30c6, 0x30c8, 0x30c8, + 0x30ca, 0x30ce, 0x30cf, 0x30cf, 0x30cf, 0x30cf, 0x30d2, 0x30d2, + 0x30d2, 0x30d2, 0x30d5, 0x30d5, 0x30d5, 0x30d5, 0x30d8, 0x30d8, + 0x30d8, 0x30d8, 0x30d8, 0x30d8, 0x30d8, 0x30db, 0x30db, 0x30db, +}; + +static const unsigned short gNormalizeTable3340[] = { + /* U+3340 */ + 0x30db, 0x30db, 0x30db, 0x30de, 0x30de, 0x30de, 0x30de, 0x30de, + 0x30df, 0x30df, 0x30df, 0x30e1, 0x30e1, 0x30e1, 0x30e4, 0x30e4, + 0x30e6, 0x30ea, 0x30ea, 0x30eb, 0x30eb, 0x30ec, 0x30ec, 0x30ef, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, + 0x0031, 0x0031, 0x0031, 0x0031, 0x0032, 0x0032, 0x0032, 0x0032, + 0x0032, 0x0068, 0x0064, 0x0061, 0x0062, 0x006f, 0x0070, 0x0064, + 0x0064, 0x0064, 0x0069, 0x5e73, 0x662d, 0x5927, 0x660e, 0x682a, +}; + +static const unsigned short gNormalizeTable3380[] = { + /* U+3380 */ + 0x0070, 0x006e, 0x03bc, 0x006d, 0x006b, 0x006b, 0x006d, 0x0067, + 0x0063, 0x006b, 0x0070, 0x006e, 0x03bc, 0x03bc, 0x006d, 0x006b, + 0x0068, 0x006b, 0x006d, 0x0067, 0x0074, 0x03bc, 0x006d, 0x0064, + 0x006b, 0x0066, 0x006e, 0x03bc, 0x006d, 0x0063, 0x006b, 0x006d, + 0x0063, 0x006d, 0x006b, 0x006d, 0x0063, 0x006d, 0x006b, 0x006d, + 0x006d, 0x0070, 0x006b, 0x006d, 0x0067, 0x0072, 0x0072, 0x0072, + 0x0070, 0x006e, 0x03bc, 0x006d, 0x0070, 0x006e, 0x03bc, 0x006d, + 0x006b, 0x006d, 0x0070, 0x006e, 0x03bc, 0x006d, 0x006b, 0x006d, +}; + +static const unsigned short gNormalizeTable33c0[] = { + /* U+33c0 */ + 0x006b, 0x006d, 0x0061, 0x0062, 0x0063, 0x0063, 0x0063, 0x0063, + 0x0064, 0x0067, 0x0068, 0x0068, 0x0069, 0x006b, 0x006b, 0x006b, + 0x006c, 0x006c, 0x006c, 0x006c, 0x006d, 0x006d, 0x006d, 0x0070, + 0x0070, 0x0070, 0x0070, 0x0073, 0x0073, 0x0077, 0x0076, 0x0061, + 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, + 0x0039, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, 0x0031, + 0x0031, 0x0031, 0x0031, 0x0032, 0x0032, 0x0032, 0x0032, 0x0032, + 0x0032, 0x0032, 0x0032, 0x0032, 0x0032, 0x0033, 0x0033, 0x0067, +}; + +static const unsigned short gNormalizeTablea640[] = { + /* U+a640 */ + 0xa641, 0xa641, 0xa643, 0xa643, 0xa645, 0xa645, 0xa647, 0xa647, + 0xa649, 0xa649, 0xa64b, 0xa64b, 0xa64d, 0xa64d, 0xa64f, 0xa64f, + 0xa651, 0xa651, 0xa653, 0xa653, 0xa655, 0xa655, 0xa657, 0xa657, + 0xa659, 0xa659, 0xa65b, 0xa65b, 0xa65d, 0xa65d, 0xa65f, 0xa65f, + 0xa660, 0xa661, 0xa663, 0xa663, 0xa665, 0xa665, 0xa667, 0xa667, + 0xa669, 0xa669, 0xa66b, 0xa66b, 0xa66d, 0xa66d, 0xa66e, 0xa66f, + 0xa670, 0xa671, 0xa672, 0xa673, 0xa674, 0xa675, 0xa676, 0xa677, + 0xa678, 0xa679, 0xa67a, 0xa67b, 0xa67c, 0xa67d, 0xa67e, 0xa67f, +}; + +static const unsigned short gNormalizeTablea680[] = { + /* U+a680 */ + 0xa681, 0xa681, 0xa683, 0xa683, 0xa685, 0xa685, 0xa687, 0xa687, + 0xa689, 0xa689, 0xa68b, 0xa68b, 0xa68d, 0xa68d, 0xa68f, 0xa68f, + 0xa691, 0xa691, 0xa693, 0xa693, 0xa695, 0xa695, 0xa697, 0xa697, + 0xa698, 0xa699, 0xa69a, 0xa69b, 0xa69c, 0xa69d, 0xa69e, 0xa69f, + 0xa6a0, 0xa6a1, 0xa6a2, 0xa6a3, 0xa6a4, 0xa6a5, 0xa6a6, 0xa6a7, + 0xa6a8, 0xa6a9, 0xa6aa, 0xa6ab, 0xa6ac, 0xa6ad, 0xa6ae, 0xa6af, + 0xa6b0, 0xa6b1, 0xa6b2, 0xa6b3, 0xa6b4, 0xa6b5, 0xa6b6, 0xa6b7, + 0xa6b8, 0xa6b9, 0xa6ba, 0xa6bb, 0xa6bc, 0xa6bd, 0xa6be, 0xa6bf, +}; + +static const unsigned short gNormalizeTablea700[] = { + /* U+a700 */ + 0xa700, 0xa701, 0xa702, 0xa703, 0xa704, 0xa705, 0xa706, 0xa707, + 0xa708, 0xa709, 0xa70a, 0xa70b, 0xa70c, 0xa70d, 0xa70e, 0xa70f, + 0xa710, 0xa711, 0xa712, 0xa713, 0xa714, 0xa715, 0xa716, 0xa717, + 0xa718, 0xa719, 0xa71a, 0xa71b, 0xa71c, 0xa71d, 0xa71e, 0xa71f, + 0xa720, 0xa721, 0xa723, 0xa723, 0xa725, 0xa725, 0xa727, 0xa727, + 0xa729, 0xa729, 0xa72b, 0xa72b, 0xa72d, 0xa72d, 0xa72f, 0xa72f, + 0xa730, 0xa731, 0xa733, 0xa733, 0xa735, 0xa735, 0xa737, 0xa737, + 0xa739, 0xa739, 0xa73b, 0xa73b, 0xa73d, 0xa73d, 0xa73f, 0xa73f, +}; + +static const unsigned short gNormalizeTablea740[] = { + /* U+a740 */ + 0xa741, 0xa741, 0xa743, 0xa743, 0xa745, 0xa745, 0xa747, 0xa747, + 0xa749, 0xa749, 0xa74b, 0xa74b, 0xa74d, 0xa74d, 0xa74f, 0xa74f, + 0xa751, 0xa751, 0xa753, 0xa753, 0xa755, 0xa755, 0xa757, 0xa757, + 0xa759, 0xa759, 0xa75b, 0xa75b, 0xa75d, 0xa75d, 0xa75f, 0xa75f, + 0xa761, 0xa761, 0xa763, 0xa763, 0xa765, 0xa765, 0xa767, 0xa767, + 0xa769, 0xa769, 0xa76b, 0xa76b, 0xa76d, 0xa76d, 0xa76f, 0xa76f, + 0xa76f, 0xa771, 0xa772, 0xa773, 0xa774, 0xa775, 0xa776, 0xa777, + 0xa778, 0xa77a, 0xa77a, 0xa77c, 0xa77c, 0x1d79, 0xa77f, 0xa77f, +}; + +static const unsigned short gNormalizeTablea780[] = { + /* U+a780 */ + 0xa781, 0xa781, 0xa783, 0xa783, 0xa785, 0xa785, 0xa787, 0xa787, + 0xa788, 0xa789, 0xa78a, 0xa78c, 0xa78c, 0xa78d, 0xa78e, 0xa78f, + 0xa790, 0xa791, 0xa792, 0xa793, 0xa794, 0xa795, 0xa796, 0xa797, + 0xa798, 0xa799, 0xa79a, 0xa79b, 0xa79c, 0xa79d, 0xa79e, 0xa79f, + 0xa7a0, 0xa7a1, 0xa7a2, 0xa7a3, 0xa7a4, 0xa7a5, 0xa7a6, 0xa7a7, + 0xa7a8, 0xa7a9, 0xa7aa, 0xa7ab, 0xa7ac, 0xa7ad, 0xa7ae, 0xa7af, + 0xa7b0, 0xa7b1, 0xa7b2, 0xa7b3, 0xa7b4, 0xa7b5, 0xa7b6, 0xa7b7, + 0xa7b8, 0xa7b9, 0xa7ba, 0xa7bb, 0xa7bc, 0xa7bd, 0xa7be, 0xa7bf, +}; + +static const unsigned short gNormalizeTablef900[] = { + /* U+f900 */ + 0x8c48, 0x66f4, 0x8eca, 0x8cc8, 0x6ed1, 0x4e32, 0x53e5, 0x9f9c, + 0x9f9c, 0x5951, 0x91d1, 0x5587, 0x5948, 0x61f6, 0x7669, 0x7f85, + 0x863f, 0x87ba, 0x88f8, 0x908f, 0x6a02, 0x6d1b, 0x70d9, 0x73de, + 0x843d, 0x916a, 0x99f1, 0x4e82, 0x5375, 0x6b04, 0x721b, 0x862d, + 0x9e1e, 0x5d50, 0x6feb, 0x85cd, 0x8964, 0x62c9, 0x81d8, 0x881f, + 0x5eca, 0x6717, 0x6d6a, 0x72fc, 0x90ce, 0x4f86, 0x51b7, 0x52de, + 0x64c4, 0x6ad3, 0x7210, 0x76e7, 0x8001, 0x8606, 0x865c, 0x8def, + 0x9732, 0x9b6f, 0x9dfa, 0x788c, 0x797f, 0x7da0, 0x83c9, 0x9304, +}; + +static const unsigned short gNormalizeTablef940[] = { + /* U+f940 */ + 0x9e7f, 0x8ad6, 0x58df, 0x5f04, 0x7c60, 0x807e, 0x7262, 0x78ca, + 0x8cc2, 0x96f7, 0x58d8, 0x5c62, 0x6a13, 0x6dda, 0x6f0f, 0x7d2f, + 0x7e37, 0x964b, 0x52d2, 0x808b, 0x51dc, 0x51cc, 0x7a1c, 0x7dbe, + 0x83f1, 0x9675, 0x8b80, 0x62cf, 0x6a02, 0x8afe, 0x4e39, 0x5be7, + 0x6012, 0x7387, 0x7570, 0x5317, 0x78fb, 0x4fbf, 0x5fa9, 0x4e0d, + 0x6ccc, 0x6578, 0x7d22, 0x53c3, 0x585e, 0x7701, 0x8449, 0x8aaa, + 0x6bba, 0x8fb0, 0x6c88, 0x62fe, 0x82e5, 0x63a0, 0x7565, 0x4eae, + 0x5169, 0x51c9, 0x6881, 0x7ce7, 0x826f, 0x8ad2, 0x91cf, 0x52f5, +}; + +static const unsigned short gNormalizeTablef980[] = { + /* U+f980 */ + 0x5442, 0x5973, 0x5eec, 0x65c5, 0x6ffe, 0x792a, 0x95ad, 0x9a6a, + 0x9e97, 0x9ece, 0x529b, 0x66c6, 0x6b77, 0x8f62, 0x5e74, 0x6190, + 0x6200, 0x649a, 0x6f23, 0x7149, 0x7489, 0x79ca, 0x7df4, 0x806f, + 0x8f26, 0x84ee, 0x9023, 0x934a, 0x5217, 0x52a3, 0x54bd, 0x70c8, + 0x88c2, 0x8aaa, 0x5ec9, 0x5ff5, 0x637b, 0x6bae, 0x7c3e, 0x7375, + 0x4ee4, 0x56f9, 0x5be7, 0x5dba, 0x601c, 0x73b2, 0x7469, 0x7f9a, + 0x8046, 0x9234, 0x96f6, 0x9748, 0x9818, 0x4f8b, 0x79ae, 0x91b4, + 0x96b8, 0x60e1, 0x4e86, 0x50da, 0x5bee, 0x5c3f, 0x6599, 0x6a02, +}; + +static const unsigned short gNormalizeTablef9c0[] = { + /* U+f9c0 */ + 0x71ce, 0x7642, 0x84fc, 0x907c, 0x9f8d, 0x6688, 0x962e, 0x5289, + 0x677b, 0x67f3, 0x6d41, 0x6e9c, 0x7409, 0x7559, 0x786b, 0x7d10, + 0x985e, 0x516d, 0x622e, 0x9678, 0x502b, 0x5d19, 0x6dea, 0x8f2a, + 0x5f8b, 0x6144, 0x6817, 0x7387, 0x9686, 0x5229, 0x540f, 0x5c65, + 0x6613, 0x674e, 0x68a8, 0x6ce5, 0x7406, 0x75e2, 0x7f79, 0x88cf, + 0x88e1, 0x91cc, 0x96e2, 0x533f, 0x6eba, 0x541d, 0x71d0, 0x7498, + 0x85fa, 0x96a3, 0x9c57, 0x9e9f, 0x6797, 0x6dcb, 0x81e8, 0x7acb, + 0x7b20, 0x7c92, 0x72c0, 0x7099, 0x8b58, 0x4ec0, 0x8336, 0x523a, +}; + +static const unsigned short gNormalizeTablefa00[] = { + /* U+fa00 */ + 0x5207, 0x5ea6, 0x62d3, 0x7cd6, 0x5b85, 0x6d1e, 0x66b4, 0x8f3b, + 0x884c, 0x964d, 0x898b, 0x5ed3, 0x5140, 0x55c0, 0xfa0e, 0xfa0f, + 0x585a, 0xfa11, 0x6674, 0xfa13, 0xfa14, 0x51de, 0x732a, 0x76ca, + 0x793c, 0x795e, 0x7965, 0x798f, 0x9756, 0x7cbe, 0x7fbd, 0xfa1f, + 0x8612, 0xfa21, 0x8af8, 0xfa23, 0xfa24, 0x9038, 0x90fd, 0xfa27, + 0xfa28, 0xfa29, 0x98ef, 0x98fc, 0x9928, 0x9db4, 0xfa2e, 0xfa2f, + 0x4fae, 0x50e7, 0x514d, 0x52c9, 0x52e4, 0x5351, 0x559d, 0x5606, + 0x5668, 0x5840, 0x58a8, 0x5c64, 0x5c6e, 0x6094, 0x6168, 0x618e, +}; + +static const unsigned short gNormalizeTablefa40[] = { + /* U+fa40 */ + 0x61f2, 0x654f, 0x65e2, 0x6691, 0x6885, 0x6d77, 0x6e1a, 0x6f22, + 0x716e, 0x722b, 0x7422, 0x7891, 0x793e, 0x7949, 0x7948, 0x7950, + 0x7956, 0x795d, 0x798d, 0x798e, 0x7a40, 0x7a81, 0x7bc0, 0x7df4, + 0x7e09, 0x7e41, 0x7f72, 0x8005, 0x81ed, 0x8279, 0x8279, 0x8457, + 0x8910, 0x8996, 0x8b01, 0x8b39, 0x8cd3, 0x8d08, 0x8fb6, 0x9038, + 0x96e3, 0x97ff, 0x983b, 0x6075, 0xfa6c, 0x8218, 0xfa6e, 0xfa6f, + 0x4e26, 0x51b5, 0x5168, 0x4f80, 0x5145, 0x5180, 0x52c7, 0x52fa, + 0x559d, 0x5555, 0x5599, 0x55e2, 0x585a, 0x58b3, 0x5944, 0x5954, +}; + +static const unsigned short gNormalizeTablefa80[] = { + /* U+fa80 */ + 0x5a62, 0x5b28, 0x5ed2, 0x5ed9, 0x5f69, 0x5fad, 0x60d8, 0x614e, + 0x6108, 0x618e, 0x6160, 0x61f2, 0x6234, 0x63c4, 0x641c, 0x6452, + 0x6556, 0x6674, 0x6717, 0x671b, 0x6756, 0x6b79, 0x6bba, 0x6d41, + 0x6edb, 0x6ecb, 0x6f22, 0x701e, 0x716e, 0x77a7, 0x7235, 0x72af, + 0x732a, 0x7471, 0x7506, 0x753b, 0x761d, 0x761f, 0x76ca, 0x76db, + 0x76f4, 0x774a, 0x7740, 0x78cc, 0x7ab1, 0x7bc0, 0x7c7b, 0x7d5b, + 0x7df4, 0x7f3e, 0x8005, 0x8352, 0x83ef, 0x8779, 0x8941, 0x8986, + 0x8996, 0x8abf, 0x8af8, 0x8acb, 0x8b01, 0x8afe, 0x8aed, 0x8b39, +}; + +static const unsigned short gNormalizeTablefac0[] = { + /* U+fac0 */ + 0x8b8a, 0x8d08, 0x8f38, 0x9072, 0x9199, 0x9276, 0x967c, 0x96e3, + 0x9756, 0x97db, 0x97ff, 0x980b, 0x983b, 0x9b12, 0x9f9c, 0xfacf, + 0xfad0, 0xfad1, 0x3b9d, 0x4018, 0x4039, 0xfad5, 0xfad6, 0xfad7, + 0x9f43, 0x9f8e, 0xfada, 0xfadb, 0xfadc, 0xfadd, 0xfade, 0xfadf, + 0xfae0, 0xfae1, 0xfae2, 0xfae3, 0xfae4, 0xfae5, 0xfae6, 0xfae7, + 0xfae8, 0xfae9, 0xfaea, 0xfaeb, 0xfaec, 0xfaed, 0xfaee, 0xfaef, + 0xfaf0, 0xfaf1, 0xfaf2, 0xfaf3, 0xfaf4, 0xfaf5, 0xfaf6, 0xfaf7, + 0xfaf8, 0xfaf9, 0xfafa, 0xfafb, 0xfafc, 0xfafd, 0xfafe, 0xfaff, +}; + +static const unsigned short gNormalizeTablefb00[] = { + /* U+fb00 */ + 0x0066, 0x0066, 0x0066, 0x0066, 0x0066, 0x0073, 0x0073, 0xfb07, + 0xfb08, 0xfb09, 0xfb0a, 0xfb0b, 0xfb0c, 0xfb0d, 0xfb0e, 0xfb0f, + 0xfb10, 0xfb11, 0xfb12, 0x0574, 0x0574, 0x0574, 0x057e, 0x0574, + 0xfb18, 0xfb19, 0xfb1a, 0xfb1b, 0xfb1c, 0x05d9, 0xfb1e, 0x05f2, + 0x05e2, 0x05d0, 0x05d3, 0x05d4, 0x05db, 0x05dc, 0x05dd, 0x05e8, + 0x05ea, 0x002b, 0x05e9, 0x05e9, 0x05e9, 0x05e9, 0x05d0, 0x05d0, + 0x05d0, 0x05d1, 0x05d2, 0x05d3, 0x05d4, 0x05d5, 0x05d6, 0xfb37, + 0x05d8, 0x05d9, 0x05da, 0x05db, 0x05dc, 0xfb3d, 0x05de, 0xfb3f, +}; + +static const unsigned short gNormalizeTablefb40[] = { + /* U+fb40 */ + 0x05e0, 0x05e1, 0xfb42, 0x05e3, 0x05e4, 0xfb45, 0x05e6, 0x05e7, + 0x05e8, 0x05e9, 0x05ea, 0x05d5, 0x05d1, 0x05db, 0x05e4, 0x05d0, + 0x0671, 0x0671, 0x067b, 0x067b, 0x067b, 0x067b, 0x067e, 0x067e, + 0x067e, 0x067e, 0x0680, 0x0680, 0x0680, 0x0680, 0x067a, 0x067a, + 0x067a, 0x067a, 0x067f, 0x067f, 0x067f, 0x067f, 0x0679, 0x0679, + 0x0679, 0x0679, 0x06a4, 0x06a4, 0x06a4, 0x06a4, 0x06a6, 0x06a6, + 0x06a6, 0x06a6, 0x0684, 0x0684, 0x0684, 0x0684, 0x0683, 0x0683, + 0x0683, 0x0683, 0x0686, 0x0686, 0x0686, 0x0686, 0x0687, 0x0687, +}; + +static const unsigned short gNormalizeTablefb80[] = { + /* U+fb80 */ + 0x0687, 0x0687, 0x068d, 0x068d, 0x068c, 0x068c, 0x068e, 0x068e, + 0x0688, 0x0688, 0x0698, 0x0698, 0x0691, 0x0691, 0x06a9, 0x06a9, + 0x06a9, 0x06a9, 0x06af, 0x06af, 0x06af, 0x06af, 0x06b3, 0x06b3, + 0x06b3, 0x06b3, 0x06b1, 0x06b1, 0x06b1, 0x06b1, 0x06ba, 0x06ba, + 0x06bb, 0x06bb, 0x06bb, 0x06bb, 0x06d5, 0x06d5, 0x06c1, 0x06c1, + 0x06c1, 0x06c1, 0x06be, 0x06be, 0x06be, 0x06be, 0x06d2, 0x06d2, + 0x06d2, 0x06d2, 0xfbb2, 0xfbb3, 0xfbb4, 0xfbb5, 0xfbb6, 0xfbb7, + 0xfbb8, 0xfbb9, 0xfbba, 0xfbbb, 0xfbbc, 0xfbbd, 0xfbbe, 0xfbbf, +}; + +static const unsigned short gNormalizeTablefbc0[] = { + /* U+fbc0 */ + 0xfbc0, 0xfbc1, 0xfbc2, 0xfbc3, 0xfbc4, 0xfbc5, 0xfbc6, 0xfbc7, + 0xfbc8, 0xfbc9, 0xfbca, 0xfbcb, 0xfbcc, 0xfbcd, 0xfbce, 0xfbcf, + 0xfbd0, 0xfbd1, 0xfbd2, 0x06ad, 0x06ad, 0x06ad, 0x06ad, 0x06c7, + 0x06c7, 0x06c6, 0x06c6, 0x06c8, 0x06c8, 0x06c7, 0x06cb, 0x06cb, + 0x06c5, 0x06c5, 0x06c9, 0x06c9, 0x06d0, 0x06d0, 0x06d0, 0x06d0, + 0x0649, 0x0649, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, + 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, + 0x064a, 0x064a, 0x064a, 0x064a, 0x06cc, 0x06cc, 0x06cc, 0x06cc, +}; + +static const unsigned short gNormalizeTablefc00[] = { + /* U+fc00 */ + 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x0628, 0x0628, 0x0628, + 0x0628, 0x0628, 0x0628, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, + 0x062a, 0x062b, 0x062b, 0x062b, 0x062b, 0x062c, 0x062c, 0x062d, + 0x062d, 0x062e, 0x062e, 0x062e, 0x0633, 0x0633, 0x0633, 0x0633, + 0x0635, 0x0635, 0x0636, 0x0636, 0x0636, 0x0636, 0x0637, 0x0637, + 0x0638, 0x0639, 0x0639, 0x063a, 0x063a, 0x0641, 0x0641, 0x0641, + 0x0641, 0x0641, 0x0641, 0x0642, 0x0642, 0x0642, 0x0642, 0x0643, + 0x0643, 0x0643, 0x0643, 0x0643, 0x0643, 0x0643, 0x0643, 0x0644, +}; + +static const unsigned short gNormalizeTablefc40[] = { + /* U+fc40 */ + 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0x0645, 0x0645, 0x0645, + 0x0645, 0x0645, 0x0645, 0x0646, 0x0646, 0x0646, 0x0646, 0x0646, + 0x0646, 0x0647, 0x0647, 0x0647, 0x0647, 0x064a, 0x064a, 0x064a, + 0x064a, 0x064a, 0x064a, 0x0630, 0x0631, 0x0649, 0x0020, 0x0020, + 0x0020, 0x0020, 0x0020, 0x0020, 0x064a, 0x064a, 0x064a, 0x064a, + 0x064a, 0x064a, 0x0628, 0x0628, 0x0628, 0x0628, 0x0628, 0x0628, + 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062b, 0x062b, + 0x062b, 0x062b, 0x062b, 0x062b, 0x0641, 0x0641, 0x0642, 0x0642, +}; + +static const unsigned short gNormalizeTablefc80[] = { + /* U+fc80 */ + 0x0643, 0x0643, 0x0643, 0x0643, 0x0643, 0x0644, 0x0644, 0x0644, + 0x0645, 0x0645, 0x0646, 0x0646, 0x0646, 0x0646, 0x0646, 0x0646, + 0x0649, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, + 0x064a, 0x064a, 0x064a, 0x064a, 0x0628, 0x0628, 0x0628, 0x0628, + 0x0628, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062b, 0x062c, + 0x062c, 0x062d, 0x062d, 0x062e, 0x062e, 0x0633, 0x0633, 0x0633, + 0x0633, 0x0635, 0x0635, 0x0635, 0x0636, 0x0636, 0x0636, 0x0636, + 0x0637, 0x0638, 0x0639, 0x0639, 0x063a, 0x063a, 0x0641, 0x0641, +}; + +static const unsigned short gNormalizeTablefcc0[] = { + /* U+fcc0 */ + 0x0641, 0x0641, 0x0642, 0x0642, 0x0643, 0x0643, 0x0643, 0x0643, + 0x0643, 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0x0645, 0x0645, + 0x0645, 0x0645, 0x0646, 0x0646, 0x0646, 0x0646, 0x0646, 0x0647, + 0x0647, 0x0647, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, 0x064a, + 0x064a, 0x0628, 0x0628, 0x062a, 0x062a, 0x062b, 0x062b, 0x0633, + 0x0633, 0x0634, 0x0634, 0x0643, 0x0643, 0x0644, 0x0646, 0x0646, + 0x064a, 0x064a, 0x0640, 0x0640, 0x0640, 0x0637, 0x0637, 0x0639, + 0x0639, 0x063a, 0x063a, 0x0633, 0x0633, 0x0634, 0x0634, 0x062d, +}; + +static const unsigned short gNormalizeTablefd00[] = { + /* U+fd00 */ + 0x062d, 0x062c, 0x062c, 0x062e, 0x062e, 0x0635, 0x0635, 0x0636, + 0x0636, 0x0634, 0x0634, 0x0634, 0x0634, 0x0634, 0x0633, 0x0635, + 0x0636, 0x0637, 0x0637, 0x0639, 0x0639, 0x063a, 0x063a, 0x0633, + 0x0633, 0x0634, 0x0634, 0x062d, 0x062d, 0x062c, 0x062c, 0x062e, + 0x062e, 0x0635, 0x0635, 0x0636, 0x0636, 0x0634, 0x0634, 0x0634, + 0x0634, 0x0634, 0x0633, 0x0635, 0x0636, 0x0634, 0x0634, 0x0634, + 0x0634, 0x0633, 0x0634, 0x0637, 0x0633, 0x0633, 0x0633, 0x0634, + 0x0634, 0x0634, 0x0637, 0x0638, 0x0627, 0x0627, 0xfd3e, 0xfd3f, +}; + +static const unsigned short gNormalizeTablefd40[] = { + /* U+fd40 */ + 0xfd40, 0xfd41, 0xfd42, 0xfd43, 0xfd44, 0xfd45, 0xfd46, 0xfd47, + 0xfd48, 0xfd49, 0xfd4a, 0xfd4b, 0xfd4c, 0xfd4d, 0xfd4e, 0xfd4f, + 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, + 0x062c, 0x062c, 0x062d, 0x062d, 0x0633, 0x0633, 0x0633, 0x0633, + 0x0633, 0x0633, 0x0633, 0x0633, 0x0635, 0x0635, 0x0635, 0x0634, + 0x0634, 0x0634, 0x0634, 0x0634, 0x0634, 0x0634, 0x0636, 0x0636, + 0x0636, 0x0637, 0x0637, 0x0637, 0x0637, 0x0639, 0x0639, 0x0639, + 0x0639, 0x063a, 0x063a, 0x063a, 0x0641, 0x0641, 0x0642, 0x0642, +}; + +static const unsigned short gNormalizeTablefd80[] = { + /* U+fd80 */ + 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, + 0x0644, 0x0645, 0x0645, 0x0645, 0x0645, 0x0645, 0x0645, 0x0645, + 0xfd90, 0xfd91, 0x0645, 0x0647, 0x0647, 0x0646, 0x0646, 0x0646, + 0x0646, 0x0646, 0x0646, 0x0646, 0x064a, 0x064a, 0x0628, 0x062a, + 0x062a, 0x062a, 0x062a, 0x062a, 0x062a, 0x062c, 0x062c, 0x062c, + 0x0633, 0x0635, 0x0634, 0x0636, 0x0644, 0x0644, 0x064a, 0x064a, + 0x064a, 0x0645, 0x0642, 0x0646, 0x0642, 0x0644, 0x0639, 0x0643, + 0x0646, 0x0645, 0x0644, 0x0643, 0x0644, 0x0646, 0x062c, 0x062d, +}; + +static const unsigned short gNormalizeTablefdc0[] = { + /* U+fdc0 */ + 0x0645, 0x0641, 0x0628, 0x0643, 0x0639, 0x0635, 0x0633, 0x0646, + 0xfdc8, 0xfdc9, 0xfdca, 0xfdcb, 0xfdcc, 0xfdcd, 0xfdce, 0xfdcf, + 0xfdd0, 0xfdd1, 0xfdd2, 0xfdd3, 0xfdd4, 0xfdd5, 0xfdd6, 0xfdd7, + 0xfdd8, 0xfdd9, 0xfdda, 0xfddb, 0xfddc, 0xfddd, 0xfdde, 0xfddf, + 0xfde0, 0xfde1, 0xfde2, 0xfde3, 0xfde4, 0xfde5, 0xfde6, 0xfde7, + 0xfde8, 0xfde9, 0xfdea, 0xfdeb, 0xfdec, 0xfded, 0xfdee, 0xfdef, + 0x0635, 0x0642, 0x0627, 0x0627, 0x0645, 0x0635, 0x0631, 0x0639, + 0x0648, 0x0635, 0x0635, 0x062c, 0x0631, 0xfdfd, 0xfdfe, 0xfdff, +}; + +static const unsigned short gNormalizeTablefe00[] = { + /* U+fe00 */ + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x002c, 0x3001, 0x3002, 0x003a, 0x003b, 0x0021, 0x003f, 0x3016, + 0x3017, 0x002e, 0xfe1a, 0xfe1b, 0xfe1c, 0xfe1d, 0xfe1e, 0xfe1f, + 0xfe20, 0xfe21, 0xfe22, 0xfe23, 0xfe24, 0xfe25, 0xfe26, 0xfe27, + 0xfe28, 0xfe29, 0xfe2a, 0xfe2b, 0xfe2c, 0xfe2d, 0xfe2e, 0xfe2f, + 0x002e, 0x2014, 0x2013, 0x005f, 0x005f, 0x0028, 0x0029, 0x007b, + 0x007d, 0x3014, 0x3015, 0x3010, 0x3011, 0x300a, 0x300b, 0x3008, +}; + +static const unsigned short gNormalizeTablefe40[] = { + /* U+fe40 */ + 0x3009, 0x300c, 0x300d, 0x300e, 0x300f, 0xfe45, 0xfe46, 0x005b, + 0x005d, 0x0020, 0x0020, 0x0020, 0x0020, 0x005f, 0x005f, 0x005f, + 0x002c, 0x3001, 0x002e, 0xfe53, 0x003b, 0x003a, 0x003f, 0x0021, + 0x2014, 0x0028, 0x0029, 0x007b, 0x007d, 0x3014, 0x3015, 0x0023, + 0x0026, 0x002a, 0x002b, 0x002d, 0x003c, 0x003e, 0x003d, 0xfe67, + 0x005c, 0x0024, 0x0025, 0x0040, 0xfe6c, 0xfe6d, 0xfe6e, 0xfe6f, + 0x0020, 0x0640, 0x0020, 0xfe73, 0x0020, 0xfe75, 0x0020, 0x0640, + 0x0020, 0x0640, 0x0020, 0x0640, 0x0020, 0x0640, 0x0020, 0x0640, +}; + +static const unsigned short gNormalizeTablefe80[] = { + /* U+fe80 */ + 0x0621, 0x0627, 0x0627, 0x0627, 0x0627, 0x0648, 0x0648, 0x0627, + 0x0627, 0x064a, 0x064a, 0x064a, 0x064a, 0x0627, 0x0627, 0x0628, + 0x0628, 0x0628, 0x0628, 0x0629, 0x0629, 0x062a, 0x062a, 0x062a, + 0x062a, 0x062b, 0x062b, 0x062b, 0x062b, 0x062c, 0x062c, 0x062c, + 0x062c, 0x062d, 0x062d, 0x062d, 0x062d, 0x062e, 0x062e, 0x062e, + 0x062e, 0x062f, 0x062f, 0x0630, 0x0630, 0x0631, 0x0631, 0x0632, + 0x0632, 0x0633, 0x0633, 0x0633, 0x0633, 0x0634, 0x0634, 0x0634, + 0x0634, 0x0635, 0x0635, 0x0635, 0x0635, 0x0636, 0x0636, 0x0636, +}; + +static const unsigned short gNormalizeTablefec0[] = { + /* U+fec0 */ + 0x0636, 0x0637, 0x0637, 0x0637, 0x0637, 0x0638, 0x0638, 0x0638, + 0x0638, 0x0639, 0x0639, 0x0639, 0x0639, 0x063a, 0x063a, 0x063a, + 0x063a, 0x0641, 0x0641, 0x0641, 0x0641, 0x0642, 0x0642, 0x0642, + 0x0642, 0x0643, 0x0643, 0x0643, 0x0643, 0x0644, 0x0644, 0x0644, + 0x0644, 0x0645, 0x0645, 0x0645, 0x0645, 0x0646, 0x0646, 0x0646, + 0x0646, 0x0647, 0x0647, 0x0647, 0x0647, 0x0648, 0x0648, 0x0649, + 0x0649, 0x064a, 0x064a, 0x064a, 0x064a, 0x0644, 0x0644, 0x0644, + 0x0644, 0x0644, 0x0644, 0x0644, 0x0644, 0xfefd, 0xfefe, 0x0020, +}; + +static const unsigned short gNormalizeTableff00[] = { + /* U+ff00 */ + 0xff00, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, +}; + +static const unsigned short gNormalizeTableff40[] = { + /* U+ff40 */ + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x2985, + 0x2986, 0x3002, 0x300c, 0x300d, 0x3001, 0x30fb, 0x30f2, 0x30a1, + 0x30a3, 0x30a5, 0x30a7, 0x30a9, 0x30e3, 0x30e5, 0x30e7, 0x30c3, + 0x30fc, 0x30a2, 0x30a4, 0x30a6, 0x30a8, 0x30aa, 0x30ab, 0x30ad, + 0x30af, 0x30b1, 0x30b3, 0x30b5, 0x30b7, 0x30b9, 0x30bb, 0x30bd, +}; + +static const unsigned short gNormalizeTableff80[] = { + /* U+ff80 */ + 0x30bf, 0x30c1, 0x30c4, 0x30c6, 0x30c8, 0x30ca, 0x30cb, 0x30cc, + 0x30cd, 0x30ce, 0x30cf, 0x30d2, 0x30d5, 0x30d8, 0x30db, 0x30de, + 0x30df, 0x30e0, 0x30e1, 0x30e2, 0x30e4, 0x30e6, 0x30e8, 0x30e9, + 0x30ea, 0x30eb, 0x30ec, 0x30ed, 0x30ef, 0x30f3, 0x3099, 0x309a, + 0x0020, 0x1100, 0x1101, 0x11aa, 0x1102, 0x11ac, 0x11ad, 0x1103, + 0x1104, 0x1105, 0x11b0, 0x11b1, 0x11b2, 0x11b3, 0x11b4, 0x11b5, + 0x111a, 0x1106, 0x1107, 0x1108, 0x1121, 0x1109, 0x110a, 0x110b, + 0x110c, 0x110d, 0x110e, 0x110f, 0x1110, 0x1111, 0x1112, 0xffbf, +}; + +static const unsigned short gNormalizeTableffc0[] = { + /* U+ffc0 */ + 0xffc0, 0xffc1, 0x1161, 0x1162, 0x1163, 0x1164, 0x1165, 0x1166, + 0xffc8, 0xffc9, 0x1167, 0x1168, 0x1169, 0x116a, 0x116b, 0x116c, + 0xffd0, 0xffd1, 0x116d, 0x116e, 0x116f, 0x1170, 0x1171, 0x1172, + 0xffd8, 0xffd9, 0x1173, 0x1174, 0x1175, 0xffdd, 0xffde, 0xffdf, + 0x00a2, 0x00a3, 0x00ac, 0x0020, 0x00a6, 0x00a5, 0x20a9, 0xffe7, + 0x2502, 0x2190, 0x2191, 0x2192, 0x2193, 0x25a0, 0x25cb, 0xffef, + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x0020, 0xfff9, 0xfffa, 0xfffb, 0xfffc, 0xfffd, 0xfffe, 0xffff, +}; + +static const unsigned short* gNormalizeTable[] = { + 0, + gNormalizeTable0040, + gNormalizeTable0080, + gNormalizeTable00c0, + gNormalizeTable0100, + gNormalizeTable0140, + gNormalizeTable0180, + gNormalizeTable01c0, + gNormalizeTable0200, + gNormalizeTable0240, + gNormalizeTable0280, + gNormalizeTable02c0, + 0, + gNormalizeTable0340, + gNormalizeTable0380, + gNormalizeTable03c0, + gNormalizeTable0400, + gNormalizeTable0440, + gNormalizeTable0480, + gNormalizeTable04c0, + gNormalizeTable0500, + gNormalizeTable0540, + gNormalizeTable0580, + 0, + gNormalizeTable0600, + gNormalizeTable0640, + 0, + gNormalizeTable06c0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + gNormalizeTable0900, + gNormalizeTable0940, + 0, + gNormalizeTable09c0, + gNormalizeTable0a00, + gNormalizeTable0a40, + 0, + 0, + 0, + gNormalizeTable0b40, + gNormalizeTable0b80, + gNormalizeTable0bc0, + 0, + gNormalizeTable0c40, + 0, + gNormalizeTable0cc0, + 0, + gNormalizeTable0d40, + 0, + gNormalizeTable0dc0, + gNormalizeTable0e00, + 0, + gNormalizeTable0e80, + gNormalizeTable0ec0, + gNormalizeTable0f00, + gNormalizeTable0f40, + gNormalizeTable0f80, + 0, + gNormalizeTable1000, + 0, + gNormalizeTable1080, + gNormalizeTable10c0, + 0, + gNormalizeTable1140, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + gNormalizeTable1780, + 0, + gNormalizeTable1800, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + gNormalizeTable1b00, + gNormalizeTable1b40, + 0, + 0, + 0, + 0, + 0, + 0, + gNormalizeTable1d00, + gNormalizeTable1d40, + gNormalizeTable1d80, + 0, + gNormalizeTable1e00, + gNormalizeTable1e40, + gNormalizeTable1e80, + gNormalizeTable1ec0, + gNormalizeTable1f00, + gNormalizeTable1f40, + gNormalizeTable1f80, + gNormalizeTable1fc0, + gNormalizeTable2000, + gNormalizeTable2040, + gNormalizeTable2080, + 0, + gNormalizeTable2100, + gNormalizeTable2140, + gNormalizeTable2180, + gNormalizeTable21c0, + gNormalizeTable2200, + gNormalizeTable2240, + gNormalizeTable2280, + gNormalizeTable22c0, + gNormalizeTable2300, + 0, + 0, + 0, + 0, + gNormalizeTable2440, + gNormalizeTable2480, + gNormalizeTable24c0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + gNormalizeTable2a00, + gNormalizeTable2a40, + 0, + gNormalizeTable2ac0, + 0, + 0, + 0, + 0, + gNormalizeTable2c00, + gNormalizeTable2c40, + gNormalizeTable2c80, + gNormalizeTable2cc0, + 0, + gNormalizeTable2d40, + 0, + 0, + 0, + 0, + gNormalizeTable2e80, + gNormalizeTable2ec0, + gNormalizeTable2f00, + gNormalizeTable2f40, + gNormalizeTable2f80, + gNormalizeTable2fc0, + gNormalizeTable3000, + gNormalizeTable3040, + gNormalizeTable3080, + gNormalizeTable30c0, + gNormalizeTable3100, + gNormalizeTable3140, + gNormalizeTable3180, + 0, + gNormalizeTable3200, + gNormalizeTable3240, + gNormalizeTable3280, + gNormalizeTable32c0, + gNormalizeTable3300, + gNormalizeTable3340, + gNormalizeTable3380, + gNormalizeTable33c0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + gNormalizeTablea640, + gNormalizeTablea680, + 0, + gNormalizeTablea700, + gNormalizeTablea740, + gNormalizeTablea780, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + gNormalizeTablef900, + gNormalizeTablef940, + gNormalizeTablef980, + gNormalizeTablef9c0, + gNormalizeTablefa00, + gNormalizeTablefa40, + gNormalizeTablefa80, + gNormalizeTablefac0, + gNormalizeTablefb00, + gNormalizeTablefb40, + gNormalizeTablefb80, + gNormalizeTablefbc0, + gNormalizeTablefc00, + gNormalizeTablefc40, + gNormalizeTablefc80, + gNormalizeTablefcc0, + gNormalizeTablefd00, + gNormalizeTablefd40, + gNormalizeTablefd80, + gNormalizeTablefdc0, + gNormalizeTablefe00, + gNormalizeTablefe40, + gNormalizeTablefe80, + gNormalizeTablefec0, + gNormalizeTableff00, + gNormalizeTableff40, + gNormalizeTableff80, + gNormalizeTableffc0, +}; + +unsigned int normalize_character(const unsigned int c) { + if (c >= 0x10000 || !gNormalizeTable[c >> 6]) return c; return gNormalizeTable[c >> 6][c & 0x3f]; } - diff --git a/mailnews/extensions/fts3/src/fts3_porter.c b/mailnews/extensions/fts3/src/fts3_porter.c index 8727612474..a78c0ed078 100644 --- a/mailnews/extensions/fts3/src/fts3_porter.c +++ b/mailnews/extensions/fts3/src/fts3_porter.c @@ -55,30 +55,27 @@ */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +# include +# include +# include +# include +# include -#include -#include -#include -#include -#include - -#include "fts3_tokenizer.h" +# include "fts3_tokenizer.h" /* need some defined to compile without sqlite3 code */ -#define sqlite3_malloc malloc -#define sqlite3_free free -#define sqlite3_realloc realloc +# define sqlite3_malloc malloc +# define sqlite3_free free +# define sqlite3_realloc realloc static const unsigned char sqlite3Utf8Trans1[] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, + 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, + 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x00, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, + 0x07, 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, }; typedef unsigned char u8; @@ -92,25 +89,24 @@ typedef unsigned char u8; * encoded byte. The same holds true at exit. * @param c The character to encode; this should be an unsigned int. */ -#define WRITE_UTF8(zOut, c) { \ - if( c<0x0080 ){ \ - *zOut++ = (u8)(c&0xff); \ - } \ - else if( c<0x0800 ){ \ - *zOut++ = 0xC0 + (u8)((c>>6) & 0x1F); \ - *zOut++ = 0x80 + (u8)(c & 0x3F); \ - } \ - else if( c<0x10000 ){ \ - *zOut++ = 0xE0 + (u8)((c>>12) & 0x0F); \ - *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ - *zOut++ = 0x80 + (u8)(c & 0x3F); \ - }else{ \ - *zOut++ = 0xf0 + (u8)((c>>18) & 0x07); \ - *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); \ - *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ - *zOut++ = 0x80 + (u8)(c & 0x3F); \ - } \ -} +# define WRITE_UTF8(zOut, c) \ + { \ + if (c < 0x0080) { \ + *zOut++ = (u8)(c & 0xff); \ + } else if (c < 0x0800) { \ + *zOut++ = 0xC0 + (u8)((c >> 6) & 0x1F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } else if (c < 0x10000) { \ + *zOut++ = 0xE0 + (u8)((c >> 12) & 0x0F); \ + *zOut++ = 0x80 + (u8)((c >> 6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } else { \ + *zOut++ = 0xf0 + (u8)((c >> 18) & 0x07); \ + *zOut++ = 0x80 + (u8)((c >> 12) & 0x3F); \ + *zOut++ = 0x80 + (u8)((c >> 6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ + } /** * Fudge factor to avoid buffer overwrites when WRITE_UTF8 is involved. @@ -125,7 +121,7 @@ typedef unsigned char u8; * potential growth for 2-byte to 4-byte growth. We can afford to do this * because we're not talking about a lot of memory here as a rule. */ -#define MAX_UTF8_GROWTH_FACTOR 2 +# define MAX_UTF8_GROWTH_FACTOR 2 /** * Helper from sqlite3.c to read a single UTF8 character. @@ -145,18 +141,20 @@ typedef unsigned char u8; * @param c The 'unsigned int' to hold the resulting character value. Do not * use a short or a char. */ -#define READ_UTF8(zIn, zTerm, c) { \ - c = *(zIn++); \ - if( c>=0xc0 ){ \ - c = sqlite3Utf8Trans1[c-0xc0]; \ - while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ - c = (c<<6) + (0x3f & *(zIn++)); \ - } \ - if( c<0x80 \ - || (c&0xFFFFF800)==0xD800 \ - || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ - } \ -} +# define READ_UTF8(zIn, zTerm, c) \ + { \ + c = *(zIn++); \ + if (c >= 0xc0) { \ + c = sqlite3Utf8Trans1[c - 0xc0]; \ + while (zIn != zTerm && (*zIn & 0xc0) == 0x80) { \ + c = (c << 6) + (0x3f & *(zIn++)); \ + } \ + if (c < 0x80 || (c & 0xFFFFF800) == 0xD800 || \ + (c & 0xFFFFFFFE) == 0xFFFE) { \ + c = 0xFFFD; \ + } \ + } \ + } /* end of compatible block to complie codes */ @@ -164,7 +162,7 @@ typedef unsigned char u8; ** Class derived from sqlite3_tokenizer */ typedef struct porter_tokenizer { - sqlite3_tokenizer base; /* Base class */ + sqlite3_tokenizer base; /* Base class */ } porter_tokenizer; /* @@ -172,12 +170,12 @@ typedef struct porter_tokenizer { */ typedef struct porter_tokenizer_cursor { sqlite3_tokenizer_cursor base; - const char *zInput; /* input we are tokenizing */ - int nInput; /* size of the input */ - int iOffset; /* current position in zInput */ - int iToken; /* index of next token to be returned */ - unsigned char *zToken; /* storage for current token */ - int nAllocated; /* space allocated to zToken buffer */ + const char *zInput; /* input we are tokenizing */ + int nInput; /* size of the input */ + int iOffset; /* current position in zInput */ + int iToken; /* index of next token to be returned */ + unsigned char *zToken; /* storage for current token */ + int nAllocated; /* space allocated to zToken buffer */ /** * Store the offset of the second character in the bi-gram pair that we just * emitted so that we can consider it being the first character in a bi-gram @@ -186,7 +184,7 @@ typedef struct porter_tokenizer_cursor { * an acceptable sentinel value because the 0th offset can never be the * offset of the second in a bi-gram pair. * - * For example, let us say we are tokenizing a string of 4 CJK characters + * For example, let us say we are tokenizing a string of 4 CJK characters * represented by the byte-string "11223344" where each repeated digit * indicates 2-bytes of storage used to encode the character in UTF-8. * (It actually takes 3, btw.) Then on the passes to emit each token, @@ -197,10 +195,9 @@ typedef struct porter_tokenizer_cursor { * 3344: iOffset = 6, iPrevBigramOffset = 4 * (nothing will be emitted): iOffset = 8, iPrevBigramOffset = 6 */ - int iPrevBigramOffset; /* previous result was bi-gram */ + int iPrevBigramOffset; /* previous result was bi-gram */ } porter_tokenizer_cursor; - /* Forward declaration */ static const sqlite3_tokenizer_module porterTokenizerModule; @@ -210,13 +207,11 @@ extern unsigned int normalize_character(const unsigned int c); /* ** Create a new tokenizer instance. */ -static int porterCreate( - int argc, const char * const *argv, - sqlite3_tokenizer **ppTokenizer -){ +static int porterCreate(int argc, const char *const *argv, + sqlite3_tokenizer **ppTokenizer) { porter_tokenizer *t; - t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t)); - if( t==NULL ) return SQLITE_NOMEM; + t = (porter_tokenizer *)sqlite3_malloc(sizeof(*t)); + if (t == NULL) return SQLITE_NOMEM; memset(t, 0, sizeof(*t)); *ppTokenizer = &t->base; return SQLITE_OK; @@ -225,7 +220,7 @@ static int porterCreate( /* ** Destroy a tokenizer */ -static int porterDestroy(sqlite3_tokenizer *pTokenizer){ +static int porterDestroy(sqlite3_tokenizer *pTokenizer) { sqlite3_free(pTokenizer); return SQLITE_OK; } @@ -233,30 +228,30 @@ static int porterDestroy(sqlite3_tokenizer *pTokenizer){ /* ** Prepare to begin tokenizing a particular string. The input ** string to be tokenized is zInput[0..nInput-1]. A cursor -** used to incrementally tokenize this string is returned in +** used to incrementally tokenize this string is returned in ** *ppCursor. */ static int porterOpen( - sqlite3_tokenizer *pTokenizer, /* The tokenizer */ - const char *zInput, int nInput, /* String to be tokenized */ - sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ -){ + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *zInput, int nInput, /* String to be tokenized */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +) { porter_tokenizer_cursor *c; - c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); - if( c==NULL ) return SQLITE_NOMEM; + c = (porter_tokenizer_cursor *)sqlite3_malloc(sizeof(*c)); + if (c == NULL) return SQLITE_NOMEM; c->zInput = zInput; - if( zInput==0 ){ + if (zInput == 0) { c->nInput = 0; - }else if( nInput<0 ){ + } else if (nInput < 0) { c->nInput = (int)strlen(zInput); - }else{ + } else { c->nInput = nInput; } - c->iOffset = 0; /* start tokenizing at the beginning */ + c->iOffset = 0; /* start tokenizing at the beginning */ c->iToken = 0; - c->zToken = NULL; /* no space allocated, yet. */ + c->zToken = NULL; /* no space allocated, yet. */ c->nAllocated = 0; c->iPrevBigramOffset = 0; @@ -268,8 +263,8 @@ static int porterOpen( ** Close a tokenization cursor previously opened by a call to ** porterOpen() above. */ -static int porterClose(sqlite3_tokenizer_cursor *pCursor){ - porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; +static int porterClose(sqlite3_tokenizer_cursor *pCursor) { + porter_tokenizer_cursor *c = (porter_tokenizer_cursor *)pCursor; sqlite3_free(c->zToken); sqlite3_free(c); return SQLITE_OK; @@ -277,15 +272,13 @@ static int porterClose(sqlite3_tokenizer_cursor *pCursor){ /* ** Vowel or consonant */ -static const char cType[] = { - 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, - 1, 1, 1, 2, 1 -}; +static const char cType[] = {0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, + 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 2, 1}; /* ** isConsonant() and isVowel() determine if their first character in ** the string they point to is a consonant or a vowel, according -** to Porter ruls. +** to Porter ruls. ** ** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'. ** 'Y' is a consonant unless it follows another consonant, @@ -295,23 +288,23 @@ static const char cType[] = { ** is that 'y' is a consonant unless it is followed by another ** consonant. */ -static int isVowel(const char*); -static int isConsonant(const char *z){ +static int isVowel(const char *); +static int isConsonant(const char *z) { int j; char x = *z; - if( x==0 ) return 0; - assert( x>='a' && x<='z' ); - j = cType[x-'a']; - if( j<2 ) return j; - return z[1]==0 || isVowel(z + 1); + if (x == 0) return 0; + assert(x >= 'a' && x <= 'z'); + j = cType[x - 'a']; + if (j < 2) return j; + return z[1] == 0 || isVowel(z + 1); } -static int isVowel(const char *z){ +static int isVowel(const char *z) { int j; char x = *z; - if( x==0 ) return 0; - assert( x>='a' && x<='z' ); - j = cType[x-'a']; - if( j<2 ) return 1-j; + if (x == 0) return 0; + assert(x >= 'a' && x <= 'z'); + j = cType[x - 'a']; + if (j < 2) return 1 - j; return isConsonant(z + 1); } @@ -334,47 +327,69 @@ static int isVowel(const char *z){ ** In this routine z[] is in reverse order. So we are really looking ** for an instance of of a consonant followed by a vowel. */ -static int m_gt_0(const char *z){ - while( isVowel(z) ){ z++; } - if( *z==0 ) return 0; - while( isConsonant(z) ){ z++; } - return *z!=0; +static int m_gt_0(const char *z) { + while (isVowel(z)) { + z++; + } + if (*z == 0) return 0; + while (isConsonant(z)) { + z++; + } + return *z != 0; } /* Like mgt0 above except we are looking for a value of m which is ** exactly 1 */ -static int m_eq_1(const char *z){ - while( isVowel(z) ){ z++; } - if( *z==0 ) return 0; - while( isConsonant(z) ){ z++; } - if( *z==0 ) return 0; - while( isVowel(z) ){ z++; } - if( *z==0 ) return 1; - while( isConsonant(z) ){ z++; } - return *z==0; +static int m_eq_1(const char *z) { + while (isVowel(z)) { + z++; + } + if (*z == 0) return 0; + while (isConsonant(z)) { + z++; + } + if (*z == 0) return 0; + while (isVowel(z)) { + z++; + } + if (*z == 0) return 1; + while (isConsonant(z)) { + z++; + } + return *z == 0; } /* Like mgt0 above except we are looking for a value of m>1 instead ** or m>0 */ -static int m_gt_1(const char *z){ - while( isVowel(z) ){ z++; } - if( *z==0 ) return 0; - while( isConsonant(z) ){ z++; } - if( *z==0 ) return 0; - while( isVowel(z) ){ z++; } - if( *z==0 ) return 0; - while( isConsonant(z) ){ z++; } - return *z!=0; +static int m_gt_1(const char *z) { + while (isVowel(z)) { + z++; + } + if (*z == 0) return 0; + while (isConsonant(z)) { + z++; + } + if (*z == 0) return 0; + while (isVowel(z)) { + z++; + } + if (*z == 0) return 0; + while (isConsonant(z)) { + z++; + } + return *z != 0; } /* ** Return TRUE if there is a vowel anywhere within z[0..n-1] */ -static int hasVowel(const char *z){ - while( isConsonant(z) ){ z++; } - return *z!=0; +static int hasVowel(const char *z) { + while (isConsonant(z)) { + z++; + } + return *z != 0; } /* @@ -383,8 +398,8 @@ static int hasVowel(const char *z){ ** The text is reversed here. So we are really looking at ** the first two characters of z[]. */ -static int doubleConsonant(const char *z){ - return isConsonant(z) && z[0]==z[1] && isConsonant(z+1); +static int doubleConsonant(const char *z) { + return isConsonant(z) && z[0] == z[1] && isConsonant(z + 1); } /* @@ -395,37 +410,38 @@ static int doubleConsonant(const char *z){ ** The word is reversed here. So we are really checking the ** first three letters and the first one cannot be in [wxy]. */ -static int star_oh(const char *z){ - return - z[0]!=0 && isConsonant(z) && - z[0]!='w' && z[0]!='x' && z[0]!='y' && - z[1]!=0 && isVowel(z+1) && - z[2]!=0 && isConsonant(z+2); +static int star_oh(const char *z) { + return z[0] != 0 && isConsonant(z) && z[0] != 'w' && z[0] != 'x' && + z[0] != 'y' && z[1] != 0 && isVowel(z + 1) && z[2] != 0 && + isConsonant(z + 2); } /* ** If the word ends with zFrom and xCond() is true for the stem -** of the word that precedes the zFrom ending, then change the +** of the word that precedes the zFrom ending, then change the ** ending to zTo. ** ** The input word *pz and zFrom are both in reverse order. zTo -** is in normal order. +** is in normal order. ** ** Return TRUE if zFrom matches. Return FALSE if zFrom does not ** match. Not that TRUE is returned even if xCond() fails and ** no substitution occurs. */ static int stem( - char **pz, /* The word being stemmed (Reversed) */ - const char *zFrom, /* If the ending matches this... (Reversed) */ - const char *zTo, /* ... change the ending to this (not reversed) */ - int (*xCond)(const char*) /* Condition that must be true */ -){ + char **pz, /* The word being stemmed (Reversed) */ + const char *zFrom, /* If the ending matches this... (Reversed) */ + const char *zTo, /* ... change the ending to this (not reversed) */ + int (*xCond)(const char *) /* Condition that must be true */ +) { char *z = *pz; - while( *zFrom && *zFrom==*z ){ z++; zFrom++; } - if( *zFrom!=0 ) return 0; - if( xCond && !xCond(z) ) return 1; - while( *zTo ){ + while (*zFrom && *zFrom == *z) { + z++; + zFrom++; + } + if (*zFrom != 0) return 0; + if (xCond && !xCond(z)) return 1; + while (*zTo) { *(--z) = *(zTo++); } *pz = z; @@ -434,14 +450,13 @@ static int stem( /** * Voiced sound mark is only on Japanese. It is like accent. It combines with - * previous character. Example, "サ" (Katakana) with "゛" (voiced sound mark) is - * "ザ". Although full-width character mapping has combined character like "ザ", - * there is no combined character on half-width Katanaka character mapping. + * previous character. Example, "サ" (Katakana) with "゛" (voiced sound mark) + * is "ザ". Although full-width character mapping has combined character like + * "ザ", there is no combined character on half-width Katanaka character + * mapping. */ -static int isVoicedSoundMark(const unsigned int c) -{ - if (c == 0xff9e || c == 0xff9f || c == 0x3099 || c == 0x309a) - return 1; +static int isVoicedSoundMark(const unsigned int c) { + if (c == 0xff9e || c == 0xff9f || c == 0x3099 || c == 0x309a) return 1; return 0; } @@ -449,7 +464,7 @@ static int isVoicedSoundMark(const unsigned int c) * How many unicode characters to take from the front and back of a term in * |copy_stemmer|. */ -#define COPY_STEMMER_COPY_HALF_LEN 10 +# define COPY_STEMMER_COPY_HALF_LEN 10 /** * Normalizing but non-stemming term copying. @@ -492,7 +507,7 @@ static int isVoicedSoundMark(const unsigned int c) * @param pnBytesOut Integer to write the number of bytes in zOut into. */ static void copy_stemmer(const unsigned char *zIn, const int nBytesIn, - unsigned char *zOut, int *pnBytesOut){ + unsigned char *zOut, int *pnBytesOut) { const unsigned char *zInTerm = zIn + nBytesIn; unsigned char *zOutStart = zOut; unsigned int c; @@ -508,8 +523,7 @@ static void copy_stemmer(const unsigned char *zIn, const int nBytesIn, /* ignore voiced/semi-voiced sound mark */ if (!isVoicedSoundMark(c)) { /* advance one non-voiced sound mark character. */ - if (zBackStart) - READ_UTF8(zBackStart, zOut, trashC); + if (zBackStart) READ_UTF8(zBackStart, zOut, trashC); WRITE_UTF8(zOut, c); charCount++; @@ -530,7 +544,6 @@ static void copy_stemmer(const unsigned char *zIn, const int nBytesIn, *pnBytesOut = zOut - zOutStart; } - /* ** Stem the input word zIn[0..nIn-1]. Store the output in zOut. ** zOut is at least big enough to hold nIn bytes. Write the actual @@ -545,28 +558,24 @@ static void copy_stemmer(const unsigned char *zIn, const int nBytesIn, ** word contains digits, 3 bytes are taken from the beginning and ** 3 bytes from the end. For long words without digits, 10 bytes ** are taken from each end. US-ASCII case folding still applies. -** -** If the input word contains not digits but does characters not -** in [a-zA-Z] then no stemming is attempted and this routine just +** +** If the input word contains not digits but does characters not +** in [a-zA-Z] then no stemming is attempted and this routine just ** copies the input into the input into the output with US-ASCII ** case folding. ** ** Stemming never increases the length of the word. So there is ** no chance of overflowing the zOut buffer. */ -static void porter_stemmer( - const unsigned char *zIn, - unsigned int nIn, - unsigned char *zOut, - int *pnOut -){ +static void porter_stemmer(const unsigned char *zIn, unsigned int nIn, + unsigned char *zOut, int *pnOut) { unsigned int i, j, c; char zReverse[28]; char *z, *z2; const unsigned char *zTerm = zIn + nIn; const unsigned char *zTmp = zIn; - if( nIn<3 || nIn>=sizeof(zReverse)-7 ){ + if (nIn < 3 || nIn >= sizeof(zReverse) - 7) { /* The word is too big or too small for the porter stemmer. ** Fallback to the copy stemmer */ copy_stemmer(zIn, nIn, zOut, pnOut); @@ -575,196 +584,185 @@ static void porter_stemmer( for (j = sizeof(zReverse) - 6; zTmp < zTerm; j--) { READ_UTF8(zTmp, zTerm, c); c = normalize_character(c); - if( c>='a' && c<='z' ){ + if (c >= 'a' && c <= 'z') { zReverse[j] = c; - }else{ + } else { /* The use of a character not in [a-zA-Z] means that we fallback ** to the copy stemmer */ copy_stemmer(zIn, nIn, zOut, pnOut); return; } } - memset(&zReverse[sizeof(zReverse)-5], 0, 5); - z = &zReverse[j+1]; - + memset(&zReverse[sizeof(zReverse) - 5], 0, 5); + z = &zReverse[j + 1]; /* Step 1a */ - if( z[0]=='s' ){ - if( - !stem(&z, "sess", "ss", 0) && - !stem(&z, "sei", "i", 0) && - !stem(&z, "ss", "ss", 0) - ){ + if (z[0] == 's') { + if (!stem(&z, "sess", "ss", 0) && !stem(&z, "sei", "i", 0) && + !stem(&z, "ss", "ss", 0)) { z++; } } - /* Step 1b */ + /* Step 1b */ z2 = z; - if( stem(&z, "dee", "ee", m_gt_0) ){ + if (stem(&z, "dee", "ee", m_gt_0)) { /* Do nothing. The work was all in the test */ - }else if( - (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel)) - && z!=z2 - ){ - if( stem(&z, "ta", "ate", 0) || - stem(&z, "lb", "ble", 0) || - stem(&z, "zi", "ize", 0) ){ - /* Do nothing. The work was all in the test */ - }else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){ - z++; - }else if( m_eq_1(z) && star_oh(z) ){ - *(--z) = 'e'; - } + } else if ((stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel)) && + z != z2) { + if (stem(&z, "ta", "ate", 0) || stem(&z, "lb", "ble", 0) || + stem(&z, "zi", "ize", 0)) { + /* Do nothing. The work was all in the test */ + } else if (doubleConsonant(z) && (*z != 'l' && *z != 's' && *z != 'z')) { + z++; + } else if (m_eq_1(z) && star_oh(z)) { + *(--z) = 'e'; + } } /* Step 1c */ - if( z[0]=='y' && hasVowel(z+1) ){ + if (z[0] == 'y' && hasVowel(z + 1)) { z[0] = 'i'; } /* Step 2 */ - switch( z[1] ){ - case 'a': - (void) (stem(&z, "lanoita", "ate", m_gt_0) || + switch (z[1]) { + case 'a': + (void)(stem(&z, "lanoita", "ate", m_gt_0) || stem(&z, "lanoit", "tion", m_gt_0)); - break; - case 'c': - (void) (stem(&z, "icne", "ence", m_gt_0) || + break; + case 'c': + (void)(stem(&z, "icne", "ence", m_gt_0) || stem(&z, "icna", "ance", m_gt_0)); - break; - case 'e': - (void) (stem(&z, "rezi", "ize", m_gt_0)); - break; - case 'g': - (void) (stem(&z, "igol", "log", m_gt_0)); - break; - case 'l': - (void) (stem(&z, "ilb", "ble", m_gt_0) || - stem(&z, "illa", "al", m_gt_0) || - stem(&z, "iltne", "ent", m_gt_0) || - stem(&z, "ile", "e", m_gt_0) || + break; + case 'e': + (void)(stem(&z, "rezi", "ize", m_gt_0)); + break; + case 'g': + (void)(stem(&z, "igol", "log", m_gt_0)); + break; + case 'l': + (void)(stem(&z, "ilb", "ble", m_gt_0) || stem(&z, "illa", "al", m_gt_0) || + stem(&z, "iltne", "ent", m_gt_0) || stem(&z, "ile", "e", m_gt_0) || stem(&z, "ilsuo", "ous", m_gt_0)); - break; - case 'o': - (void) (stem(&z, "noitazi", "ize", m_gt_0) || + break; + case 'o': + (void)(stem(&z, "noitazi", "ize", m_gt_0) || stem(&z, "noita", "ate", m_gt_0) || stem(&z, "rota", "ate", m_gt_0)); - break; - case 's': - (void) (stem(&z, "msila", "al", m_gt_0) || + break; + case 's': + (void)(stem(&z, "msila", "al", m_gt_0) || stem(&z, "ssenevi", "ive", m_gt_0) || stem(&z, "ssenluf", "ful", m_gt_0) || stem(&z, "ssensuo", "ous", m_gt_0)); - break; - case 't': - (void) (stem(&z, "itila", "al", m_gt_0) || + break; + case 't': + (void)(stem(&z, "itila", "al", m_gt_0) || stem(&z, "itivi", "ive", m_gt_0) || stem(&z, "itilib", "ble", m_gt_0)); - break; + break; } /* Step 3 */ - switch( z[0] ){ - case 'e': - (void) (stem(&z, "etaci", "ic", m_gt_0) || - stem(&z, "evita", "", m_gt_0) || + switch (z[0]) { + case 'e': + (void)(stem(&z, "etaci", "ic", m_gt_0) || stem(&z, "evita", "", m_gt_0) || stem(&z, "ezila", "al", m_gt_0)); - break; - case 'i': - (void) (stem(&z, "itici", "ic", m_gt_0)); - break; - case 'l': - (void) (stem(&z, "laci", "ic", m_gt_0) || - stem(&z, "luf", "", m_gt_0)); - break; - case 's': - (void) (stem(&z, "ssen", "", m_gt_0)); - break; + break; + case 'i': + (void)(stem(&z, "itici", "ic", m_gt_0)); + break; + case 'l': + (void)(stem(&z, "laci", "ic", m_gt_0) || stem(&z, "luf", "", m_gt_0)); + break; + case 's': + (void)(stem(&z, "ssen", "", m_gt_0)); + break; } /* Step 4 */ - switch( z[1] ){ - case 'a': - if( z[0]=='l' && m_gt_1(z+2) ){ - z += 2; - } - break; - case 'c': - if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e') && m_gt_1(z+4) ){ - z += 4; - } - break; - case 'e': - if( z[0]=='r' && m_gt_1(z+2) ){ - z += 2; - } - break; - case 'i': - if( z[0]=='c' && m_gt_1(z+2) ){ - z += 2; - } - break; - case 'l': - if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){ - z += 4; - } - break; - case 'n': - if( z[0]=='t' ){ - if( z[2]=='a' ){ - if( m_gt_1(z+3) ){ - z += 3; - } - }else if( z[2]=='e' ){ - (void) (stem(&z, "tneme", "", m_gt_1) || - stem(&z, "tnem", "", m_gt_1) || - stem(&z, "tne", "", m_gt_1)); - } - } - break; - case 'o': - if( z[0]=='u' ){ - if( m_gt_1(z+2) ){ - z += 2; - } - }else if( z[3]=='s' || z[3]=='t' ){ - (void) (stem(&z, "noi", "", m_gt_1)); - } - break; - case 's': - if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){ - z += 3; - } - break; - case 't': - (void) (stem(&z, "eta", "", m_gt_1) || - stem(&z, "iti", "", m_gt_1)); - break; - case 'u': - if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){ - z += 3; - } - break; - case 'v': - case 'z': - if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){ - z += 3; - } - break; + switch (z[1]) { + case 'a': + if (z[0] == 'l' && m_gt_1(z + 2)) { + z += 2; + } + break; + case 'c': + if (z[0] == 'e' && z[2] == 'n' && (z[3] == 'a' || z[3] == 'e') && + m_gt_1(z + 4)) { + z += 4; + } + break; + case 'e': + if (z[0] == 'r' && m_gt_1(z + 2)) { + z += 2; + } + break; + case 'i': + if (z[0] == 'c' && m_gt_1(z + 2)) { + z += 2; + } + break; + case 'l': + if (z[0] == 'e' && z[2] == 'b' && (z[3] == 'a' || z[3] == 'i') && + m_gt_1(z + 4)) { + z += 4; + } + break; + case 'n': + if (z[0] == 't') { + if (z[2] == 'a') { + if (m_gt_1(z + 3)) { + z += 3; + } + } else if (z[2] == 'e') { + (void)(stem(&z, "tneme", "", m_gt_1) || + stem(&z, "tnem", "", m_gt_1) || stem(&z, "tne", "", m_gt_1)); + } + } + break; + case 'o': + if (z[0] == 'u') { + if (m_gt_1(z + 2)) { + z += 2; + } + } else if (z[3] == 's' || z[3] == 't') { + (void)(stem(&z, "noi", "", m_gt_1)); + } + break; + case 's': + if (z[0] == 'm' && z[2] == 'i' && m_gt_1(z + 3)) { + z += 3; + } + break; + case 't': + (void)(stem(&z, "eta", "", m_gt_1) || stem(&z, "iti", "", m_gt_1)); + break; + case 'u': + if (z[0] == 's' && z[2] == 'o' && m_gt_1(z + 3)) { + z += 3; + } + break; + case 'v': + case 'z': + if (z[0] == 'e' && z[2] == 'i' && m_gt_1(z + 3)) { + z += 3; + } + break; } /* Step 5a */ - if( z[0]=='e' ){ - if( m_gt_1(z+1) ){ + if (z[0] == 'e') { + if (m_gt_1(z + 1)) { z++; - }else if( m_eq_1(z+1) && !star_oh(z+1) ){ + } else if (m_eq_1(z + 1) && !star_oh(z + 1)) { z++; } } /* Step 5b */ - if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){ + if (m_gt_1(z) && z[0] == 'l' && z[1] == 'l') { z++; } @@ -773,7 +771,7 @@ static void porter_stemmer( */ *pnOut = i = strlen(z); zOut[i] = 0; - while( *z ){ + while (*z) { zOut[--i] = *(z++); } } @@ -783,12 +781,12 @@ static void porter_stemmer( * Letters and numbers can; punctuation (and 'del') can't. */ static const char porterIdChar[] = { -/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ + /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ }; /** @@ -803,7 +801,8 @@ static const char porterIdChar[] = { * 0x20 space. 0x205f is a "medium mathematical space" and defined as roughly * equivalent to an 0x20 space. */ -#define IS_UNI_SPACE(x) (((x)>=0x2000&&(x)<=0x200a) || (x)==0x202f || (x)==0x205f) +# define IS_UNI_SPACE(x) \ + (((x) >= 0x2000 && (x) <= 0x200a) || (x) == 0x202f || (x) == 0x205f) /** * What we are checking for: * - 0x3001: Ideographic comma (-> 0x2c ',') @@ -815,36 +814,38 @@ static const char porterIdChar[] = { * * It is possible we should be treating other things as delimiters! */ -#define IS_JA_DELIM(x) (((x)==0x3001)||((x)==0xFF64)||((x)==0xFF0E)||((x)==0x3002)||((x)==0xFF61)||((x)==0xFF0C)) +# define IS_JA_DELIM(x) \ + (((x) == 0x3001) || ((x) == 0xFF64) || ((x) == 0xFF0E) || \ + ((x) == 0x3002) || ((x) == 0xFF61) || ((x) == 0xFF0C)) /** * The previous character was a delimiter (which includes the start of the * string). */ -#define BIGRAM_RESET 0 +# define BIGRAM_RESET 0 /** * The previous character was a CJK character and we have only seen one of them. * If we had seen more than one in a row it would be the BIGRAM_USE state. */ -#define BIGRAM_UNKNOWN 1 +# define BIGRAM_UNKNOWN 1 /** * We have seen two or more CJK characters in a row. */ -#define BIGRAM_USE 2 +# define BIGRAM_USE 2 /** * The previous character was ASCII or something in the unicode general scripts * area that we do not believe is a delimiter. We call it 'alpha' as in * alphabetic/alphanumeric and something that should be tokenized based on * delimiters rather than on a bi-gram basis. */ -#define BIGRAM_ALPHA 3 +# define BIGRAM_ALPHA 3 static int isDelim( - const unsigned char *zCur, /* IN: current pointer of token */ - const unsigned char *zTerm, /* IN: one character beyond end of token */ - int *len, /* OUT: analyzed bytes in this token */ - int *state /* IN/OUT: analyze state */ -){ + const unsigned char *zCur, /* IN: current pointer of token */ + const unsigned char *zTerm, /* IN: one character beyond end of token */ + int *len, /* OUT: analyzed bytes in this token */ + int *state /* IN/OUT: analyze state */ +) { const unsigned char *zIn = zCur; unsigned int c; int delim; @@ -855,16 +856,16 @@ static int isDelim( *len = zIn - zCur; /* ASCII character range has rule */ - if( c < 0x80 ){ + if (c < 0x80) { // This is original porter stemmer isDelim logic. // 0x0 - 0x1f are all control characters, 0x20 is space, 0x21-0x2f are // punctuation. delim = (c < 0x30 || !porterIdChar[c - 0x30]); // cases: "&a", "&." - if (*state == BIGRAM_USE || *state == BIGRAM_UNKNOWN ){ + if (*state == BIGRAM_USE || *state == BIGRAM_UNKNOWN) { /* previous maybe CJK and current is ascii */ *state = BIGRAM_ALPHA; /*ascii*/ - delim = 1; /* must break */ + delim = 1; /* must break */ } else if (delim == 1) { // cases: "a.", ".." /* this is delimiter character */ @@ -881,11 +882,11 @@ static int isDelim( /* voiced/semi-voiced sound mark is ignore */ if (isVoicedSoundMark(c) && *state != BIGRAM_ALPHA) { /* ignore this because it is combined with previous char */ - return 0; + return 0; } /* this isn't CJK range, so return as no delim */ - // Anything less than 0x2000 (except to U+0E00-U+0EFF and U+1780-U+17FF) + // Anything less than 0x2000 (except to U+0E00-U+0EFF and U+1780-U+17FF) // is the general scripts area and should not be bi-gram indexed. // 0xa000 - 0a4cf is the Yi area. It is apparently a phonetic language whose // usage does not appear to have simple delimiter rules, so we're leaving it @@ -893,11 +894,9 @@ static int isDelim( // (We previously bailed on this range too.) // Addition, U+0E00-U+0E7F is Thai, U+0E80-U+0EFF is Laos, // and U+1780-U+17FF is Khmer. It is no easy way to break each word. - // So these should use bi-gram too. + // So these should use bi-gram too. // cases: "aa", ".a", "&a" - if (c < 0xe00 || - (c >= 0xf00 && c < 0x1780) || - (c >= 0x1800 && c < 0x2000)) { + if (c < 0xe00 || (c >= 0xf00 && c < 0x1780) || (c >= 0x1800 && c < 0x2000)) { *state = BIGRAM_ALPHA; /* not really ASCII but same idea; tokenize it */ return 0; } @@ -906,23 +905,23 @@ static int isDelim( /* this is space character or delim character */ // cases: "a.", "..", "&." - if( IS_UNI_SPACE(c) || IS_JA_DELIM(c) ){ + if (IS_UNI_SPACE(c) || IS_JA_DELIM(c)) { *state = BIGRAM_RESET; /* reset */ - return 1; /* it actually is a delimiter; report as such */ + return 1; /* it actually is a delimiter; report as such */ } // (at this point we must be a bi-grammable char) // cases: "a&" - if( *state==BIGRAM_ALPHA ){ + if (*state == BIGRAM_ALPHA) { /* Previous is ascii and current maybe CJK */ *state = BIGRAM_UNKNOWN; /* mark as unknown */ - return 1; /* break to emit the ASCII token*/ + return 1; /* break to emit the ASCII token*/ } /* We have no rule for CJK!. use bi-gram */ // cases: "&&" - if( *state==BIGRAM_UNKNOWN || *state==BIGRAM_USE ){ + if (*state == BIGRAM_UNKNOWN || *state == BIGRAM_USE) { /* previous state is unknown. mark as bi-gram */ *state = BIGRAM_USE; return 1; /* break to emit the digram */ @@ -930,7 +929,7 @@ static int isDelim( // cases: ".&" (*state == BIGRAM_RESET) *state = BIGRAM_UNKNOWN; /* mark as unknown */ - return 0; /* no need to break; nothing to emit */ + return 0; /* no need to break; nothing to emit */ } /** @@ -980,19 +979,19 @@ static int isDelim( * BIGRAM_USE, gets set to BIGRAM_USE. */ static int porterNext( - sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by porterOpen */ - const char **pzToken, /* OUT: *pzToken is the token text */ - int *pnBytes, /* OUT: Number of bytes in token */ - int *piStartOffset, /* OUT: Starting offset of token */ - int *piEndOffset, /* OUT: Ending offset of token */ - int *piPosition /* OUT: Position integer of token */ -){ - porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; - const unsigned char *z = (unsigned char *) c->zInput; + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by porterOpen */ + const char **pzToken, /* OUT: *pzToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +) { + porter_tokenizer_cursor *c = (porter_tokenizer_cursor *)pCursor; + const unsigned char *z = (unsigned char *)c->zInput; int len = 0; int state; - while( c->iOffset < c->nInput ){ + while (c->iOffset < c->nInput) { int iStartOffset, numChars; /* @@ -1010,13 +1009,13 @@ static int porterNext( * delimiter we hit was a CJK character, the next time through we will * not treat it as a delimiter though; the entry state for that scan is * BIGRAM_RESET so the transition is not treated as a delimiter! - * + * * The CJK pass always starts with the second character in a bi-gram emitted * as a token in the previous step. No delimiter skipping is required * because we know that first character might produce a token for us. It * only 'might' produce a token because the previous pass performed no * lookahead and cannot be sure it is followed by another CJK character. - * This is why + * This is why */ // If we have a previous bigram offset @@ -1090,23 +1089,20 @@ static int porterNext( // It is possible we have no token to emit here if iPrevBigramOffset was not // 0 on entry and there was no second CJK character. iPrevBigramOffset // will now be 0 if that is the case (and c->iOffset == iStartOffset). - if (// allow two-character words only if in bigram + if ( // allow two-character words only if in bigram (numChars == 2 && state == BIGRAM_USE) || // otherwise, drop two-letter words (considered stop-words) - (numChars >=3) || + (numChars >= 3) || // wildcard case: - (numChars == 1 && iStartOffset == 0 && - (c->iOffset >= 3) && - (c->iOffset == c->nInput - 1) && - (z[c->iOffset] == '*'))) { + (numChars == 1 && iStartOffset == 0 && (c->iOffset >= 3) && + (c->iOffset == c->nInput - 1) && (z[c->iOffset] == '*'))) { /* figure out the number of bytes to copy/stem */ int n = c->iOffset - iStartOffset; /* make sure there is enough buffer space */ if (n * MAX_UTF8_GROWTH_FACTOR > c->nAllocated) { c->nAllocated = n * MAX_UTF8_GROWTH_FACTOR + 20; c->zToken = sqlite3_realloc(c->zToken, c->nAllocated); - if (c->zToken == NULL) - return SQLITE_NOMEM; + if (c->zToken == NULL) return SQLITE_NOMEM; } if (state == BIGRAM_USE) { @@ -1115,7 +1111,7 @@ static int porterNext( } else { porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes); } - *pzToken = (const char*)c->zToken; + *pzToken = (const char *)c->zToken; *piStartOffset = iStartOffset; *piEndOffset = c->iOffset; *piPosition = c->iToken++; @@ -1129,12 +1125,7 @@ static int porterNext( ** The set of routines that implement the porter-stemmer tokenizer */ static const sqlite3_tokenizer_module porterTokenizerModule = { - 0, - porterCreate, - porterDestroy, - porterOpen, - porterClose, - porterNext, + 0, porterCreate, porterDestroy, porterOpen, porterClose, porterNext, }; /* @@ -1142,8 +1133,7 @@ static const sqlite3_tokenizer_module porterTokenizerModule = { ** tokenizer in *ppModule */ void sqlite3Fts3PorterTokenizerModule( - sqlite3_tokenizer_module const**ppModule -){ + sqlite3_tokenizer_module const **ppModule) { *ppModule = &porterTokenizerModule; } diff --git a/mailnews/extensions/fts3/src/fts3_tokenizer.h b/mailnews/extensions/fts3/src/fts3_tokenizer.h index 80ca8383e5..aaaecad2b6 100644 --- a/mailnews/extensions/fts3/src/fts3_tokenizer.h +++ b/mailnews/extensions/fts3/src/fts3_tokenizer.h @@ -50,7 +50,6 @@ typedef struct sqlite3_tokenizer sqlite3_tokenizer; typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; struct sqlite3_tokenizer_module { - /* ** Structure version. Should always be set to 0. */ @@ -73,10 +72,9 @@ struct sqlite3_tokenizer_module { ** sqlite3_tokenizer.pModule variable should not be initialised by ** this callback. The caller will do so. */ - int (*xCreate)( - int argc, /* Size of argv array */ - const char *const*argv, /* Tokenizer argument strings */ - sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ + int (*xCreate)(int argc, /* Size of argv array */ + const char *const *argv, /* Tokenizer argument strings */ + sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ ); /* @@ -91,9 +89,9 @@ struct sqlite3_tokenizer_module { ** until the cursor is closed (using the xClose() method). */ int (*xOpen)( - sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ - const char *pInput, int nBytes, /* Input buffer */ - sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ + sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ + const char *pInput, int nBytes, /* Input buffer */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ ); /* @@ -127,21 +125,21 @@ struct sqlite3_tokenizer_module { ** should be converted to zInput. */ int (*xNext)( - sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ - const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ - int *piStartOffset, /* OUT: Byte offset of token in input buffer */ - int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ - int *piPosition /* OUT: Number of tokens returned before this one */ + sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ + const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ + int *piStartOffset, /* OUT: Byte offset of token in input buffer */ + int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ + int *piPosition /* OUT: Number of tokens returned before this one */ ); }; struct sqlite3_tokenizer { - const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ + const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ /* Tokenizer implementations will typically add additional fields */ }; struct sqlite3_tokenizer_cursor { - sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ + sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ /* Tokenizer implementations will typically add additional fields */ }; diff --git a/mailnews/extensions/fts3/src/nsFts3Tokenizer.cpp b/mailnews/extensions/fts3/src/nsFts3Tokenizer.cpp index 212c79946a..89f0e50d31 100644 --- a/mailnews/extensions/fts3/src/nsFts3Tokenizer.cpp +++ b/mailnews/extensions/fts3/src/nsFts3Tokenizer.cpp @@ -13,45 +13,36 @@ #include "nsString.h" extern "C" void sqlite3Fts3PorterTokenizerModule( - sqlite3_tokenizer_module const**ppModule); + sqlite3_tokenizer_module const **ppModule); -extern "C" void glodaRankFunc(sqlite3_context *pCtx, - int nVal, +extern "C" void glodaRankFunc(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal); -NS_IMPL_ISUPPORTS(nsFts3Tokenizer,nsIFts3Tokenizer) +NS_IMPL_ISUPPORTS(nsFts3Tokenizer, nsIFts3Tokenizer) -nsFts3Tokenizer::nsFts3Tokenizer() -{ -} +nsFts3Tokenizer::nsFts3Tokenizer() {} -nsFts3Tokenizer::~nsFts3Tokenizer() -{ -} +nsFts3Tokenizer::~nsFts3Tokenizer() {} NS_IMETHODIMP -nsFts3Tokenizer::RegisterTokenizer(mozIStorageConnection *connection) -{ +nsFts3Tokenizer::RegisterTokenizer(mozIStorageConnection *connection) { nsresult rv; nsCOMPtr selectStatement; // -- register the tokenizer - rv = connection->CreateStatement(NS_LITERAL_CSTRING( - "SELECT fts3_tokenizer(?1, ?2)"), - getter_AddRefs(selectStatement)); + rv = connection->CreateStatement( + NS_LITERAL_CSTRING("SELECT fts3_tokenizer(?1, ?2)"), + getter_AddRefs(selectStatement)); NS_ENSURE_SUCCESS(rv, rv); - const sqlite3_tokenizer_module* module = nullptr; + const sqlite3_tokenizer_module *module = nullptr; sqlite3Fts3PorterTokenizerModule(&module); - if (!module) - return NS_ERROR_FAILURE; + if (!module) return NS_ERROR_FAILURE; - rv = selectStatement->BindUTF8StringByIndex( - 0, NS_LITERAL_CSTRING("mozporter")); + rv = selectStatement->BindUTF8StringByIndex(0, + NS_LITERAL_CSTRING("mozporter")); NS_ENSURE_SUCCESS(rv, rv); - rv = selectStatement->BindBlobByIndex(1, - (uint8_t*)&module, - sizeof(module)); + rv = selectStatement->BindBlobByIndex(1, (uint8_t *)&module, sizeof(module)); NS_ENSURE_SUCCESS(rv, rv); bool hasMore; @@ -61,11 +52,9 @@ nsFts3Tokenizer::RegisterTokenizer(mozIStorageConnection *connection) // -- register the ranking function nsCOMPtr func = new nsGlodaRankerFunction(); NS_ENSURE_TRUE(func, NS_ERROR_OUT_OF_MEMORY); - rv = connection->CreateFunction( - NS_LITERAL_CSTRING("glodaRank"), - -1, // variable argument support - func - ); + rv = connection->CreateFunction(NS_LITERAL_CSTRING("glodaRank"), + -1, // variable argument support + func); NS_ENSURE_SUCCESS(rv, rv); return rv; diff --git a/mailnews/extensions/fts3/src/nsFts3Tokenizer.h b/mailnews/extensions/fts3/src/nsFts3Tokenizer.h index eb1e83bd5a..8e6bb7dab4 100644 --- a/mailnews/extensions/fts3/src/nsFts3Tokenizer.h +++ b/mailnews/extensions/fts3/src/nsFts3Tokenizer.h @@ -13,13 +13,13 @@ extern const sqlite3_tokenizer_module* getWindowsTokenizer(); class nsFts3Tokenizer final : public nsIFts3Tokenizer { -public: - NS_DECL_ISUPPORTS - NS_DECL_NSIFTS3TOKENIZER + public: + NS_DECL_ISUPPORTS + NS_DECL_NSIFTS3TOKENIZER - nsFts3Tokenizer(); + nsFts3Tokenizer(); -private: + private: ~nsFts3Tokenizer(); }; diff --git a/mailnews/extensions/fts3/src/nsFts3TokenizerCID.h b/mailnews/extensions/fts3/src/nsFts3TokenizerCID.h index a75a0a28fe..4d17592638 100644 --- a/mailnews/extensions/fts3/src/nsFts3TokenizerCID.h +++ b/mailnews/extensions/fts3/src/nsFts3TokenizerCID.h @@ -6,10 +6,12 @@ #ifndef nsFts3TokenizerCID_h__ #define nsFts3TokenizerCID_h__ -#define NS_FTS3TOKENIZER_CONTRACTID \ - "@mozilla.org/messenger/fts3tokenizer;1" -#define NS_FTS3TOKENIZER_CID \ -{ 0xa67d724d, 0x0015, 0x4e2e, \ -{ 0x8c, 0xad, 0xb8, 0x47, 0x75, 0x33, 0x09, 0x24 }} +#define NS_FTS3TOKENIZER_CONTRACTID "@mozilla.org/messenger/fts3tokenizer;1" +#define NS_FTS3TOKENIZER_CID \ + { \ + 0xa67d724d, 0x0015, 0x4e2e, { \ + 0x8c, 0xad, 0xb8, 0x47, 0x75, 0x33, 0x09, 0x24 \ + } \ + } #endif /* nsFts3TokenizerCID_h__ */ diff --git a/mailnews/extensions/fts3/src/nsGlodaRankerFunction.cpp b/mailnews/extensions/fts3/src/nsGlodaRankerFunction.cpp index a0e834fdf5..a66b6485ff 100644 --- a/mailnews/extensions/fts3/src/nsGlodaRankerFunction.cpp +++ b/mailnews/extensions/fts3/src/nsGlodaRankerFunction.cpp @@ -12,18 +12,14 @@ #include "nsComponentManagerUtils.h" #ifndef SQLITE_VERSION_NUMBER -#error "We need SQLITE_VERSION_NUMBER defined!" +# error "We need SQLITE_VERSION_NUMBER defined!" #endif NS_IMPL_ISUPPORTS(nsGlodaRankerFunction, mozIStorageFunction) -nsGlodaRankerFunction::nsGlodaRankerFunction() -{ -} +nsGlodaRankerFunction::nsGlodaRankerFunction() {} -nsGlodaRankerFunction::~nsGlodaRankerFunction() -{ -} +nsGlodaRankerFunction::~nsGlodaRankerFunction() {} static uint32_t COLUMN_SATURATION[] = {10, 1, 1, 1, 1}; @@ -37,8 +33,7 @@ static uint32_t COLUMN_SATURATION[] = {10, 1, 1, 1, 1}; */ NS_IMETHODIMP nsGlodaRankerFunction::OnFunctionCall(mozIStorageValueArray *aArguments, - nsIVariant **_result) -{ + nsIVariant **_result) { // all argument names are maintained from the original SQLite code. uint32_t nVal; nsresult rv = aArguments->GetNumEntries(&nVal); @@ -50,16 +45,14 @@ nsGlodaRankerFunction::OnFunctionCall(mozIStorageValueArray *aArguments, * to contain the number of reportable phrases in the users full-text * query, and nCol to the number of columns in the table. */ - if (nVal < 1) - return NS_ERROR_INVALID_ARG; + if (nVal < 1) return NS_ERROR_INVALID_ARG; uint32_t lenArgsData; uint32_t *aArgsData = (uint32_t *)aArguments->AsSharedBlob(0, &lenArgsData); uint32_t nPhrase = aArgsData[0]; uint32_t nCol = aArgsData[1]; - if (nVal != (1 + nCol)) - return NS_ERROR_INVALID_ARG; + if (nVal != (1 + nCol)) return NS_ERROR_INVALID_ARG; double score = 0.0; @@ -69,12 +62,12 @@ nsGlodaRankerFunction::OnFunctionCall(mozIStorageValueArray *aArguments, for (uint32_t iPhrase = 0; iPhrase < nPhrase; iPhrase++) { // in SQ for (uint32_t iCol = 0; iCol < nCol; iCol++) { - uint32_t nHitCount = aArgsData[2 + (iPhrase+1)*nCol + iCol]; - double weight = aArguments->AsDouble(iCol+1); + uint32_t nHitCount = aArgsData[2 + (iPhrase + 1) * nCol + iCol]; + double weight = aArguments->AsDouble(iCol + 1); if (nHitCount > 0) { - score += (nHitCount > COLUMN_SATURATION[iCol]) ? - (COLUMN_SATURATION[iCol] * weight) : - (nHitCount * weight); + score += (nHitCount > COLUMN_SATURATION[iCol]) + ? (COLUMN_SATURATION[iCol] * weight) + : (nHitCount * weight); } } } @@ -90,14 +83,14 @@ nsGlodaRankerFunction::OnFunctionCall(mozIStorageValueArray *aArguments, ** the hit count and global hit counts for each column are found in ** aPhraseinfo[iCol*3] and aPhraseinfo[iCol*3+1], respectively. */ - uint32_t *aPhraseinfo = &aArgsData[2 + iPhrase*nCol*3]; + uint32_t *aPhraseinfo = &aArgsData[2 + iPhrase * nCol * 3]; for (uint32_t iCol = 0; iCol < nCol; iCol++) { uint32_t nHitCount = aPhraseinfo[3 * iCol]; - double weight = aArguments->AsDouble(iCol+1); + double weight = aArguments->AsDouble(iCol + 1); if (nHitCount > 0) { - score += (nHitCount > COLUMN_SATURATION[iCol]) ? - (COLUMN_SATURATION[iCol] * weight) : - (nHitCount * weight); + score += (nHitCount > COLUMN_SATURATION[iCol]) + ? (COLUMN_SATURATION[iCol] * weight) + : (nHitCount * weight); } } } diff --git a/mailnews/extensions/fts3/src/nsGlodaRankerFunction.h b/mailnews/extensions/fts3/src/nsGlodaRankerFunction.h index 5c5c920d05..0a19073a90 100644 --- a/mailnews/extensions/fts3/src/nsGlodaRankerFunction.h +++ b/mailnews/extensions/fts3/src/nsGlodaRankerFunction.h @@ -11,15 +11,15 @@ * Basically a port of the example FTS3 ranking function to mozStorage's * view of the universe. This might get fancier at some point. */ -class nsGlodaRankerFunction final : public mozIStorageFunction -{ -public: +class nsGlodaRankerFunction final : public mozIStorageFunction { + public: NS_DECL_ISUPPORTS NS_DECL_MOZISTORAGEFUNCTION nsGlodaRankerFunction(); -private: + + private: ~nsGlodaRankerFunction(); }; -#endif // _nsGlodaRankerFunction_h_ +#endif // _nsGlodaRankerFunction_h_ diff --git a/mailnews/extensions/mailviews/src/nsMsgMailViewList.cpp b/mailnews/extensions/mailviews/src/nsMsgMailViewList.cpp index e8a03e7638..2877d26eab 100644 --- a/mailnews/extensions/mailviews/src/nsMsgMailViewList.cpp +++ b/mailnews/extensions/mailviews/src/nsMsgMailViewList.cpp @@ -24,287 +24,265 @@ #define kDefaultViewNotJunk "Not Junk" #define kDefaultViewHasAttachments "Has Attachments" -nsMsgMailView::nsMsgMailView() -{ - mViewSearchTerms = nsArray::Create(); - NS_ASSERTION(mViewSearchTerms, "Failed to allocate a nsIMutableArray for mViewSearchTerms"); +nsMsgMailView::nsMsgMailView() { + mViewSearchTerms = nsArray::Create(); + NS_ASSERTION(mViewSearchTerms, + "Failed to allocate a nsIMutableArray for mViewSearchTerms"); } NS_IMPL_ADDREF(nsMsgMailView) NS_IMPL_RELEASE(nsMsgMailView) NS_IMPL_QUERY_INTERFACE(nsMsgMailView, nsIMsgMailView) -nsMsgMailView::~nsMsgMailView() -{ - if (mViewSearchTerms) - mViewSearchTerms->Clear(); +nsMsgMailView::~nsMsgMailView() { + if (mViewSearchTerms) mViewSearchTerms->Clear(); } -NS_IMETHODIMP nsMsgMailView::GetMailViewName(char16_t ** aMailViewName) -{ - NS_ENSURE_ARG_POINTER(aMailViewName); +NS_IMETHODIMP nsMsgMailView::GetMailViewName(char16_t **aMailViewName) { + NS_ENSURE_ARG_POINTER(aMailViewName); + *aMailViewName = ToNewUnicode(mName); + return NS_OK; +} + +NS_IMETHODIMP nsMsgMailView::SetMailViewName(const char16_t *aMailViewName) { + mName = aMailViewName; + return NS_OK; +} + +NS_IMETHODIMP nsMsgMailView::GetPrettyName(char16_t **aMailViewName) { + NS_ENSURE_ARG_POINTER(aMailViewName); + + nsresult rv = NS_OK; + if (!mBundle) { + nsCOMPtr bundleService = + mozilla::services::GetStringBundleService(); + NS_ENSURE_TRUE(bundleService, NS_ERROR_UNEXPECTED); + bundleService->CreateBundle( + "chrome://messenger/locale/mailviews.properties", + getter_AddRefs(mBundle)); + } + + NS_ENSURE_TRUE(mBundle, NS_ERROR_FAILURE); + + // see if mName has an associated pretty name inside our string bundle and if + // so, use that as the pretty name otherwise just return mName + nsAutoString mailViewName; + if (mName.EqualsLiteral(kDefaultViewPeopleIKnow)) { + rv = mBundle->GetStringFromName("mailViewPeopleIKnow", mailViewName); + *aMailViewName = ToNewUnicode(mailViewName); + } else if (mName.EqualsLiteral(kDefaultViewRecent)) { + rv = mBundle->GetStringFromName("mailViewRecentMail", mailViewName); + *aMailViewName = ToNewUnicode(mailViewName); + } else if (mName.EqualsLiteral(kDefaultViewFiveDays)) { + rv = mBundle->GetStringFromName("mailViewLastFiveDays", mailViewName); + *aMailViewName = ToNewUnicode(mailViewName); + } else if (mName.EqualsLiteral(kDefaultViewNotJunk)) { + rv = mBundle->GetStringFromName("mailViewNotJunk", mailViewName); + *aMailViewName = ToNewUnicode(mailViewName); + } else if (mName.EqualsLiteral(kDefaultViewHasAttachments)) { + rv = mBundle->GetStringFromName("mailViewHasAttachments", mailViewName); + *aMailViewName = ToNewUnicode(mailViewName); + } else { *aMailViewName = ToNewUnicode(mName); - return NS_OK; + } + + return rv; } -NS_IMETHODIMP nsMsgMailView::SetMailViewName(const char16_t * aMailViewName) -{ - mName = aMailViewName; - return NS_OK; +NS_IMETHODIMP nsMsgMailView::GetSearchTerms(nsIMutableArray **aSearchTerms) { + NS_ENSURE_ARG_POINTER(aSearchTerms); + NS_IF_ADDREF(*aSearchTerms = mViewSearchTerms); + return NS_OK; } -NS_IMETHODIMP nsMsgMailView::GetPrettyName(char16_t ** aMailViewName) -{ - NS_ENSURE_ARG_POINTER(aMailViewName); - - nsresult rv = NS_OK; - if (!mBundle) - { - nsCOMPtr bundleService = - mozilla::services::GetStringBundleService(); - NS_ENSURE_TRUE(bundleService, NS_ERROR_UNEXPECTED); - bundleService->CreateBundle("chrome://messenger/locale/mailviews.properties", - getter_AddRefs(mBundle)); - } - - NS_ENSURE_TRUE(mBundle, NS_ERROR_FAILURE); - - // see if mName has an associated pretty name inside our string bundle and if so, use that as the pretty name - // otherwise just return mName - nsAutoString mailViewName; - if (mName.EqualsLiteral(kDefaultViewPeopleIKnow)) { - rv = mBundle->GetStringFromName("mailViewPeopleIKnow", mailViewName); - *aMailViewName = ToNewUnicode(mailViewName); - } else if (mName.EqualsLiteral(kDefaultViewRecent)) { - rv = mBundle->GetStringFromName("mailViewRecentMail", mailViewName); - *aMailViewName = ToNewUnicode(mailViewName); - } else if (mName.EqualsLiteral(kDefaultViewFiveDays)) { - rv = mBundle->GetStringFromName("mailViewLastFiveDays", mailViewName); - *aMailViewName = ToNewUnicode(mailViewName); - } else if (mName.EqualsLiteral(kDefaultViewNotJunk)) { - rv = mBundle->GetStringFromName("mailViewNotJunk", mailViewName); - *aMailViewName = ToNewUnicode(mailViewName); - } else if (mName.EqualsLiteral(kDefaultViewHasAttachments)) { - rv = mBundle->GetStringFromName("mailViewHasAttachments", mailViewName); - *aMailViewName = ToNewUnicode(mailViewName); - } else { - *aMailViewName = ToNewUnicode(mName); - } - - return rv; +NS_IMETHODIMP nsMsgMailView::SetSearchTerms(nsIMutableArray *aSearchTerms) { + mViewSearchTerms = aSearchTerms; + return NS_OK; } -NS_IMETHODIMP nsMsgMailView::GetSearchTerms(nsIMutableArray **aSearchTerms) -{ - NS_ENSURE_ARG_POINTER(aSearchTerms); - NS_IF_ADDREF(*aSearchTerms = mViewSearchTerms); - return NS_OK; +NS_IMETHODIMP nsMsgMailView::AppendTerm(nsIMsgSearchTerm *aTerm) { + NS_ENSURE_TRUE(aTerm, NS_ERROR_NULL_POINTER); + + return mViewSearchTerms->AppendElement(aTerm); } -NS_IMETHODIMP nsMsgMailView::SetSearchTerms(nsIMutableArray *aSearchTerms) -{ - mViewSearchTerms = aSearchTerms; - return NS_OK; -} - -NS_IMETHODIMP nsMsgMailView::AppendTerm(nsIMsgSearchTerm *aTerm) -{ - NS_ENSURE_TRUE(aTerm, NS_ERROR_NULL_POINTER); - - return mViewSearchTerms->AppendElement(aTerm); -} - -NS_IMETHODIMP nsMsgMailView::CreateTerm(nsIMsgSearchTerm **aResult) -{ - NS_ENSURE_ARG_POINTER(aResult); - nsCOMPtr searchTerm = do_CreateInstance("@mozilla.org/messenger/searchTerm;1"); - searchTerm.forget(aResult); - return NS_OK; +NS_IMETHODIMP nsMsgMailView::CreateTerm(nsIMsgSearchTerm **aResult) { + NS_ENSURE_ARG_POINTER(aResult); + nsCOMPtr searchTerm = + do_CreateInstance("@mozilla.org/messenger/searchTerm;1"); + searchTerm.forget(aResult); + return NS_OK; } ///////////////////////////////////////////////////////////////////////////// // nsMsgMailViewList implementation ///////////////////////////////////////////////////////////////////////////// -nsMsgMailViewList::nsMsgMailViewList() -{ - LoadMailViews(); -} +nsMsgMailViewList::nsMsgMailViewList() { LoadMailViews(); } NS_IMPL_ADDREF(nsMsgMailViewList) NS_IMPL_RELEASE(nsMsgMailViewList) NS_IMPL_QUERY_INTERFACE(nsMsgMailViewList, nsIMsgMailViewList) -nsMsgMailViewList::~nsMsgMailViewList() -{ +nsMsgMailViewList::~nsMsgMailViewList() {} +NS_IMETHODIMP nsMsgMailViewList::GetMailViewCount(uint32_t *aCount) { + NS_ENSURE_ARG_POINTER(aCount); + + *aCount = m_mailViews.Length(); + return NS_OK; } -NS_IMETHODIMP nsMsgMailViewList::GetMailViewCount(uint32_t * aCount) -{ - NS_ENSURE_ARG_POINTER(aCount); +NS_IMETHODIMP nsMsgMailViewList::GetMailViewAt(uint32_t aMailViewIndex, + nsIMsgMailView **aMailView) { + NS_ENSURE_ARG_POINTER(aMailView); - *aCount = m_mailViews.Length(); - return NS_OK; + uint32_t mailViewCount = m_mailViews.Length(); + + NS_ENSURE_ARG(mailViewCount > aMailViewIndex); + + NS_IF_ADDREF(*aMailView = m_mailViews[aMailViewIndex]); + return NS_OK; } -NS_IMETHODIMP nsMsgMailViewList::GetMailViewAt(uint32_t aMailViewIndex, nsIMsgMailView ** aMailView) -{ - NS_ENSURE_ARG_POINTER(aMailView); +NS_IMETHODIMP nsMsgMailViewList::AddMailView(nsIMsgMailView *aMailView) { + NS_ENSURE_ARG_POINTER(aMailView); - uint32_t mailViewCount = m_mailViews.Length(); - - NS_ENSURE_ARG(mailViewCount > aMailViewIndex); - - NS_IF_ADDREF(*aMailView = m_mailViews[aMailViewIndex]); - return NS_OK; + m_mailViews.AppendElement(aMailView); + return NS_OK; } -NS_IMETHODIMP nsMsgMailViewList::AddMailView(nsIMsgMailView * aMailView) -{ - NS_ENSURE_ARG_POINTER(aMailView); +NS_IMETHODIMP nsMsgMailViewList::RemoveMailView(nsIMsgMailView *aMailView) { + NS_ENSURE_ARG_POINTER(aMailView); - m_mailViews.AppendElement(aMailView); - return NS_OK; + m_mailViews.RemoveElement(aMailView); + return NS_OK; } -NS_IMETHODIMP nsMsgMailViewList::RemoveMailView(nsIMsgMailView * aMailView) -{ - NS_ENSURE_ARG_POINTER(aMailView); - - m_mailViews.RemoveElement(aMailView); - return NS_OK; +NS_IMETHODIMP nsMsgMailViewList::CreateMailView(nsIMsgMailView **aMailView) { + NS_ENSURE_ARG_POINTER(aMailView); + NS_ADDREF(*aMailView = new nsMsgMailView); + return NS_OK; } -NS_IMETHODIMP nsMsgMailViewList::CreateMailView(nsIMsgMailView ** aMailView) -{ - NS_ENSURE_ARG_POINTER(aMailView); - NS_ADDREF(*aMailView = new nsMsgMailView); - return NS_OK; +NS_IMETHODIMP nsMsgMailViewList::Save() { + // brute force...remove all the old filters in our filter list, then we'll + // re-add our current list + nsCOMPtr msgFilter; + uint32_t numFilters = 0; + if (mFilterList) mFilterList->GetFilterCount(&numFilters); + while (numFilters) { + mFilterList->RemoveFilterAt(numFilters - 1); + numFilters--; + } + + // now convert our mail view list into a filter list and save it + ConvertMailViewListToFilterList(); + + // now save the filters to our file + return mFilterList ? mFilterList->SaveToDefaultFile() : NS_ERROR_FAILURE; } -NS_IMETHODIMP nsMsgMailViewList::Save() -{ - // brute force...remove all the old filters in our filter list, then we'll re-add our current - // list - nsCOMPtr msgFilter; - uint32_t numFilters = 0; - if (mFilterList) - mFilterList->GetFilterCount(&numFilters); - while (numFilters) - { - mFilterList->RemoveFilterAt(numFilters - 1); - numFilters--; - } - - // now convert our mail view list into a filter list and save it - ConvertMailViewListToFilterList(); - - // now save the filters to our file - return mFilterList ? mFilterList->SaveToDefaultFile() : NS_ERROR_FAILURE; -} - -nsresult nsMsgMailViewList::ConvertMailViewListToFilterList() -{ +nsresult nsMsgMailViewList::ConvertMailViewListToFilterList() { uint32_t mailViewCount = m_mailViews.Length(); nsCOMPtr mailView; nsCOMPtr newMailFilter; nsString mailViewName; - for (uint32_t index = 0; index < mailViewCount; index++) - { - GetMailViewAt(index, getter_AddRefs(mailView)); - if (!mailView) - continue; - mailView->GetMailViewName(getter_Copies(mailViewName)); - mFilterList->CreateFilter(mailViewName, getter_AddRefs(newMailFilter)); - if (!newMailFilter) - continue; + for (uint32_t index = 0; index < mailViewCount; index++) { + GetMailViewAt(index, getter_AddRefs(mailView)); + if (!mailView) continue; + mailView->GetMailViewName(getter_Copies(mailViewName)); + mFilterList->CreateFilter(mailViewName, getter_AddRefs(newMailFilter)); + if (!newMailFilter) continue; - nsCOMPtr searchTerms; - mailView->GetSearchTerms(getter_AddRefs(searchTerms)); - newMailFilter->SetSearchTerms(searchTerms); - mFilterList->InsertFilterAt(index, newMailFilter); + nsCOMPtr searchTerms; + mailView->GetSearchTerms(getter_AddRefs(searchTerms)); + newMailFilter->SetSearchTerms(searchTerms); + mFilterList->InsertFilterAt(index, newMailFilter); } return NS_OK; } -nsresult nsMsgMailViewList::LoadMailViews() -{ - nsCOMPtr file; - nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file)); +nsresult nsMsgMailViewList::LoadMailViews() { + nsCOMPtr file; + nsresult rv = + NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = file->AppendNative(nsDependentCString("mailViews.dat")); + + // if the file doesn't exist, we should try to get it from the defaults + // directory and copy it over + bool exists = false; + file->Exists(&exists); + if (!exists) { + nsCOMPtr mailSession = + do_GetService(NS_MSGMAILSESSION_CONTRACTID, &rv); NS_ENSURE_SUCCESS(rv, rv); + nsCOMPtr defaultMessagesFile; + nsCOMPtr profileDir; + rv = mailSession->GetDataFilesDir("messenger", + getter_AddRefs(defaultMessagesFile)); + rv = defaultMessagesFile->AppendNative(nsDependentCString("mailViews.dat")); - rv = file->AppendNative(nsDependentCString("mailViews.dat")); + // get the profile directory + rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, + getter_AddRefs(profileDir)); - // if the file doesn't exist, we should try to get it from the defaults directory and copy it over - bool exists = false; - file->Exists(&exists); - if (!exists) - { - nsCOMPtr mailSession = do_GetService(NS_MSGMAILSESSION_CONTRACTID, &rv); - NS_ENSURE_SUCCESS(rv, rv); - nsCOMPtr defaultMessagesFile; - nsCOMPtr profileDir; - rv = mailSession->GetDataFilesDir("messenger", getter_AddRefs(defaultMessagesFile)); - rv = defaultMessagesFile->AppendNative(nsDependentCString("mailViews.dat")); + // now copy the file over to the profile directory + defaultMessagesFile->CopyToNative(profileDir, EmptyCString()); + } + // this is kind of a hack but I think it will be an effective hack. The filter + // service already knows how to take a nsIFile and parse the contents into + // filters which are very similar to mail views. Instead of re-writing all of + // that dirty parsing code, let's just re-use it then convert the results into + // a data strcuture we wish to give to our consumers. - // get the profile directory - rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(profileDir)); + nsCOMPtr filterService = + do_GetService(NS_MSGFILTERSERVICE_CONTRACTID, &rv); + nsCOMPtr mfilterList; - // now copy the file over to the profile directory - defaultMessagesFile->CopyToNative(profileDir, EmptyCString()); - } - // this is kind of a hack but I think it will be an effective hack. The filter service already knows how to - // take a nsIFile and parse the contents into filters which are very similar to mail views. Instead of - // re-writing all of that dirty parsing code, let's just re-use it then convert the results into a data strcuture - // we wish to give to our consumers. + rv = filterService->OpenFilterList(file, nullptr, nullptr, + getter_AddRefs(mFilterList)); + NS_ENSURE_SUCCESS(rv, rv); - nsCOMPtr filterService = do_GetService(NS_MSGFILTERSERVICE_CONTRACTID, &rv); - nsCOMPtr mfilterList; - - rv = filterService->OpenFilterList(file, nullptr, nullptr, getter_AddRefs(mFilterList)); - NS_ENSURE_SUCCESS(rv, rv); - - return ConvertFilterListToMailViews(); + return ConvertFilterListToMailViews(); } /** * Converts the filter list into our mail view objects, * stripping out just the info we need. */ -nsresult nsMsgMailViewList::ConvertFilterListToMailViews() -{ - nsresult rv = NS_OK; - m_mailViews.Clear(); +nsresult nsMsgMailViewList::ConvertFilterListToMailViews() { + nsresult rv = NS_OK; + m_mailViews.Clear(); - // iterate over each filter in the list - uint32_t numFilters = 0; - mFilterList->GetFilterCount(&numFilters); - for (uint32_t index = 0; index < numFilters; index++) - { - nsCOMPtr msgFilter; - rv = mFilterList->GetFilterAt(index, getter_AddRefs(msgFilter)); - if (NS_FAILED(rv) || !msgFilter) - continue; + // iterate over each filter in the list + uint32_t numFilters = 0; + mFilterList->GetFilterCount(&numFilters); + for (uint32_t index = 0; index < numFilters; index++) { + nsCOMPtr msgFilter; + rv = mFilterList->GetFilterAt(index, getter_AddRefs(msgFilter)); + if (NS_FAILED(rv) || !msgFilter) continue; - // create a new nsIMsgMailView for this item - nsCOMPtr newMailView; - rv = CreateMailView(getter_AddRefs(newMailView)); - NS_ENSURE_SUCCESS(rv, rv); + // create a new nsIMsgMailView for this item + nsCOMPtr newMailView; + rv = CreateMailView(getter_AddRefs(newMailView)); + NS_ENSURE_SUCCESS(rv, rv); - nsString filterName; - msgFilter->GetFilterName(filterName); - newMailView->SetMailViewName(filterName.get()); + nsString filterName; + msgFilter->GetFilterName(filterName); + newMailView->SetMailViewName(filterName.get()); - nsCOMPtr filterSearchTerms; - rv = msgFilter->GetSearchTerms(getter_AddRefs(filterSearchTerms)); - NS_ENSURE_SUCCESS(rv, rv); - rv = newMailView->SetSearchTerms(filterSearchTerms); - NS_ENSURE_SUCCESS(rv, rv); + nsCOMPtr filterSearchTerms; + rv = msgFilter->GetSearchTerms(getter_AddRefs(filterSearchTerms)); + NS_ENSURE_SUCCESS(rv, rv); + rv = newMailView->SetSearchTerms(filterSearchTerms); + NS_ENSURE_SUCCESS(rv, rv); - // now append this new mail view to our global list view - m_mailViews.AppendElement(newMailView); - } + // now append this new mail view to our global list view + m_mailViews.AppendElement(newMailView); + } - return rv; + return rv; } diff --git a/mailnews/extensions/mailviews/src/nsMsgMailViewList.h b/mailnews/extensions/mailviews/src/nsMsgMailViewList.h index 9c81926bc1..cbc4a6051f 100644 --- a/mailnews/extensions/mailviews/src/nsMsgMailViewList.h +++ b/mailnews/extensions/mailviews/src/nsMsgMailViewList.h @@ -3,7 +3,6 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - #ifndef _nsMsgMailViewList_H_ #define _nsMsgMailViewList_H_ @@ -17,38 +16,37 @@ #include "nsIMsgFilterList.h" // a mail View is just a name and an array of search terms -class nsMsgMailView : public nsIMsgMailView -{ -public: +class nsMsgMailView : public nsIMsgMailView { + public: NS_DECL_ISUPPORTS NS_DECL_NSIMSGMAILVIEW nsMsgMailView(); -protected: + protected: virtual ~nsMsgMailView(); nsString mName; nsCOMPtr mBundle; nsCOMPtr mViewSearchTerms; }; - -class nsMsgMailViewList : public nsIMsgMailViewList -{ -public: +class nsMsgMailViewList : public nsIMsgMailViewList { + public: NS_DECL_ISUPPORTS NS_DECL_NSIMSGMAILVIEWLIST nsMsgMailViewList(); -protected: + protected: virtual ~nsMsgMailViewList(); - nsresult LoadMailViews(); // reads in user defined mail views from our default file + nsresult + LoadMailViews(); // reads in user defined mail views from our default file nsresult ConvertFilterListToMailViews(); nsresult ConvertMailViewListToFilterList(); nsCOMArray m_mailViews; - nsCOMPtr mFilterList; // our internal filter list representation + nsCOMPtr + mFilterList; // our internal filter list representation }; #endif diff --git a/mailnews/extensions/mailviews/src/nsMsgMailViewsCID.h b/mailnews/extensions/mailviews/src/nsMsgMailViewsCID.h index 9487e5f4c9..47345807ea 100644 --- a/mailnews/extensions/mailviews/src/nsMsgMailViewsCID.h +++ b/mailnews/extensions/mailviews/src/nsMsgMailViewsCID.h @@ -6,12 +6,13 @@ #ifndef nsMsgMailViewsCID_h__ #define nsMsgMailViewsCID_h__ -#define NS_MSGMAILVIEWLIST_CONTRACTID \ - "@mozilla.org/messenger/mailviewlist;1" +#define NS_MSGMAILVIEWLIST_CONTRACTID "@mozilla.org/messenger/mailviewlist;1" -#define NS_MSGMAILVIEWLIST_CID \ -{ /* A0258267-44FD-4886-A858-8192615178EC */ \ - 0xa0258267, 0x44fd, 0x4886, \ - { 0xa8, 0x58, 0x81, 0x92, 0x61, 0x51, 0x78, 0xec }} +#define NS_MSGMAILVIEWLIST_CID \ + { /* A0258267-44FD-4886-A858-8192615178EC */ \ + 0xa0258267, 0x44fd, 0x4886, { \ + 0xa8, 0x58, 0x81, 0x92, 0x61, 0x51, 0x78, 0xec \ + } \ + } #endif /* nsMsgMailViewsCID_h__*/ diff --git a/mailnews/extensions/mdn/src/nsMsgMdnCID.h b/mailnews/extensions/mdn/src/nsMsgMdnCID.h index 07cf5b9776..f800813522 100644 --- a/mailnews/extensions/mdn/src/nsMsgMdnCID.h +++ b/mailnews/extensions/mdn/src/nsMsgMdnCID.h @@ -6,11 +6,12 @@ #ifndef nsMsgMdnCID_h__ #define nsMsgMdnCID_h__ -#define NS_MSGMDNGENERATOR_CONTRACTID \ - "@mozilla.org/messenger-mdn/generator;1" -#define NS_MSGMDNGENERATOR_CID \ -{ /* ec917b13-8f73-4d4d-9146-d7f7aafe9076 */ \ - 0xec917b13, 0x8f73, 0x4d4d, \ - { 0x91, 0x46, 0xd7, 0xf7, 0xaa, 0xfe, 0x90, 0x76 }} +#define NS_MSGMDNGENERATOR_CONTRACTID "@mozilla.org/messenger-mdn/generator;1" +#define NS_MSGMDNGENERATOR_CID \ + { /* ec917b13-8f73-4d4d-9146-d7f7aafe9076 */ \ + 0xec917b13, 0x8f73, 0x4d4d, { \ + 0x91, 0x46, 0xd7, 0xf7, 0xaa, 0xfe, 0x90, 0x76 \ + } \ + } #endif /* nsMsgMdnCID_h__ */ diff --git a/mailnews/extensions/mdn/src/nsMsgMdnGenerator.cpp b/mailnews/extensions/mdn/src/nsMsgMdnGenerator.cpp index cc118c92c6..b47af6357b 100644 --- a/mailnews/extensions/mdn/src/nsMsgMdnGenerator.cpp +++ b/mailnews/extensions/mdn/src/nsMsgMdnGenerator.cpp @@ -37,129 +37,122 @@ using namespace mozilla::mailnews; -#define MDN_NOT_IN_TO_CC ((int) 0x0001) -#define MDN_OUTSIDE_DOMAIN ((int) 0x0002) +#define MDN_NOT_IN_TO_CC ((int)0x0001) +#define MDN_OUTSIDE_DOMAIN ((int)0x0002) -#define HEADER_RETURN_PATH "Return-Path" -#define HEADER_DISPOSITION_NOTIFICATION_TO "Disposition-Notification-To" -#define HEADER_APPARENTLY_TO "Apparently-To" -#define HEADER_ORIGINAL_RECIPIENT "Original-Recipient" -#define HEADER_REPORTING_UA "Reporting-UA" -#define HEADER_MDN_GATEWAY "MDN-Gateway" -#define HEADER_FINAL_RECIPIENT "Final-Recipient" -#define HEADER_DISPOSITION "Disposition" -#define HEADER_ORIGINAL_MESSAGE_ID "Original-Message-ID" -#define HEADER_FAILURE "Failure" -#define HEADER_ERROR "Error" -#define HEADER_WARNING "Warning" -#define HEADER_RETURN_RECEIPT_TO "Return-Receipt-To" -#define HEADER_X_ACCEPT_LANGUAGE "X-Accept-Language" +#define HEADER_RETURN_PATH "Return-Path" +#define HEADER_DISPOSITION_NOTIFICATION_TO "Disposition-Notification-To" +#define HEADER_APPARENTLY_TO "Apparently-To" +#define HEADER_ORIGINAL_RECIPIENT "Original-Recipient" +#define HEADER_REPORTING_UA "Reporting-UA" +#define HEADER_MDN_GATEWAY "MDN-Gateway" +#define HEADER_FINAL_RECIPIENT "Final-Recipient" +#define HEADER_DISPOSITION "Disposition" +#define HEADER_ORIGINAL_MESSAGE_ID "Original-Message-ID" +#define HEADER_FAILURE "Failure" +#define HEADER_ERROR "Error" +#define HEADER_WARNING "Warning" +#define HEADER_RETURN_RECEIPT_TO "Return-Receipt-To" +#define HEADER_X_ACCEPT_LANGUAGE "X-Accept-Language" -#define PUSH_N_FREE_STRING(p) \ - do { if (p) { rv = WriteString(p); PR_smprintf_free(p); p=0; \ - if (NS_FAILED(rv)) return rv; } \ - else { return NS_ERROR_OUT_OF_MEMORY; } } while (0) +#define PUSH_N_FREE_STRING(p) \ + do { \ + if (p) { \ + rv = WriteString(p); \ + PR_smprintf_free(p); \ + p = 0; \ + if (NS_FAILED(rv)) return rv; \ + } else { \ + return NS_ERROR_OUT_OF_MEMORY; \ + } \ + } while (0) // String bundle for mdn. Class static. #define MDN_STRINGBUNDLE_URL "chrome://messenger/locale/msgmdn.properties" #if defined(DEBUG_jefft) -#define DEBUG_MDN(s) printf("%s\n", s) +# define DEBUG_MDN(s) printf("%s\n", s) #else -#define DEBUG_MDN(s) +# define DEBUG_MDN(s) #endif // machine parsible string; should not be localized char DispositionTypes[7][16] = { - "displayed", - "dispatched", - "processed", - "deleted", - "denied", - "failed", - "" -}; + "displayed", "dispatched", "processed", "deleted", "denied", "failed", ""}; NS_IMPL_ISUPPORTS(nsMsgMdnGenerator, nsIMsgMdnGenerator, nsIUrlListener) -nsMsgMdnGenerator::nsMsgMdnGenerator() -{ - m_disposeType = eDisplayed; - m_outputStream = nullptr; - m_reallySendMdn = false; - m_autoSend = false; - m_autoAction = false; - m_mdnEnabled = false; - m_notInToCcOp = eNeverSendOp; - m_outsideDomainOp = eNeverSendOp; - m_otherOp = eNeverSendOp; +nsMsgMdnGenerator::nsMsgMdnGenerator() { + m_disposeType = eDisplayed; + m_outputStream = nullptr; + m_reallySendMdn = false; + m_autoSend = false; + m_autoAction = false; + m_mdnEnabled = false; + m_notInToCcOp = eNeverSendOp; + m_outsideDomainOp = eNeverSendOp; + m_otherOp = eNeverSendOp; } -nsMsgMdnGenerator::~nsMsgMdnGenerator() -{ -} +nsMsgMdnGenerator::~nsMsgMdnGenerator() {} nsresult nsMsgMdnGenerator::FormatStringFromName(const char *aName, const char16_t *aString, - nsAString& aResultString) -{ - DEBUG_MDN("nsMsgMdnGenerator::FormatStringFromName"); + nsAString &aResultString) { + DEBUG_MDN("nsMsgMdnGenerator::FormatStringFromName"); - nsCOMPtr bundleService = + nsCOMPtr bundleService = mozilla::services::GetStringBundleService(); - NS_ENSURE_TRUE(bundleService, NS_ERROR_UNEXPECTED); + NS_ENSURE_TRUE(bundleService, NS_ERROR_UNEXPECTED); - nsCOMPtr bundle; - nsresult rv = bundleService->CreateBundle(MDN_STRINGBUNDLE_URL, - getter_AddRefs(bundle)); - NS_ENSURE_SUCCESS(rv,rv); + nsCOMPtr bundle; + nsresult rv = + bundleService->CreateBundle(MDN_STRINGBUNDLE_URL, getter_AddRefs(bundle)); + NS_ENSURE_SUCCESS(rv, rv); - const char16_t *formatStrings[1] = { aString }; - rv = bundle->FormatStringFromName(aName, - formatStrings, 1, aResultString); - NS_ENSURE_SUCCESS(rv,rv); - return rv; + const char16_t *formatStrings[1] = {aString}; + rv = bundle->FormatStringFromName(aName, formatStrings, 1, aResultString); + NS_ENSURE_SUCCESS(rv, rv); + return rv; } nsresult nsMsgMdnGenerator::GetStringFromName(const char *aName, - nsAString& aResultString) -{ - DEBUG_MDN("nsMsgMdnGenerator::GetStringFromName"); + nsAString &aResultString) { + DEBUG_MDN("nsMsgMdnGenerator::GetStringFromName"); - nsCOMPtr bundleService = + nsCOMPtr bundleService = mozilla::services::GetStringBundleService(); - NS_ENSURE_TRUE(bundleService, NS_ERROR_UNEXPECTED); + NS_ENSURE_TRUE(bundleService, NS_ERROR_UNEXPECTED); - nsCOMPtr bundle; - nsresult rv = bundleService->CreateBundle(MDN_STRINGBUNDLE_URL, - getter_AddRefs(bundle)); - NS_ENSURE_SUCCESS(rv,rv); + nsCOMPtr bundle; + nsresult rv = + bundleService->CreateBundle(MDN_STRINGBUNDLE_URL, getter_AddRefs(bundle)); + NS_ENSURE_SUCCESS(rv, rv); - rv = bundle->GetStringFromName(aName, aResultString); - NS_ENSURE_SUCCESS(rv,rv); - return rv; + rv = bundle->GetStringFromName(aName, aResultString); + NS_ENSURE_SUCCESS(rv, rv); + return rv; } nsresult nsMsgMdnGenerator::StoreMDNSentFlag(nsIMsgFolder *folder, - nsMsgKey key) -{ - DEBUG_MDN("nsMsgMdnGenerator::StoreMDNSentFlag"); + nsMsgKey key) { + DEBUG_MDN("nsMsgMdnGenerator::StoreMDNSentFlag"); - nsCOMPtr msgDB; - nsresult rv = folder->GetMsgDatabase(getter_AddRefs(msgDB)); - NS_ENSURE_SUCCESS(rv, rv); - rv = msgDB->MarkMDNSent(key, true, nullptr); + nsCOMPtr msgDB; + nsresult rv = folder->GetMsgDatabase(getter_AddRefs(msgDB)); + NS_ENSURE_SUCCESS(rv, rv); + rv = msgDB->MarkMDNSent(key, true, nullptr); - nsCOMPtr imapFolder = do_QueryInterface(folder); - // Store the $MDNSent flag if the folder is an Imap Mail Folder - if (imapFolder) - return imapFolder->StoreImapFlags(kImapMsgMDNSentFlag, true, &key, 1, nullptr); - return rv; + nsCOMPtr imapFolder = do_QueryInterface(folder); + // Store the $MDNSent flag if the folder is an Imap Mail Folder + if (imapFolder) + return imapFolder->StoreImapFlags(kImapMsgMDNSentFlag, true, &key, 1, + nullptr); + return rv; } nsresult nsMsgMdnGenerator::ClearMDNNeededFlag(nsIMsgFolder *folder, - nsMsgKey key) -{ + nsMsgKey key) { DEBUG_MDN("nsMsgMdnGenerator::ClearMDNNeededFlag"); nsCOMPtr msgDB; @@ -168,883 +161,758 @@ nsresult nsMsgMdnGenerator::ClearMDNNeededFlag(nsIMsgFolder *folder, return msgDB->MarkMDNNeeded(key, false, nullptr); } -bool nsMsgMdnGenerator::ProcessSendMode() -{ - DEBUG_MDN("nsMsgMdnGenerator::ProcessSendMode"); - int32_t miscState = 0; +bool nsMsgMdnGenerator::ProcessSendMode() { + DEBUG_MDN("nsMsgMdnGenerator::ProcessSendMode"); + int32_t miscState = 0; - if (m_identity) - { - m_identity->GetEmail(m_email); - if (m_email.IsEmpty()) - return m_reallySendMdn; + if (m_identity) { + m_identity->GetEmail(m_email); + if (m_email.IsEmpty()) return m_reallySendMdn; - const char *accountDomain = strchr(m_email.get(), '@'); - if (!accountDomain) - return m_reallySendMdn; + const char *accountDomain = strchr(m_email.get(), '@'); + if (!accountDomain) return m_reallySendMdn; - if (MailAddrMatch(m_email.get(), m_dntRrt.get())) // return address is self, don't send - return false; + if (MailAddrMatch(m_email.get(), + m_dntRrt.get())) // return address is self, don't send + return false; - // *** fix me see Bug 132504 for more information - // *** what if the message has been filtered to different account - if (!PL_strcasestr(m_dntRrt.get(), accountDomain)) - miscState |= MDN_OUTSIDE_DOMAIN; - if (NotInToOrCc()) - miscState |= MDN_NOT_IN_TO_CC; - m_reallySendMdn = true; - // ********* - // How are we gona deal with the auto forwarding issues? Some server - // didn't bother to add addition header or modify existing header to - // the message when forwarding. They simply copy the exact same - // message to another user's mailbox. Some change To: to - // Apparently-To: - // Unfortunately, there is nothing we can do. It's out of our control. - // ********* - // starting from lowest denominator to highest - if (!miscState) - { // under normal situation: recipent is in to and cc list, - // and the sender is from the same domain - switch (m_otherOp) - { - default: - case eNeverSendOp: - m_reallySendMdn = false; - break; - case eAutoSendOp: - m_autoSend = true; - break; - case eAskMeOp: - m_autoSend = false; - break; - case eDeniedOp: - m_autoSend = true; - m_disposeType = eDenied; - break; - } - } - else if (miscState == (MDN_OUTSIDE_DOMAIN | MDN_NOT_IN_TO_CC)) - { - if (m_outsideDomainOp != m_notInToCcOp) - { - m_autoSend = false; // ambiguous; always ask user - } - else - { - switch (m_outsideDomainOp) - { - default: - case eNeverSendOp: - m_reallySendMdn = false; - break; - case eAutoSendOp: - m_autoSend = true; - break; - case eAskMeOp: - m_autoSend = false; - break; - } - } - } - else if (miscState & MDN_OUTSIDE_DOMAIN) - { - switch (m_outsideDomainOp) - { - default: - case eNeverSendOp: - m_reallySendMdn = false; - break; - case eAutoSendOp: - m_autoSend = true; - break; - case eAskMeOp: - m_autoSend = false; - break; - } - } - else if (miscState & MDN_NOT_IN_TO_CC) - { - switch (m_notInToCcOp) - { - default: - case eNeverSendOp: - m_reallySendMdn = false; - break; - case eAutoSendOp: - m_autoSend = true; - break; - case eAskMeOp: - m_autoSend = false; - break; - } + // *** fix me see Bug 132504 for more information + // *** what if the message has been filtered to different account + if (!PL_strcasestr(m_dntRrt.get(), accountDomain)) + miscState |= MDN_OUTSIDE_DOMAIN; + if (NotInToOrCc()) miscState |= MDN_NOT_IN_TO_CC; + m_reallySendMdn = true; + // ********* + // How are we gona deal with the auto forwarding issues? Some server + // didn't bother to add addition header or modify existing header to + // the message when forwarding. They simply copy the exact same + // message to another user's mailbox. Some change To: to + // Apparently-To: + // Unfortunately, there is nothing we can do. It's out of our control. + // ********* + // starting from lowest denominator to highest + if (!miscState) { // under normal situation: recipent is in to and cc list, + // and the sender is from the same domain + switch (m_otherOp) { + default: + case eNeverSendOp: + m_reallySendMdn = false; + break; + case eAutoSendOp: + m_autoSend = true; + break; + case eAskMeOp: + m_autoSend = false; + break; + case eDeniedOp: + m_autoSend = true; + m_disposeType = eDenied; + break; + } + } else if (miscState == (MDN_OUTSIDE_DOMAIN | MDN_NOT_IN_TO_CC)) { + if (m_outsideDomainOp != m_notInToCcOp) { + m_autoSend = false; // ambiguous; always ask user + } else { + switch (m_outsideDomainOp) { + default: + case eNeverSendOp: + m_reallySendMdn = false; + break; + case eAutoSendOp: + m_autoSend = true; + break; + case eAskMeOp: + m_autoSend = false; + break; } + } + } else if (miscState & MDN_OUTSIDE_DOMAIN) { + switch (m_outsideDomainOp) { + default: + case eNeverSendOp: + m_reallySendMdn = false; + break; + case eAutoSendOp: + m_autoSend = true; + break; + case eAskMeOp: + m_autoSend = false; + break; + } + } else if (miscState & MDN_NOT_IN_TO_CC) { + switch (m_notInToCcOp) { + default: + case eNeverSendOp: + m_reallySendMdn = false; + break; + case eAutoSendOp: + m_autoSend = true; + break; + case eAskMeOp: + m_autoSend = false; + break; + } } - return m_reallySendMdn; + } + return m_reallySendMdn; } -bool nsMsgMdnGenerator::MailAddrMatch(const char *addr1, const char *addr2) -{ - // Comparing two email addresses returns true if matched; local/account - // part comparison is case sensitive; domain part comparison is case - // insensitive - DEBUG_MDN("nsMsgMdnGenerator::MailAddrMatch"); - bool isMatched = true; - const char *atSign1 = nullptr, *atSign2 = nullptr; - const char *lt = nullptr, *local1 = nullptr, *local2 = nullptr; - const char *end1 = nullptr, *end2 = nullptr; +bool nsMsgMdnGenerator::MailAddrMatch(const char *addr1, const char *addr2) { + // Comparing two email addresses returns true if matched; local/account + // part comparison is case sensitive; domain part comparison is case + // insensitive + DEBUG_MDN("nsMsgMdnGenerator::MailAddrMatch"); + bool isMatched = true; + const char *atSign1 = nullptr, *atSign2 = nullptr; + const char *lt = nullptr, *local1 = nullptr, *local2 = nullptr; + const char *end1 = nullptr, *end2 = nullptr; - if (!addr1 || !addr2) - return false; + if (!addr1 || !addr2) return false; - lt = strchr(addr1, '<'); - local1 = !lt ? addr1 : lt+1; - lt = strchr(addr2, '<'); - local2 = !lt ? addr2 : lt+1; - end1 = strchr(local1, '>'); - if (!end1) - end1 = addr1 + strlen(addr1); - end2 = strchr(local2, '>'); - if (!end2) - end2 = addr2 + strlen(addr2); - atSign1 = strchr(local1, '@'); - atSign2 = strchr(local2, '@'); - if (!atSign1 || !atSign2 // ill formed addr spec - || (atSign1 - local1) != (atSign2 - local2)) - isMatched = false; - else if (strncmp(local1, local2, (atSign1-local1))) // case sensitive - // compare for local part - isMatched = false; - else if ((end1 - atSign1) != (end2 - atSign2) || - PL_strncasecmp(atSign1, atSign2, (end1-atSign1))) // case - // insensitive compare for domain part - isMatched = false; - return isMatched; + lt = strchr(addr1, '<'); + local1 = !lt ? addr1 : lt + 1; + lt = strchr(addr2, '<'); + local2 = !lt ? addr2 : lt + 1; + end1 = strchr(local1, '>'); + if (!end1) end1 = addr1 + strlen(addr1); + end2 = strchr(local2, '>'); + if (!end2) end2 = addr2 + strlen(addr2); + atSign1 = strchr(local1, '@'); + atSign2 = strchr(local2, '@'); + if (!atSign1 || !atSign2 // ill formed addr spec + || (atSign1 - local1) != (atSign2 - local2)) + isMatched = false; + else if (strncmp(local1, local2, (atSign1 - local1))) // case sensitive + // compare for local part + isMatched = false; + else if ((end1 - atSign1) != (end2 - atSign2) || + PL_strncasecmp(atSign1, atSign2, (end1 - atSign1))) // case + // insensitive compare for domain part + isMatched = false; + return isMatched; } -bool nsMsgMdnGenerator::NotInToOrCc() -{ - DEBUG_MDN("nsMsgMdnGenerator::NotInToOrCc"); - nsCString reply_to; - nsCString to; - nsCString cc; +bool nsMsgMdnGenerator::NotInToOrCc() { + DEBUG_MDN("nsMsgMdnGenerator::NotInToOrCc"); + nsCString reply_to; + nsCString to; + nsCString cc; - m_identity->GetReplyTo(reply_to); - m_headers->ExtractHeader(HEADER_TO, true, to); - m_headers->ExtractHeader(HEADER_CC, true, cc); + m_identity->GetReplyTo(reply_to); + m_headers->ExtractHeader(HEADER_TO, true, to); + m_headers->ExtractHeader(HEADER_CC, true, cc); // start with a simple check if ((!to.IsEmpty() && PL_strcasestr(to.get(), m_email.get())) || (!cc.IsEmpty() && PL_strcasestr(cc.get(), m_email.get()))) { - return false; + return false; } - if ((!reply_to.IsEmpty() && !to.IsEmpty() && PL_strcasestr(to.get(), reply_to.get())) || - (!reply_to.IsEmpty() && !cc.IsEmpty() && PL_strcasestr(cc.get(), reply_to.get()))) { - return false; + if ((!reply_to.IsEmpty() && !to.IsEmpty() && + PL_strcasestr(to.get(), reply_to.get())) || + (!reply_to.IsEmpty() && !cc.IsEmpty() && + PL_strcasestr(cc.get(), reply_to.get()))) { + return false; } return true; } -bool nsMsgMdnGenerator::ValidateReturnPath() -{ - DEBUG_MDN("nsMsgMdnGenerator::ValidateReturnPath"); - // ValidateReturnPath applies to Automatic Send Mode only. If we were not - // in auto send mode we simply by passing the check - if (!m_autoSend) - return m_reallySendMdn; +bool nsMsgMdnGenerator::ValidateReturnPath() { + DEBUG_MDN("nsMsgMdnGenerator::ValidateReturnPath"); + // ValidateReturnPath applies to Automatic Send Mode only. If we were not + // in auto send mode we simply by passing the check + if (!m_autoSend) return m_reallySendMdn; - nsCString returnPath; - m_headers->ExtractHeader(HEADER_RETURN_PATH, false, returnPath); - if (returnPath.IsEmpty()) - { - m_autoSend = false; - return m_reallySendMdn; - } - m_autoSend = MailAddrMatch(returnPath.get(), m_dntRrt.get()); + nsCString returnPath; + m_headers->ExtractHeader(HEADER_RETURN_PATH, false, returnPath); + if (returnPath.IsEmpty()) { + m_autoSend = false; return m_reallySendMdn; + } + m_autoSend = MailAddrMatch(returnPath.get(), m_dntRrt.get()); + return m_reallySendMdn; } -nsresult nsMsgMdnGenerator::CreateMdnMsg() -{ - DEBUG_MDN("nsMsgMdnGenerator::CreateMdnMsg"); - nsresult rv; +nsresult nsMsgMdnGenerator::CreateMdnMsg() { + DEBUG_MDN("nsMsgMdnGenerator::CreateMdnMsg"); + nsresult rv; - nsCOMPtr tmpFile; - rv = GetSpecialDirectoryWithFileName(NS_OS_TEMP_DIR, - "mdnmsg", - getter_AddRefs(m_file)); - NS_ENSURE_SUCCESS(rv, rv); + nsCOMPtr tmpFile; + rv = GetSpecialDirectoryWithFileName(NS_OS_TEMP_DIR, "mdnmsg", + getter_AddRefs(m_file)); + NS_ENSURE_SUCCESS(rv, rv); - rv = m_file->CreateUnique(nsIFile::NORMAL_FILE_TYPE, 00600); - NS_ENSURE_SUCCESS(rv, rv); - rv = MsgNewBufferedFileOutputStream(getter_AddRefs(m_outputStream), - m_file, - PR_CREATE_FILE | PR_WRONLY | PR_TRUNCATE, - 0664); - NS_ASSERTION(NS_SUCCEEDED(rv),"creating mdn: failed to output stream"); - if (NS_FAILED(rv)) - return NS_OK; + rv = m_file->CreateUnique(nsIFile::NORMAL_FILE_TYPE, 00600); + NS_ENSURE_SUCCESS(rv, rv); + rv = MsgNewBufferedFileOutputStream(getter_AddRefs(m_outputStream), m_file, + PR_CREATE_FILE | PR_WRONLY | PR_TRUNCATE, + 0664); + NS_ASSERTION(NS_SUCCEEDED(rv), "creating mdn: failed to output stream"); + if (NS_FAILED(rv)) return NS_OK; - rv = CreateFirstPart(); - if (NS_SUCCEEDED(rv)) - { - rv = CreateSecondPart(); - if (NS_SUCCEEDED(rv)) - rv = CreateThirdPart(); - } + rv = CreateFirstPart(); + if (NS_SUCCEEDED(rv)) { + rv = CreateSecondPart(); + if (NS_SUCCEEDED(rv)) rv = CreateThirdPart(); + } - if (m_outputStream) - { - m_outputStream->Flush(); - m_outputStream->Close(); - } - if (NS_FAILED(rv)) - m_file->Remove(false); - else - rv = SendMdnMsg(); + if (m_outputStream) { + m_outputStream->Flush(); + m_outputStream->Close(); + } + if (NS_FAILED(rv)) + m_file->Remove(false); + else + rv = SendMdnMsg(); - return NS_OK; + return NS_OK; } -nsresult nsMsgMdnGenerator::CreateFirstPart() -{ - DEBUG_MDN("nsMsgMdnGenerator::CreateFirstPart"); - char *convbuf = nullptr, *tmpBuffer = nullptr; - char *parm = nullptr; - nsString firstPart1; - nsString firstPart2; - nsresult rv = NS_OK; - nsCOMPtr compUtils; +nsresult nsMsgMdnGenerator::CreateFirstPart() { + DEBUG_MDN("nsMsgMdnGenerator::CreateFirstPart"); + char *convbuf = nullptr, *tmpBuffer = nullptr; + char *parm = nullptr; + nsString firstPart1; + nsString firstPart2; + nsresult rv = NS_OK; + nsCOMPtr compUtils; - if (m_mimeSeparator.IsEmpty()) - { - compUtils = do_GetService(NS_MSGCOMPUTILS_CONTRACTID, &rv); - NS_ENSURE_SUCCESS(rv, rv); - rv = compUtils->MimeMakeSeparator("mdn", getter_Copies(m_mimeSeparator)); - NS_ENSURE_SUCCESS(rv, rv); - } - if (m_mimeSeparator.IsEmpty()) - return NS_ERROR_OUT_OF_MEMORY; + if (m_mimeSeparator.IsEmpty()) { + compUtils = do_GetService(NS_MSGCOMPUTILS_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + rv = compUtils->MimeMakeSeparator("mdn", getter_Copies(m_mimeSeparator)); + NS_ENSURE_SUCCESS(rv, rv); + } + if (m_mimeSeparator.IsEmpty()) return NS_ERROR_OUT_OF_MEMORY; - tmpBuffer = (char *) PR_CALLOC(256); + tmpBuffer = (char *)PR_CALLOC(256); - if (!tmpBuffer) - return NS_ERROR_OUT_OF_MEMORY; + if (!tmpBuffer) return NS_ERROR_OUT_OF_MEMORY; - PRExplodedTime now; - PR_ExplodeTime(PR_Now(), PR_LocalTimeParameters, &now); + PRExplodedTime now; + PR_ExplodeTime(PR_Now(), PR_LocalTimeParameters, &now); - int gmtoffset = (now.tm_params.tp_gmt_offset + now.tm_params.tp_dst_offset) - / 60; + int gmtoffset = + (now.tm_params.tp_gmt_offset + now.tm_params.tp_dst_offset) / 60; /* Use PR_FormatTimeUSEnglish() to format the date in US English format, then figure out what our local GMT offset is, and append it (since PR_FormatTimeUSEnglish() can't do that.) Generate four digit years as per RFC 1123 (superseding RFC 822.) */ - PR_FormatTimeUSEnglish(tmpBuffer, 100, - "Date: %a, %d %b %Y %H:%M:%S ", - &now); + PR_FormatTimeUSEnglish(tmpBuffer, 100, "Date: %a, %d %b %Y %H:%M:%S ", &now); - PR_snprintf(tmpBuffer + strlen(tmpBuffer), 100, - "%c%02d%02d" CRLF, - (gmtoffset >= 0 ? '+' : '-'), - ((gmtoffset >= 0 ? gmtoffset : -gmtoffset) / 60), - ((gmtoffset >= 0 ? gmtoffset : -gmtoffset) % 60)); + PR_snprintf(tmpBuffer + strlen(tmpBuffer), 100, "%c%02d%02d" CRLF, + (gmtoffset >= 0 ? '+' : '-'), + ((gmtoffset >= 0 ? gmtoffset : -gmtoffset) / 60), + ((gmtoffset >= 0 ? gmtoffset : -gmtoffset) % 60)); - rv = WriteString(tmpBuffer); - PR_Free(tmpBuffer); - if (NS_FAILED(rv)) - return rv; + rv = WriteString(tmpBuffer); + PR_Free(tmpBuffer); + if (NS_FAILED(rv)) return rv; - bool conformToStandard = false; - if (compUtils) - compUtils->GetMsgMimeConformToStandard(&conformToStandard); + bool conformToStandard = false; + if (compUtils) compUtils->GetMsgMimeConformToStandard(&conformToStandard); - nsString fullName; - m_identity->GetFullName(fullName); + nsString fullName; + m_identity->GetFullName(fullName); - nsCString fullAddress; - // convert fullName to UTF8 before passing it to MakeMimeAddress - MakeMimeAddress(NS_ConvertUTF16toUTF8(fullName), m_email, fullAddress); + nsCString fullAddress; + // convert fullName to UTF8 before passing it to MakeMimeAddress + MakeMimeAddress(NS_ConvertUTF16toUTF8(fullName), m_email, fullAddress); - convbuf = nsMsgI18NEncodeMimePartIIStr(fullAddress.get(), - true, m_charset.get(), 0, conformToStandard); + convbuf = nsMsgI18NEncodeMimePartIIStr(fullAddress.get(), true, + m_charset.get(), 0, conformToStandard); - parm = PR_smprintf("From: %s" CRLF, convbuf ? convbuf : m_email.get()); + parm = PR_smprintf("From: %s" CRLF, convbuf ? convbuf : m_email.get()); - rv = FormatStringFromName("MsgMdnMsgSentTo", NS_ConvertASCIItoUTF16(m_email).get(), - firstPart1); - if (NS_FAILED(rv)) - return rv; + rv = FormatStringFromName("MsgMdnMsgSentTo", + NS_ConvertASCIItoUTF16(m_email).get(), firstPart1); + if (NS_FAILED(rv)) return rv; - PUSH_N_FREE_STRING (parm); + PUSH_N_FREE_STRING(parm); - PR_Free(convbuf); + PR_Free(convbuf); - if (compUtils) - { - nsCString msgId; - rv = compUtils->MsgGenerateMessageId(m_identity, getter_Copies(msgId)); - tmpBuffer = PR_smprintf("Message-ID: %s" CRLF, msgId.get()); - PUSH_N_FREE_STRING(tmpBuffer); - } + if (compUtils) { + nsCString msgId; + rv = compUtils->MsgGenerateMessageId(m_identity, getter_Copies(msgId)); + tmpBuffer = PR_smprintf("Message-ID: %s" CRLF, msgId.get()); + PUSH_N_FREE_STRING(tmpBuffer); + } - nsString receipt_string; - switch (m_disposeType) - { + nsString receipt_string; + switch (m_disposeType) { case nsIMsgMdnGenerator::eDisplayed: - rv = GetStringFromName( - "MdnDisplayedReceipt", - receipt_string); - break; + rv = GetStringFromName("MdnDisplayedReceipt", receipt_string); + break; case nsIMsgMdnGenerator::eDispatched: - rv = GetStringFromName( - "MdnDispatchedReceipt", - receipt_string); - break; + rv = GetStringFromName("MdnDispatchedReceipt", receipt_string); + break; case nsIMsgMdnGenerator::eProcessed: - rv = GetStringFromName( - "MdnProcessedReceipt", - receipt_string); - break; + rv = GetStringFromName("MdnProcessedReceipt", receipt_string); + break; case nsIMsgMdnGenerator::eDeleted: - rv = GetStringFromName( - "MdnDeletedReceipt", - receipt_string); - break; + rv = GetStringFromName("MdnDeletedReceipt", receipt_string); + break; case nsIMsgMdnGenerator::eDenied: - rv = GetStringFromName( - "MdnDeniedReceipt", - receipt_string); - break; + rv = GetStringFromName("MdnDeniedReceipt", receipt_string); + break; case nsIMsgMdnGenerator::eFailed: - rv = GetStringFromName( - "MdnFailedReceipt", - receipt_string); - break; + rv = GetStringFromName("MdnFailedReceipt", receipt_string); + break; default: - rv = NS_ERROR_INVALID_ARG; - break; - } + rv = NS_ERROR_INVALID_ARG; + break; + } - if (NS_FAILED(rv)) - return rv; + if (NS_FAILED(rv)) return rv; - receipt_string.AppendLiteral(" - "); + receipt_string.AppendLiteral(" - "); - char * encodedReceiptString = nsMsgI18NEncodeMimePartIIStr(NS_ConvertUTF16toUTF8(receipt_string).get(), false, - "UTF-8", 0, conformToStandard); + char *encodedReceiptString = + nsMsgI18NEncodeMimePartIIStr(NS_ConvertUTF16toUTF8(receipt_string).get(), + false, "UTF-8", 0, conformToStandard); - nsCString subject; - m_headers->ExtractHeader(HEADER_SUBJECT, false, subject); - convbuf = nsMsgI18NEncodeMimePartIIStr(subject.Length() ? subject.get() : "[no subject]", - false, m_charset.get(), 0, conformToStandard); - tmpBuffer = PR_smprintf("Subject: %s%s" CRLF, - encodedReceiptString, - (convbuf ? convbuf : (subject.Length() ? subject.get() : - "[no subject]"))); + nsCString subject; + m_headers->ExtractHeader(HEADER_SUBJECT, false, subject); + convbuf = nsMsgI18NEncodeMimePartIIStr( + subject.Length() ? subject.get() : "[no subject]", false, m_charset.get(), + 0, conformToStandard); + tmpBuffer = PR_smprintf( + "Subject: %s%s" CRLF, encodedReceiptString, + (convbuf ? convbuf + : (subject.Length() ? subject.get() : "[no subject]"))); - PUSH_N_FREE_STRING(tmpBuffer); - PR_Free(convbuf); - PR_Free(encodedReceiptString); + PUSH_N_FREE_STRING(tmpBuffer); + PR_Free(convbuf); + PR_Free(encodedReceiptString); - convbuf = nsMsgI18NEncodeMimePartIIStr(m_dntRrt.get(), true, m_charset.get(), 0, conformToStandard); - tmpBuffer = PR_smprintf("To: %s" CRLF, convbuf ? convbuf : - m_dntRrt.get()); - PUSH_N_FREE_STRING(tmpBuffer); + convbuf = nsMsgI18NEncodeMimePartIIStr(m_dntRrt.get(), true, m_charset.get(), + 0, conformToStandard); + tmpBuffer = PR_smprintf("To: %s" CRLF, convbuf ? convbuf : m_dntRrt.get()); + PUSH_N_FREE_STRING(tmpBuffer); - PR_Free(convbuf); + PR_Free(convbuf); // *** This is not in the spec. I am adding this so we could do // threading - m_headers->ExtractHeader(HEADER_MESSAGE_ID, false, m_messageId); + m_headers->ExtractHeader(HEADER_MESSAGE_ID, false, m_messageId); - if (!m_messageId.IsEmpty()) - { - if (*m_messageId.get() == '<') - tmpBuffer = PR_smprintf("References: %s" CRLF, m_messageId.get()); - else - tmpBuffer = PR_smprintf("References: <%s>" CRLF, m_messageId.get()); + if (!m_messageId.IsEmpty()) { + if (*m_messageId.get() == '<') + tmpBuffer = PR_smprintf("References: %s" CRLF, m_messageId.get()); + else + tmpBuffer = PR_smprintf("References: <%s>" CRLF, m_messageId.get()); + PUSH_N_FREE_STRING(tmpBuffer); + } + tmpBuffer = PR_smprintf("%s" CRLF, "MIME-Version: 1.0"); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf( + "Content-Type: multipart/report; \ +report-type=disposition-notification;\r\n\tboundary=\"%s\"" CRLF CRLF, + m_mimeSeparator.get()); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf("--%s" CRLF, m_mimeSeparator.get()); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf("Content-Type: text/plain; charset=UTF-8" CRLF); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = + PR_smprintf("Content-Transfer-Encoding: %s" CRLF CRLF, ENCODING_8BIT); + PUSH_N_FREE_STRING(tmpBuffer); + + if (!firstPart1.IsEmpty()) { + tmpBuffer = + PR_smprintf("%s" CRLF CRLF, NS_ConvertUTF16toUTF8(firstPart1).get()); + PUSH_N_FREE_STRING(tmpBuffer); + } + + switch (m_disposeType) { + case nsIMsgMdnGenerator::eDisplayed: + rv = GetStringFromName("MsgMdnDisplayed", firstPart2); + break; + case nsIMsgMdnGenerator::eDispatched: + rv = GetStringFromName("MsgMdnDispatched", firstPart2); + break; + case nsIMsgMdnGenerator::eProcessed: + rv = GetStringFromName("MsgMdnProcessed", firstPart2); + break; + case nsIMsgMdnGenerator::eDeleted: + rv = GetStringFromName("MsgMdnDeleted", firstPart2); + break; + case nsIMsgMdnGenerator::eDenied: + rv = GetStringFromName("MsgMdnDenied", firstPart2); + break; + case nsIMsgMdnGenerator::eFailed: + rv = GetStringFromName("MsgMdnFailed", firstPart2); + break; + default: + rv = NS_ERROR_INVALID_ARG; + break; + } + + if (NS_FAILED(rv)) return rv; + + if (!firstPart2.IsEmpty()) { + tmpBuffer = + PR_smprintf("%s" CRLF CRLF, NS_ConvertUTF16toUTF8(firstPart2).get()); + PUSH_N_FREE_STRING(tmpBuffer); + } + + return rv; +} + +nsresult nsMsgMdnGenerator::CreateSecondPart() { + DEBUG_MDN("nsMsgMdnGenerator::CreateSecondPart"); + char *tmpBuffer = nullptr; + char *convbuf = nullptr; + nsresult rv = NS_OK; + nsCOMPtr compUtils; + bool conformToStandard = false; + + tmpBuffer = PR_smprintf("--%s" CRLF, m_mimeSeparator.get()); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf("%s" CRLF, + "Content-Type: message/disposition-notification; " + "name=\042MDNPart2.txt\042"); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = PR_smprintf("%s" CRLF, "Content-Disposition: inline"); + PUSH_N_FREE_STRING(tmpBuffer); + + tmpBuffer = + PR_smprintf("Content-Transfer-Encoding: %s" CRLF CRLF, ENCODING_7BIT); + PUSH_N_FREE_STRING(tmpBuffer); + + nsCOMPtr pHTTPHandler = + do_GetService(NS_NETWORK_PROTOCOL_CONTRACTID_PREFIX "http", &rv); + if (NS_SUCCEEDED(rv) && pHTTPHandler) { + nsAutoCString userAgentString; + // Ignore error since we're testing the return value. + mozilla::Unused << pHTTPHandler->GetUserAgent(userAgentString); + + if (!userAgentString.IsEmpty()) { + // Prepend the product name with the dns name according to RFC 3798. + char hostName[256]; + PR_GetSystemInfo(PR_SI_HOSTNAME_UNTRUNCATED, hostName, sizeof hostName); + if ((hostName[0] != '\0') && (strchr(hostName, '.') != NULL)) { + userAgentString.InsertLiteral("; ", 0); + userAgentString.Insert(nsDependentCString(hostName), 0); + } + + tmpBuffer = PR_smprintf("Reporting-UA: %s" CRLF, userAgentString.get()); PUSH_N_FREE_STRING(tmpBuffer); } - tmpBuffer = PR_smprintf("%s" CRLF, "MIME-Version: 1.0"); + } + + nsCString originalRecipient; + m_headers->ExtractHeader(HEADER_ORIGINAL_RECIPIENT, false, originalRecipient); + + if (!originalRecipient.IsEmpty()) { + tmpBuffer = + PR_smprintf("Original-Recipient: %s" CRLF, originalRecipient.get()); PUSH_N_FREE_STRING(tmpBuffer); + } - tmpBuffer = PR_smprintf("Content-Type: multipart/report; \ -report-type=disposition-notification;\r\n\tboundary=\"%s\"" CRLF CRLF, - m_mimeSeparator.get()); - PUSH_N_FREE_STRING(tmpBuffer); + compUtils = do_GetService(NS_MSGCOMPUTILS_CONTRACTID, &rv); + if (compUtils) compUtils->GetMsgMimeConformToStandard(&conformToStandard); - tmpBuffer = PR_smprintf("--%s" CRLF, m_mimeSeparator.get()); - PUSH_N_FREE_STRING(tmpBuffer); + convbuf = nsMsgI18NEncodeMimePartIIStr(m_email.get(), true, m_charset.get(), + 0, conformToStandard); + tmpBuffer = PR_smprintf("Final-Recipient: rfc822;%s" CRLF, + convbuf ? convbuf : m_email.get()); + PUSH_N_FREE_STRING(tmpBuffer); - tmpBuffer = PR_smprintf("Content-Type: text/plain; charset=UTF-8" CRLF); - PUSH_N_FREE_STRING(tmpBuffer); + PR_Free(convbuf); - tmpBuffer = PR_smprintf("Content-Transfer-Encoding: %s" CRLF CRLF, - ENCODING_8BIT); - PUSH_N_FREE_STRING(tmpBuffer); + if (*m_messageId.get() == '<') + tmpBuffer = PR_smprintf("Original-Message-ID: %s" CRLF, m_messageId.get()); + else + tmpBuffer = + PR_smprintf("Original-Message-ID: <%s>" CRLF, m_messageId.get()); + PUSH_N_FREE_STRING(tmpBuffer); - if (!firstPart1.IsEmpty()) - { - tmpBuffer = PR_smprintf("%s" CRLF CRLF, NS_ConvertUTF16toUTF8(firstPart1).get()); - PUSH_N_FREE_STRING(tmpBuffer); - } + tmpBuffer = + PR_smprintf("Disposition: %s/%s; %s" CRLF CRLF, + (m_autoAction ? "automatic-action" : "manual-action"), + (m_autoSend ? "MDN-sent-automatically" : "MDN-sent-manually"), + DispositionTypes[(int)m_disposeType]); + PUSH_N_FREE_STRING(tmpBuffer); - switch (m_disposeType) - { - case nsIMsgMdnGenerator::eDisplayed: - rv = GetStringFromName( - "MsgMdnDisplayed", - firstPart2); - break; - case nsIMsgMdnGenerator::eDispatched: - rv = GetStringFromName( - "MsgMdnDispatched", - firstPart2); - break; - case nsIMsgMdnGenerator::eProcessed: - rv = GetStringFromName( - "MsgMdnProcessed", - firstPart2); - break; - case nsIMsgMdnGenerator::eDeleted: - rv = GetStringFromName( - "MsgMdnDeleted", - firstPart2); - break; - case nsIMsgMdnGenerator::eDenied: - rv = GetStringFromName( - "MsgMdnDenied", - firstPart2); - break; - case nsIMsgMdnGenerator::eFailed: - rv = GetStringFromName( - "MsgMdnFailed", - firstPart2); - break; - default: - rv = NS_ERROR_INVALID_ARG; - break; - } - - if (NS_FAILED(rv)) - return rv; - - if (!firstPart2.IsEmpty()) - { - tmpBuffer = - PR_smprintf("%s" CRLF CRLF, - NS_ConvertUTF16toUTF8(firstPart2).get()); - PUSH_N_FREE_STRING(tmpBuffer); - } - - return rv; + return rv; } -nsresult nsMsgMdnGenerator::CreateSecondPart() -{ - DEBUG_MDN("nsMsgMdnGenerator::CreateSecondPart"); - char *tmpBuffer = nullptr; - char *convbuf = nullptr; - nsresult rv = NS_OK; - nsCOMPtr compUtils; - bool conformToStandard = false; +nsresult nsMsgMdnGenerator::CreateThirdPart() { + DEBUG_MDN("nsMsgMdnGenerator::CreateThirdPart"); + char *tmpBuffer = nullptr; + nsresult rv = NS_OK; - tmpBuffer = PR_smprintf("--%s" CRLF, m_mimeSeparator.get()); - PUSH_N_FREE_STRING(tmpBuffer); + tmpBuffer = PR_smprintf("--%s" CRLF, m_mimeSeparator.get()); + PUSH_N_FREE_STRING(tmpBuffer); - tmpBuffer = PR_smprintf("%s" CRLF, "Content-Type: message/disposition-notification; name=\042MDNPart2.txt\042"); - PUSH_N_FREE_STRING(tmpBuffer); + tmpBuffer = PR_smprintf( + "%s" CRLF, + "Content-Type: text/rfc822-headers; name=\042MDNPart3.txt\042"); + PUSH_N_FREE_STRING(tmpBuffer); - tmpBuffer = PR_smprintf("%s" CRLF, "Content-Disposition: inline"); - PUSH_N_FREE_STRING(tmpBuffer); + tmpBuffer = PR_smprintf("%s" CRLF, "Content-Transfer-Encoding: 7bit"); + PUSH_N_FREE_STRING(tmpBuffer); - tmpBuffer = PR_smprintf("Content-Transfer-Encoding: %s" CRLF CRLF, - ENCODING_7BIT); - PUSH_N_FREE_STRING(tmpBuffer); + tmpBuffer = PR_smprintf("%s" CRLF CRLF, "Content-Disposition: inline"); + PUSH_N_FREE_STRING(tmpBuffer); - nsCOMPtr pHTTPHandler = - do_GetService(NS_NETWORK_PROTOCOL_CONTRACTID_PREFIX "http", &rv); - if (NS_SUCCEEDED(rv) && pHTTPHandler) - { - nsAutoCString userAgentString; - // Ignore error since we're testing the return value. - mozilla::Unused << pHTTPHandler->GetUserAgent(userAgentString); + rv = OutputAllHeaders(); - if (!userAgentString.IsEmpty()) - { - // Prepend the product name with the dns name according to RFC 3798. - char hostName[256]; - PR_GetSystemInfo(PR_SI_HOSTNAME_UNTRUNCATED, hostName, sizeof hostName); - if ((hostName[0] != '\0') && (strchr(hostName, '.') != NULL)) - { - userAgentString.InsertLiteral("; ", 0); - userAgentString.Insert(nsDependentCString(hostName), 0); + if (NS_FAILED(rv)) return rv; + + rv = WriteString(CRLF); + if (NS_FAILED(rv)) return rv; + + tmpBuffer = PR_smprintf("--%s--" CRLF, m_mimeSeparator.get()); + PUSH_N_FREE_STRING(tmpBuffer); + + return rv; +} + +nsresult nsMsgMdnGenerator::OutputAllHeaders() { + DEBUG_MDN("nsMsgMdnGenerator::OutputAllHeaders"); + nsCString all_headers; + int32_t all_headers_size = 0; + nsresult rv = NS_OK; + + rv = m_headers->GetAllHeaders(all_headers); + if (NS_FAILED(rv)) return rv; + all_headers_size = all_headers.Length(); + char *buf = (char *)all_headers.get(), + *buf_end = (char *)all_headers.get() + all_headers_size; + char *start = buf, *end = buf; + + while (buf < buf_end) { + switch (*buf) { + case 0: + if (*(buf + 1) == '\n') { + // *buf = '\r'; + end = buf; + } else if (*(buf + 1) == 0) { + // the case of message id + *buf = '>'; } + break; + case '\r': + end = buf; + *buf = 0; + break; + case '\n': + if (buf > start && *(buf - 1) == 0) { + start = buf + 1; + end = start; + } else { + end = buf; + } + *buf = 0; + break; + default: + break; + } + buf++; - tmpBuffer = PR_smprintf("Reporting-UA: %s" CRLF, - userAgentString.get()); - PUSH_N_FREE_STRING(tmpBuffer); + if (end > start && *end == 0) { + // strip out private X-Mozilla-Status header & X-Mozilla-Draft-Info && + // envelope header + if (!PL_strncasecmp(start, X_MOZILLA_STATUS, X_MOZILLA_STATUS_LEN) || + !PL_strncasecmp(start, X_MOZILLA_DRAFT_INFO, + X_MOZILLA_DRAFT_INFO_LEN) || + !PL_strncasecmp(start, "From ", 5)) { + while (end < buf_end && (*end == '\n' || *end == '\r' || *end == 0)) + end++; + start = end; + } else { + NS_ASSERTION(*end == 0, "content of end should be null"); + rv = WriteString(start); + if (NS_FAILED(rv)) return rv; + rv = WriteString(CRLF); + while (end < buf_end && (*end == '\n' || *end == '\r' || *end == 0)) + end++; + start = end; } + buf = start; } - - nsCString originalRecipient; - m_headers->ExtractHeader(HEADER_ORIGINAL_RECIPIENT, false, - originalRecipient); - - if (!originalRecipient.IsEmpty()) - { - tmpBuffer = PR_smprintf("Original-Recipient: %s" CRLF, - originalRecipient.get()); - PUSH_N_FREE_STRING(tmpBuffer); - } - - compUtils = do_GetService(NS_MSGCOMPUTILS_CONTRACTID, &rv); - if (compUtils) - compUtils->GetMsgMimeConformToStandard(&conformToStandard); - - convbuf = nsMsgI18NEncodeMimePartIIStr( - m_email.get(), true, m_charset.get(), 0, - conformToStandard); - tmpBuffer = PR_smprintf("Final-Recipient: rfc822;%s" CRLF, convbuf ? - convbuf : m_email.get()); - PUSH_N_FREE_STRING(tmpBuffer); - - PR_Free (convbuf); - - if (*m_messageId.get() == '<') - tmpBuffer = PR_smprintf("Original-Message-ID: %s" CRLF, m_messageId.get()); - else - tmpBuffer = PR_smprintf("Original-Message-ID: <%s>" CRLF, m_messageId.get()); - PUSH_N_FREE_STRING(tmpBuffer); - - tmpBuffer = PR_smprintf("Disposition: %s/%s; %s" CRLF CRLF, - (m_autoAction ? "automatic-action" : - "manual-action"), - (m_autoSend ? "MDN-sent-automatically" : - "MDN-sent-manually"), - DispositionTypes[(int) m_disposeType]); - PUSH_N_FREE_STRING(tmpBuffer); - - return rv; + } + return NS_OK; } -nsresult nsMsgMdnGenerator::CreateThirdPart() -{ - DEBUG_MDN("nsMsgMdnGenerator::CreateThirdPart"); - char *tmpBuffer = nullptr; - nsresult rv = NS_OK; +nsresult nsMsgMdnGenerator::SendMdnMsg() { + DEBUG_MDN("nsMsgMdnGenerator::SendMdnMsg"); + nsresult rv; + nsCOMPtr smtpService = + do_GetService(NS_SMTPSERVICE_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); - tmpBuffer = PR_smprintf("--%s" CRLF, m_mimeSeparator.get()); - PUSH_N_FREE_STRING(tmpBuffer); + nsCOMPtr aRequest; + nsCString identEmail; + m_identity->GetEmail(identEmail); + smtpService->SendMailMessage( + m_file, m_dntRrt.get(), m_identity, identEmail.get(), EmptyString(), this, + nullptr, nullptr, false, nullptr, getter_AddRefs(aRequest)); - tmpBuffer = PR_smprintf("%s" CRLF, "Content-Type: text/rfc822-headers; name=\042MDNPart3.txt\042"); - PUSH_N_FREE_STRING(tmpBuffer); - - tmpBuffer = PR_smprintf("%s" CRLF, "Content-Transfer-Encoding: 7bit"); - PUSH_N_FREE_STRING(tmpBuffer); - - tmpBuffer = PR_smprintf("%s" CRLF CRLF, "Content-Disposition: inline"); - PUSH_N_FREE_STRING(tmpBuffer); - - rv = OutputAllHeaders(); - - if (NS_FAILED(rv)) - return rv; - - rv = WriteString(CRLF); - if (NS_FAILED(rv)) - return rv; - - tmpBuffer = PR_smprintf("--%s--" CRLF, m_mimeSeparator.get()); - PUSH_N_FREE_STRING(tmpBuffer); - - return rv; + return NS_OK; } - -nsresult nsMsgMdnGenerator::OutputAllHeaders() -{ - DEBUG_MDN("nsMsgMdnGenerator::OutputAllHeaders"); - nsCString all_headers; - int32_t all_headers_size = 0; - nsresult rv = NS_OK; - - rv = m_headers->GetAllHeaders(all_headers); - if (NS_FAILED(rv)) - return rv; - all_headers_size = all_headers.Length(); - char *buf = (char *) all_headers.get(), - *buf_end = (char *) all_headers.get()+all_headers_size; - char *start = buf, *end = buf; - - while (buf < buf_end) - { - switch (*buf) - { - case 0: - if (*(buf+1) == '\n') - { - // *buf = '\r'; - end = buf; - } - else if (*(buf+1) == 0) - { - // the case of message id - *buf = '>'; - } - break; - case '\r': - end = buf; - *buf = 0; - break; - case '\n': - if (buf > start && *(buf-1) == 0) - { - start = buf + 1; - end = start; - } - else - { - end = buf; - } - *buf = 0; - break; - default: - break; - } - buf++; - - if (end > start && *end == 0) - { - // strip out private X-Mozilla-Status header & X-Mozilla-Draft-Info && envelope header - if (!PL_strncasecmp(start, X_MOZILLA_STATUS, X_MOZILLA_STATUS_LEN) - || !PL_strncasecmp(start, X_MOZILLA_DRAFT_INFO, X_MOZILLA_DRAFT_INFO_LEN) - || !PL_strncasecmp(start, "From ", 5)) - { - while ( end < buf_end && - (*end == '\n' || *end == '\r' || *end == 0)) - end++; - start = end; - } - else - { - NS_ASSERTION (*end == 0, "content of end should be null"); - rv = WriteString(start); - if (NS_FAILED(rv)) - return rv; - rv = WriteString(CRLF); - while ( end < buf_end && - (*end == '\n' || *end == '\r' || *end == 0)) - end++; - start = end; - } - buf = start; - } - } - return NS_OK; -} - -nsresult nsMsgMdnGenerator::SendMdnMsg() -{ - DEBUG_MDN("nsMsgMdnGenerator::SendMdnMsg"); - nsresult rv; - nsCOMPtr smtpService = do_GetService(NS_SMTPSERVICE_CONTRACTID, &rv); - NS_ENSURE_SUCCESS(rv,rv); - - nsCOMPtr aRequest; - nsCString identEmail; - m_identity->GetEmail(identEmail); - smtpService->SendMailMessage(m_file, m_dntRrt.get(), m_identity, identEmail.get(), - EmptyString(), this, nullptr, nullptr, false, - nullptr, getter_AddRefs(aRequest)); - - return NS_OK; -} - -nsresult nsMsgMdnGenerator::WriteString( const char *str ) -{ - NS_ENSURE_ARG (str); +nsresult nsMsgMdnGenerator::WriteString(const char *str) { + NS_ENSURE_ARG(str); uint32_t len = strlen(str); uint32_t wLen = 0; return m_outputStream->Write(str, len, &wLen); } -nsresult nsMsgMdnGenerator::InitAndProcess(bool *needToAskUser) -{ - DEBUG_MDN("nsMsgMdnGenerator::InitAndProcess"); - nsresult rv = m_folder->GetServer(getter_AddRefs(m_server)); - nsCOMPtr accountManager = - do_GetService(NS_MSGACCOUNTMANAGER_CONTRACTID, &rv); - if (accountManager && m_server) - { - if (!m_identity) - { - // check if this is a message delivered to the global inbox, - // in which case we find the originating account's identity. - nsCString accountKey; - m_headers->ExtractHeader(HEADER_X_MOZILLA_ACCOUNT_KEY, false, - accountKey); - nsCOMPtr account; - if (!accountKey.IsEmpty()) - accountManager->GetAccount(accountKey, getter_AddRefs(account)); - if (account) - account->GetIncomingServer(getter_AddRefs(m_server)); +nsresult nsMsgMdnGenerator::InitAndProcess(bool *needToAskUser) { + DEBUG_MDN("nsMsgMdnGenerator::InitAndProcess"); + nsresult rv = m_folder->GetServer(getter_AddRefs(m_server)); + nsCOMPtr accountManager = + do_GetService(NS_MSGACCOUNTMANAGER_CONTRACTID, &rv); + if (accountManager && m_server) { + if (!m_identity) { + // check if this is a message delivered to the global inbox, + // in which case we find the originating account's identity. + nsCString accountKey; + m_headers->ExtractHeader(HEADER_X_MOZILLA_ACCOUNT_KEY, false, accountKey); + nsCOMPtr account; + if (!accountKey.IsEmpty()) + accountManager->GetAccount(accountKey, getter_AddRefs(account)); + if (account) account->GetIncomingServer(getter_AddRefs(m_server)); - if (m_server) - { - // Find the correct identity based on the "To:" and "Cc:" header - nsCString mailTo; - nsCString mailCC; - m_headers->ExtractHeader(HEADER_TO, true, mailTo); - m_headers->ExtractHeader(HEADER_CC, true, mailCC); - nsCOMPtr servIdentities; - accountManager->GetIdentitiesForServer(m_server, getter_AddRefs(servIdentities)); - if (servIdentities) - { - nsCOMPtr ident; - nsCString identEmail; - uint32_t count = 0; - servIdentities->GetLength(&count); - // First check in the "To:" header - for (uint32_t i = 0; i < count; i++) - { - ident = do_QueryElementAt(servIdentities, i, &rv); - if (NS_FAILED(rv)) - continue; - ident->GetEmail(identEmail); - if (!mailTo.IsEmpty() && !identEmail.IsEmpty() && - mailTo.Find(identEmail, /* ignoreCase = */ true) != kNotFound) - { - m_identity = ident; - break; - } - } - // If no match, check the "Cc:" header - if (!m_identity) - { - for (uint32_t i = 0; i < count; i++) - { - ident = do_QueryElementAt(servIdentities, i, &rv); - if (NS_FAILED(rv)) - continue; - ident->GetEmail(identEmail); - if (!mailCC.IsEmpty() && !identEmail.IsEmpty() && - mailCC.Find(identEmail, /* ignoreCase = */ true) != kNotFound) - { - m_identity = ident; - break; - } - } + if (m_server) { + // Find the correct identity based on the "To:" and "Cc:" header + nsCString mailTo; + nsCString mailCC; + m_headers->ExtractHeader(HEADER_TO, true, mailTo); + m_headers->ExtractHeader(HEADER_CC, true, mailCC); + nsCOMPtr servIdentities; + accountManager->GetIdentitiesForServer(m_server, + getter_AddRefs(servIdentities)); + if (servIdentities) { + nsCOMPtr ident; + nsCString identEmail; + uint32_t count = 0; + servIdentities->GetLength(&count); + // First check in the "To:" header + for (uint32_t i = 0; i < count; i++) { + ident = do_QueryElementAt(servIdentities, i, &rv); + if (NS_FAILED(rv)) continue; + ident->GetEmail(identEmail); + if (!mailTo.IsEmpty() && !identEmail.IsEmpty() && + mailTo.Find(identEmail, /* ignoreCase = */ true) != kNotFound) { + m_identity = ident; + break; + } + } + // If no match, check the "Cc:" header + if (!m_identity) { + for (uint32_t i = 0; i < count; i++) { + ident = do_QueryElementAt(servIdentities, i, &rv); + if (NS_FAILED(rv)) continue; + ident->GetEmail(identEmail); + if (!mailCC.IsEmpty() && !identEmail.IsEmpty() && + mailCC.Find(identEmail, /* ignoreCase = */ true) != + kNotFound) { + m_identity = ident; + break; } } - - // If no match again, use the first identity - if (!m_identity) - rv = accountManager->GetFirstIdentityForServer(m_server, getter_AddRefs(m_identity)); } } - NS_ENSURE_SUCCESS(rv,rv); - if (m_identity) - { - bool useCustomPrefs = false; - m_identity->GetBoolAttribute("use_custom_prefs", &useCustomPrefs); - if (useCustomPrefs) - { - bool bVal = false; - m_server->GetBoolValue("mdn_report_enabled", &bVal); - m_mdnEnabled = bVal; - m_server->GetIntValue("mdn_not_in_to_cc", &m_notInToCcOp); - m_server->GetIntValue("mdn_outside_domain", - &m_outsideDomainOp); - m_server->GetIntValue("mdn_other", &m_otherOp); - } - else - { - bool bVal = false; - - nsCOMPtr prefBranch(do_GetService(NS_PREFSERVICE_CONTRACTID, &rv)); - if (NS_FAILED(rv)) - return rv; - - if(prefBranch) - { - prefBranch->GetBoolPref("mail.mdn.report.enabled", - &bVal); - m_mdnEnabled = bVal; - prefBranch->GetIntPref("mail.mdn.report.not_in_to_cc", - &m_notInToCcOp); - prefBranch->GetIntPref("mail.mdn.report.outside_domain", - &m_outsideDomainOp); - prefBranch->GetIntPref("mail.mdn.report.other", - &m_otherOp); - } - } - } + // If no match again, use the first identity + if (!m_identity) + rv = accountManager->GetFirstIdentityForServer( + m_server, getter_AddRefs(m_identity)); + } } + NS_ENSURE_SUCCESS(rv, rv); - rv = m_folder->GetCharset(m_charset); - if (m_mdnEnabled) - { - m_headers->ExtractHeader(HEADER_DISPOSITION_NOTIFICATION_TO, false, - m_dntRrt); - if (m_dntRrt.IsEmpty()) - m_headers->ExtractHeader(HEADER_RETURN_RECEIPT_TO, false, - m_dntRrt); - if (!m_dntRrt.IsEmpty() && ProcessSendMode() && ValidateReturnPath()) - { - if (!m_autoSend) - { - *needToAskUser = true; - rv = NS_OK; - } - else - { - *needToAskUser = false; - rv = UserAgreed(); - } + if (m_identity) { + bool useCustomPrefs = false; + m_identity->GetBoolAttribute("use_custom_prefs", &useCustomPrefs); + if (useCustomPrefs) { + bool bVal = false; + m_server->GetBoolValue("mdn_report_enabled", &bVal); + m_mdnEnabled = bVal; + m_server->GetIntValue("mdn_not_in_to_cc", &m_notInToCcOp); + m_server->GetIntValue("mdn_outside_domain", &m_outsideDomainOp); + m_server->GetIntValue("mdn_other", &m_otherOp); + } else { + bool bVal = false; + + nsCOMPtr prefBranch( + do_GetService(NS_PREFSERVICE_CONTRACTID, &rv)); + if (NS_FAILED(rv)) return rv; + + if (prefBranch) { + prefBranch->GetBoolPref("mail.mdn.report.enabled", &bVal); + m_mdnEnabled = bVal; + prefBranch->GetIntPref("mail.mdn.report.not_in_to_cc", + &m_notInToCcOp); + prefBranch->GetIntPref("mail.mdn.report.outside_domain", + &m_outsideDomainOp); + prefBranch->GetIntPref("mail.mdn.report.other", &m_otherOp); } + } } - return rv; + } + + rv = m_folder->GetCharset(m_charset); + if (m_mdnEnabled) { + m_headers->ExtractHeader(HEADER_DISPOSITION_NOTIFICATION_TO, false, + m_dntRrt); + if (m_dntRrt.IsEmpty()) + m_headers->ExtractHeader(HEADER_RETURN_RECEIPT_TO, false, m_dntRrt); + if (!m_dntRrt.IsEmpty() && ProcessSendMode() && ValidateReturnPath()) { + if (!m_autoSend) { + *needToAskUser = true; + rv = NS_OK; + } else { + *needToAskUser = false; + rv = UserAgreed(); + } + } + } + return rv; } NS_IMETHODIMP nsMsgMdnGenerator::Process(EDisposeType type, nsIMsgWindow *aWindow, - nsIMsgFolder *folder, - nsMsgKey key, + nsIMsgFolder *folder, nsMsgKey key, nsIMimeHeaders *headers, - bool autoAction, - bool *_retval) -{ - DEBUG_MDN("nsMsgMdnGenerator::Process"); - NS_ENSURE_ARG_POINTER(folder); - NS_ENSURE_ARG_POINTER(headers); - NS_ENSURE_ARG_POINTER(aWindow); - NS_ENSURE_TRUE(key != nsMsgKey_None, NS_ERROR_INVALID_ARG); - m_disposeType = type; - m_autoAction = autoAction; - m_window = aWindow; - m_folder = folder; - m_headers = headers; - m_key = key; + bool autoAction, bool *_retval) { + DEBUG_MDN("nsMsgMdnGenerator::Process"); + NS_ENSURE_ARG_POINTER(folder); + NS_ENSURE_ARG_POINTER(headers); + NS_ENSURE_ARG_POINTER(aWindow); + NS_ENSURE_TRUE(key != nsMsgKey_None, NS_ERROR_INVALID_ARG); + m_disposeType = type; + m_autoAction = autoAction; + m_window = aWindow; + m_folder = folder; + m_headers = headers; + m_key = key; - mozilla::DebugOnly rv = InitAndProcess(_retval); - NS_ASSERTION(NS_SUCCEEDED(rv), "InitAndProcess failed"); - return NS_OK; + mozilla::DebugOnly rv = InitAndProcess(_retval); + NS_ASSERTION(NS_SUCCEEDED(rv), "InitAndProcess failed"); + return NS_OK; } -NS_IMETHODIMP nsMsgMdnGenerator::UserAgreed() -{ +NS_IMETHODIMP nsMsgMdnGenerator::UserAgreed() { DEBUG_MDN("nsMsgMdnGenerator::UserAgreed"); - (void) NoteMDNRequestHandled(); + (void)NoteMDNRequestHandled(); return CreateMdnMsg(); } -NS_IMETHODIMP nsMsgMdnGenerator::UserDeclined() -{ +NS_IMETHODIMP nsMsgMdnGenerator::UserDeclined() { DEBUG_MDN("nsMsgMdnGenerator::UserDeclined"); return NoteMDNRequestHandled(); } @@ -1053,8 +921,7 @@ NS_IMETHODIMP nsMsgMdnGenerator::UserDeclined() * Set/clear flags appropriately so we won't ask user again about MDN * request for this message. */ -nsresult nsMsgMdnGenerator::NoteMDNRequestHandled() -{ +nsresult nsMsgMdnGenerator::NoteMDNRequestHandled() { nsresult rv = StoreMDNSentFlag(m_folder, m_key); NS_ASSERTION(NS_SUCCEEDED(rv), "StoreMDNSentFlag failed"); rv = ClearMDNNeededFlag(m_folder, m_key); @@ -1062,82 +929,77 @@ nsresult nsMsgMdnGenerator::NoteMDNRequestHandled() return rv; } -NS_IMETHODIMP nsMsgMdnGenerator::OnStartRunningUrl(nsIURI *url) -{ - DEBUG_MDN("nsMsgMdnGenerator::OnStartRunningUrl"); - return NS_OK; +NS_IMETHODIMP nsMsgMdnGenerator::OnStartRunningUrl(nsIURI *url) { + DEBUG_MDN("nsMsgMdnGenerator::OnStartRunningUrl"); + return NS_OK; } NS_IMETHODIMP nsMsgMdnGenerator::OnStopRunningUrl(nsIURI *url, - nsresult aExitCode) -{ - nsresult rv; + nsresult aExitCode) { + nsresult rv; - DEBUG_MDN("nsMsgMdnGenerator::OnStopRunningUrl"); - if (m_file) - m_file->Remove(false); + DEBUG_MDN("nsMsgMdnGenerator::OnStopRunningUrl"); + if (m_file) m_file->Remove(false); - if (NS_SUCCEEDED(aExitCode)) - return NS_OK; + if (NS_SUCCEEDED(aExitCode)) return NS_OK; - const char* exitString; + const char *exitString; - switch (aExitCode) - { - case NS_ERROR_UNKNOWN_HOST: - case NS_ERROR_UNKNOWN_PROXY_HOST: - exitString = "smtpSendFailedUnknownServer"; - break; - case NS_ERROR_CONNECTION_REFUSED: - case NS_ERROR_PROXY_CONNECTION_REFUSED: - exitString = "smtpSendRequestRefused"; - break; - case NS_ERROR_NET_INTERRUPT: - case NS_ERROR_ABORT: // we have no proper string for error code NS_ERROR_ABORT in compose bundle - exitString = "smtpSendInterrupted"; - break; - case NS_ERROR_NET_TIMEOUT: - case NS_ERROR_NET_RESET: - exitString = "smtpSendTimeout"; - break; - default: - exitString = errorStringNameForErrorCode(aExitCode); - break; - } + switch (aExitCode) { + case NS_ERROR_UNKNOWN_HOST: + case NS_ERROR_UNKNOWN_PROXY_HOST: + exitString = "smtpSendFailedUnknownServer"; + break; + case NS_ERROR_CONNECTION_REFUSED: + case NS_ERROR_PROXY_CONNECTION_REFUSED: + exitString = "smtpSendRequestRefused"; + break; + case NS_ERROR_NET_INTERRUPT: + case NS_ERROR_ABORT: // we have no proper string for error code + // NS_ERROR_ABORT in compose bundle + exitString = "smtpSendInterrupted"; + break; + case NS_ERROR_NET_TIMEOUT: + case NS_ERROR_NET_RESET: + exitString = "smtpSendTimeout"; + break; + default: + exitString = errorStringNameForErrorCode(aExitCode); + break; + } - nsCOMPtr smtpService(do_GetService(NS_SMTPSERVICE_CONTRACTID, &rv)); - NS_ENSURE_SUCCESS(rv,rv); + nsCOMPtr smtpService( + do_GetService(NS_SMTPSERVICE_CONTRACTID, &rv)); + NS_ENSURE_SUCCESS(rv, rv); - // Get the smtp hostname and format the string. - nsCString smtpHostName; - nsCOMPtr smtpServer; - rv = smtpService->GetServerByIdentity(m_identity, getter_AddRefs(smtpServer)); - if (NS_SUCCEEDED(rv)) - smtpServer->GetHostname(smtpHostName); + // Get the smtp hostname and format the string. + nsCString smtpHostName; + nsCOMPtr smtpServer; + rv = smtpService->GetServerByIdentity(m_identity, getter_AddRefs(smtpServer)); + if (NS_SUCCEEDED(rv)) smtpServer->GetHostname(smtpHostName); - nsAutoString hostStr; - CopyASCIItoUTF16(smtpHostName, hostStr); - const char16_t *params[] = { hostStr.get() }; + nsAutoString hostStr; + CopyASCIItoUTF16(smtpHostName, hostStr); + const char16_t *params[] = {hostStr.get()}; - nsCOMPtr bundle; - nsCOMPtr bundleService = + nsCOMPtr bundle; + nsCOMPtr bundleService = mozilla::services::GetStringBundleService(); - NS_ENSURE_TRUE(bundleService, NS_ERROR_UNEXPECTED); + NS_ENSURE_TRUE(bundleService, NS_ERROR_UNEXPECTED); - rv = bundleService->CreateBundle( + rv = bundleService->CreateBundle( "chrome://messenger/locale/messengercompose/composeMsgs.properties", getter_AddRefs(bundle)); - NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_SUCCESS(rv, rv); - nsString failed_msg, dialogTitle; + nsString failed_msg, dialogTitle; - bundle->FormatStringFromName(exitString, params, 1, failed_msg); - bundle->GetStringFromName("sendMessageErrorTitle", dialogTitle); + bundle->FormatStringFromName(exitString, params, 1, failed_msg); + bundle->GetStringFromName("sendMessageErrorTitle", dialogTitle); - nsCOMPtr dialog; - rv = m_window->GetPromptDialog(getter_AddRefs(dialog)); - if (NS_SUCCEEDED(rv)) - dialog->Alert(dialogTitle.get(),failed_msg.get()); + nsCOMPtr dialog; + rv = m_window->GetPromptDialog(getter_AddRefs(dialog)); + if (NS_SUCCEEDED(rv)) dialog->Alert(dialogTitle.get(), failed_msg.get()); - return NS_OK; + return NS_OK; } diff --git a/mailnews/extensions/mdn/src/nsMsgMdnGenerator.h b/mailnews/extensions/mdn/src/nsMsgMdnGenerator.h index 42da164f0b..3b57e4953d 100644 --- a/mailnews/extensions/mdn/src/nsMsgMdnGenerator.h +++ b/mailnews/extensions/mdn/src/nsMsgMdnGenerator.h @@ -18,25 +18,24 @@ #include "nsString.h" #include "MailNewsTypes2.h" -#define eNeverSendOp ((int32_t) 0) -#define eAutoSendOp ((int32_t) 1) -#define eAskMeOp ((int32_t) 2) -#define eDeniedOp ((int32_t) 3) +#define eNeverSendOp ((int32_t)0) +#define eAutoSendOp ((int32_t)1) +#define eAskMeOp ((int32_t)2) +#define eDeniedOp ((int32_t)3) -class nsMsgMdnGenerator : public nsIMsgMdnGenerator, public nsIUrlListener -{ -public: +class nsMsgMdnGenerator : public nsIMsgMdnGenerator, public nsIUrlListener { + public: NS_DECL_ISUPPORTS NS_DECL_NSIMSGMDNGENERATOR NS_DECL_NSIURLLISTENER nsMsgMdnGenerator(); -private: + private: virtual ~nsMsgMdnGenerator(); // Sanity Check methods - bool ProcessSendMode(); // must called prior ValidateReturnPath + bool ProcessSendMode(); // must called prior ValidateReturnPath bool ValidateReturnPath(); bool NotInToOrCc(); bool MailAddrMatch(const char *addr1, const char *addr2); @@ -52,17 +51,16 @@ private: nsresult SendMdnMsg(); // string bundle helper methods - nsresult GetStringFromName(const char *aName, nsAString& aResultString); - nsresult FormatStringFromName(const char *aName, - const char16_t *aString, - nsAString& aResultString); + nsresult GetStringFromName(const char *aName, nsAString &aResultString); + nsresult FormatStringFromName(const char *aName, const char16_t *aString, + nsAString &aResultString); // other helper methods nsresult InitAndProcess(bool *needToAskUser); nsresult OutputAllHeaders(); nsresult WriteString(const char *str); -private: + private: EDisposeType m_disposeType; nsCOMPtr m_window; nsCOMPtr m_outputStream; @@ -86,5 +84,4 @@ private: bool m_mdnEnabled; }; -#endif // _nsMsgMdnGenerator_H_ - +#endif // _nsMsgMdnGenerator_H_ diff --git a/mailnews/extensions/smime/src/nsCertPicker.cpp b/mailnews/extensions/smime/src/nsCertPicker.cpp index bd42ed951b..a5330facae 100644 --- a/mailnews/extensions/smime/src/nsCertPicker.cpp +++ b/mailnews/extensions/smime/src/nsCertPicker.cpp @@ -28,12 +28,10 @@ using namespace mozilla; MOZ_TYPE_SPECIFIC_UNIQUE_PTR_TEMPLATE(UniqueCERTCertNicknames, - CERTCertNicknames, - CERT_FreeNicknames) + CERTCertNicknames, CERT_FreeNicknames) -CERTCertNicknames* -getNSSCertNicknamesFromCertList(const UniqueCERTCertList& certList) -{ +CERTCertNicknames *getNSSCertNicknamesFromCertList( + const UniqueCERTCertList &certList) { nsAutoString expiredString, notYetValidString; nsAutoString expiredStringLeadingSpace, notYetValidStringLeadingSpace; @@ -49,15 +47,13 @@ getNSSCertNicknamesFromCertList(const UniqueCERTCertList& certList) NS_ConvertUTF16toUTF8 aUtf8ExpiredString(expiredStringLeadingSpace); NS_ConvertUTF16toUTF8 aUtf8NotYetValidString(notYetValidStringLeadingSpace); - return CERT_NicknameStringsFromCertList(certList.get(), - const_cast(aUtf8ExpiredString.get()), - const_cast(aUtf8NotYetValidString.get())); + return CERT_NicknameStringsFromCertList( + certList.get(), const_cast(aUtf8ExpiredString.get()), + const_cast(aUtf8NotYetValidString.get())); } -nsresult -FormatUIStrings(nsIX509Cert* cert, const nsAutoString& nickname, - nsAutoString& nickWithSerial, nsAutoString& details) -{ +nsresult FormatUIStrings(nsIX509Cert *cert, const nsAutoString &nickname, + nsAutoString &nickWithSerial, nsAutoString &details) { if (!NS_IsMainThread()) { NS_ERROR("nsNSSCertificate::FormatUIStrings called off the main thread"); return NS_ERROR_NOT_SAME_THREAD; @@ -105,7 +101,8 @@ FormatUIStrings(nsIX509Cert* cert, const nsAutoString& nickname, details.Append(info); } - if (NS_SUCCEEDED(validity->GetNotBeforeLocalTime(temp1)) && !temp1.IsEmpty()) { + if (NS_SUCCEEDED(validity->GetNotBeforeLocalTime(temp1)) && + !temp1.IsEmpty()) { details.Append(char16_t(' ')); if (NS_SUCCEEDED(mcs->GetSMIMEBundleString(u"CertInfoFrom", info))) { details.Append(info); @@ -114,7 +111,8 @@ FormatUIStrings(nsIX509Cert* cert, const nsAutoString& nickname, details.Append(temp1); } - if (NS_SUCCEEDED(validity->GetNotAfterLocalTime(temp1)) && !temp1.IsEmpty()) { + if (NS_SUCCEEDED(validity->GetNotAfterLocalTime(temp1)) && + !temp1.IsEmpty()) { details.Append(char16_t(' ')); if (NS_SUCCEEDED(mcs->GetSMIMEBundleString(u"CertInfoTo", info))) { details.Append(info); @@ -142,16 +140,11 @@ FormatUIStrings(nsIX509Cert* cert, const nsAutoString& nickname, } nsAutoString firstEmail; - const char* aWalkAddr; - for (aWalkAddr = CERT_GetFirstEmailAddress(nssCert.get()) - ; - aWalkAddr - ; - aWalkAddr = CERT_GetNextEmailAddress(nssCert.get(), aWalkAddr)) - { + const char *aWalkAddr; + for (aWalkAddr = CERT_GetFirstEmailAddress(nssCert.get()); aWalkAddr; + aWalkAddr = CERT_GetNextEmailAddress(nssCert.get(), aWalkAddr)) { NS_ConvertUTF8toUTF16 email(aWalkAddr); - if (email.IsEmpty()) - continue; + if (email.IsEmpty()) continue; if (firstEmail.IsEmpty()) { // If the first email address from the subject DN is also present @@ -166,8 +159,7 @@ FormatUIStrings(nsIX509Cert* cert, const nsAutoString& nickname, details.AppendLiteral(": "); } details.Append(email); - } - else { + } else { // Append current address if it's different from the first one. if (!firstEmail.Equals(email)) { details.AppendLiteral(", "); @@ -216,17 +208,11 @@ FormatUIStrings(nsIX509Cert* cert, const nsAutoString& nickname, NS_IMPL_ISUPPORTS(nsCertPicker, nsICertPickDialogs, nsIUserCertPicker) -nsCertPicker::nsCertPicker() -{ -} +nsCertPicker::nsCertPicker() {} -nsCertPicker::~nsCertPicker() -{ -} +nsCertPicker::~nsCertPicker() {} -nsresult -nsCertPicker::Init() -{ +nsresult nsCertPicker::Init() { nsresult rv; nsCOMPtr psm = do_GetService("@mozilla.org/psm;1", &rv); return rv; @@ -235,21 +221,18 @@ nsCertPicker::Init() NS_IMETHODIMP nsCertPicker::PickCertificate(nsIInterfaceRequestor *ctx, const char16_t **certNickList, - const char16_t **certDetailsList, - uint32_t count, - int32_t *selectedIndex, - bool *canceled) -{ + const char16_t **certDetailsList, uint32_t count, + int32_t *selectedIndex, bool *canceled) { nsresult rv; uint32_t i; *canceled = false; nsCOMPtr block = - do_CreateInstance(NS_DIALOGPARAMBLOCK_CONTRACTID); + do_CreateInstance(NS_DIALOGPARAMBLOCK_CONTRACTID); if (!block) return NS_ERROR_FAILURE; - block->SetNumberStrings(1+count*2); + block->SetNumberStrings(1 + count * 2); for (i = 0; i < count; i++) { rv = block->SetString(i, certNickList[i]); @@ -257,7 +240,7 @@ nsCertPicker::PickCertificate(nsIInterfaceRequestor *ctx, } for (i = 0; i < count; i++) { - rv = block->SetString(i+count, certDetailsList[i]); + rv = block->SetString(i + count, certDetailsList[i]); if (NS_FAILED(rv)) return rv; } @@ -267,9 +250,8 @@ nsCertPicker::PickCertificate(nsIInterfaceRequestor *ctx, rv = block->SetInt(1, *selectedIndex); if (NS_FAILED(rv)) return rv; - rv = nsNSSDialogHelper::openDialog(nullptr, - "chrome://messenger/content/certpicker.xul", - block); + rv = nsNSSDialogHelper::openDialog( + nullptr, "chrome://messenger/content/certpicker.xul", block); if (NS_FAILED(rv)) return rv; int32_t status; @@ -277,7 +259,7 @@ nsCertPicker::PickCertificate(nsIInterfaceRequestor *ctx, rv = block->GetInt(0, &status); if (NS_FAILED(rv)) return rv; - *canceled = (status == 0)?true:false; + *canceled = (status == 0) ? true : false; if (!*canceled) { rv = block->GetInt(1, selectedIndex); } @@ -286,34 +268,29 @@ nsCertPicker::PickCertificate(nsIInterfaceRequestor *ctx, NS_IMETHODIMP nsCertPicker::PickByUsage(nsIInterfaceRequestor *ctx, const char16_t *selectedNickname, - int32_t certUsage, - bool allowInvalid, + int32_t certUsage, bool allowInvalid, bool allowDuplicateNicknames, const nsAString &emailAddress, - bool *canceled, - nsIX509Cert **_retval) -{ + bool *canceled, nsIX509Cert **_retval) { int32_t selectedIndex = -1; bool selectionFound = false; char16_t **certNicknameList = nullptr; char16_t **certDetailsList = nullptr; - CERTCertListNode* node = nullptr; + CERTCertListNode *node = nullptr; nsresult rv = NS_OK; { - // Iterate over all certs. This assures that user is logged in to all hardware tokens. + // Iterate over all certs. This assures that user is logged in to all + // hardware tokens. nsCOMPtr ctx = new PipUIContext(); UniqueCERTCertList allcerts(PK11_ListCerts(PK11CertListUnique, ctx)); } /* find all user certs that are valid for the specified usage */ /* note that we are allowing expired certs in this list */ - UniqueCERTCertList certList( - CERT_FindUserCertsByUsage(CERT_GetDefaultCertDB(), - (SECCertUsage)certUsage, - !allowDuplicateNicknames, - !allowInvalid, - ctx)); + UniqueCERTCertList certList(CERT_FindUserCertsByUsage( + CERT_GetDefaultCertDB(), (SECCertUsage)certUsage, + !allowDuplicateNicknames, !allowInvalid, ctx)); if (!certList) { return NS_ERROR_NOT_AVAILABLE; } @@ -334,7 +311,7 @@ NS_IMETHODIMP nsCertPicker::PickByUsage(nsIInterfaceRequestor *ctx, } if (!match) { /* doesn't contain the specified address, so remove from the list */ - CERTCertListNode* freenode = node; + CERTCertListNode *freenode = node; node = CERT_LIST_NEXT(node); CERT_RemoveCertListNode(freenode); continue; @@ -349,8 +326,10 @@ NS_IMETHODIMP nsCertPicker::PickByUsage(nsIInterfaceRequestor *ctx, return NS_ERROR_NOT_AVAILABLE; } - certNicknameList = (char16_t **)moz_xmalloc(sizeof(char16_t *) * nicknames->numnicknames); - certDetailsList = (char16_t **)moz_xmalloc(sizeof(char16_t *) * nicknames->numnicknames); + certNicknameList = + (char16_t **)moz_xmalloc(sizeof(char16_t *) * nicknames->numnicknames); + certDetailsList = + (char16_t **)moz_xmalloc(sizeof(char16_t *) * nicknames->numnicknames); if (!certNicknameList || !certDetailsList) { free(certNicknameList); @@ -362,15 +341,13 @@ NS_IMETHODIMP nsCertPicker::PickByUsage(nsIInterfaceRequestor *ctx, for (CertsToUse = 0, node = CERT_LIST_HEAD(certList.get()); !CERT_LIST_END(node, certList.get()) && - CertsToUse < nicknames->numnicknames; - node = CERT_LIST_NEXT(node) - ) - { + CertsToUse < nicknames->numnicknames; + node = CERT_LIST_NEXT(node)) { RefPtr tempCert(nsNSSCertificate::Create(node->cert)); if (tempCert) { - - nsAutoString i_nickname(NS_ConvertUTF8toUTF16(nicknames->nicknames[CertsToUse])); + nsAutoString i_nickname( + NS_ConvertUTF8toUTF16(nicknames->nicknames[CertsToUse])); nsAutoString nickWithSerial; nsAutoString details; @@ -382,8 +359,8 @@ NS_IMETHODIMP nsCertPicker::PickByUsage(nsIInterfaceRequestor *ctx, } } - if (NS_SUCCEEDED(FormatUIStrings(tempCert, i_nickname, nickWithSerial, - details))) { + if (NS_SUCCEEDED( + FormatUIStrings(tempCert, i_nickname, nickWithSerial, details))) { certNicknameList[CertsToUse] = ToNewUnicode(nickWithSerial); certDetailsList[CertsToUse] = ToNewUnicode(details); if (!selectionFound) { @@ -393,8 +370,7 @@ NS_IMETHODIMP nsCertPicker::PickByUsage(nsIInterfaceRequestor *ctx, selectionFound = true; } } - } - else { + } else { certNicknameList[CertsToUse] = nullptr; certDetailsList[CertsToUse] = nullptr; } @@ -410,8 +386,8 @@ NS_IMETHODIMP nsCertPicker::PickByUsage(nsIInterfaceRequestor *ctx, if (NS_SUCCEEDED(rv)) { // Show the cert picker dialog and get the index of the selected cert. - rv = dialogs->PickCertificate(ctx, (const char16_t**)certNicknameList, - (const char16_t**)certDetailsList, + rv = dialogs->PickCertificate(ctx, (const char16_t **)certNicknameList, + (const char16_t **)certDetailsList, CertsToUse, &selectedIndex, canceled); } } @@ -429,10 +405,8 @@ NS_IMETHODIMP nsCertPicker::PickByUsage(nsIInterfaceRequestor *ctx, } if (NS_SUCCEEDED(rv) && !*canceled) { - for (i = 0, node = CERT_LIST_HEAD(certList); - !CERT_LIST_END(node, certList); + for (i = 0, node = CERT_LIST_HEAD(certList); !CERT_LIST_END(node, certList); ++i, node = CERT_LIST_NEXT(node)) { - if (i == selectedIndex) { RefPtr cert = nsNSSCertificate::Create(node->cert); if (!cert) { diff --git a/mailnews/extensions/smime/src/nsCertPicker.h b/mailnews/extensions/smime/src/nsCertPicker.h index a95a1efeb2..04b5270eda 100644 --- a/mailnews/extensions/smime/src/nsCertPicker.h +++ b/mailnews/extensions/smime/src/nsCertPicker.h @@ -9,13 +9,15 @@ #include "nsICertPickDialogs.h" #include "nsIUserCertPicker.h" -#define NS_CERT_PICKER_CID \ - { 0x735959a1, 0xaf01, 0x447e, { 0xb0, 0x2d, 0x56, 0xe9, 0x68, 0xfa, 0x52, 0xb4 } } +#define NS_CERT_PICKER_CID \ + { \ + 0x735959a1, 0xaf01, 0x447e, { \ + 0xb0, 0x2d, 0x56, 0xe9, 0x68, 0xfa, 0x52, 0xb4 \ + } \ + } -class nsCertPicker : public nsICertPickDialogs - , public nsIUserCertPicker -{ -public: +class nsCertPicker : public nsICertPickDialogs, public nsIUserCertPicker { + public: NS_DECL_ISUPPORTS NS_DECL_NSICERTPICKDIALOGS NS_DECL_NSIUSERCERTPICKER @@ -23,8 +25,8 @@ public: nsCertPicker(); nsresult Init(); -protected: + protected: virtual ~nsCertPicker(); }; -#endif // nsCertPicker_h +#endif // nsCertPicker_h diff --git a/mailnews/extensions/smime/src/nsEncryptedSMIMEURIsService.cpp b/mailnews/extensions/smime/src/nsEncryptedSMIMEURIsService.cpp index 9276a07a6e..706342b15f 100644 --- a/mailnews/extensions/smime/src/nsEncryptedSMIMEURIsService.cpp +++ b/mailnews/extensions/smime/src/nsEncryptedSMIMEURIsService.cpp @@ -6,31 +6,27 @@ NS_IMPL_ISUPPORTS(nsEncryptedSMIMEURIsService, nsIEncryptedSMIMEURIsService) -nsEncryptedSMIMEURIsService::nsEncryptedSMIMEURIsService() -{ -} +nsEncryptedSMIMEURIsService::nsEncryptedSMIMEURIsService() {} -nsEncryptedSMIMEURIsService::~nsEncryptedSMIMEURIsService() -{ -} +nsEncryptedSMIMEURIsService::~nsEncryptedSMIMEURIsService() {} -NS_IMETHODIMP nsEncryptedSMIMEURIsService::RememberEncrypted(const nsACString & uri) -{ +NS_IMETHODIMP nsEncryptedSMIMEURIsService::RememberEncrypted( + const nsACString &uri) { // Assuming duplicates are allowed. mEncryptedURIs.AppendElement(uri); return NS_OK; } -NS_IMETHODIMP nsEncryptedSMIMEURIsService::ForgetEncrypted(const nsACString & uri) -{ +NS_IMETHODIMP nsEncryptedSMIMEURIsService::ForgetEncrypted( + const nsACString &uri) { // Assuming, this will only remove one copy of the string, if the array // contains multiple copies of the same string. mEncryptedURIs.RemoveElement(uri); return NS_OK; } -NS_IMETHODIMP nsEncryptedSMIMEURIsService::IsEncrypted(const nsACString & uri, bool *_retval) -{ +NS_IMETHODIMP nsEncryptedSMIMEURIsService::IsEncrypted(const nsACString &uri, + bool *_retval) { *_retval = mEncryptedURIs.Contains(uri); return NS_OK; } diff --git a/mailnews/extensions/smime/src/nsEncryptedSMIMEURIsService.h b/mailnews/extensions/smime/src/nsEncryptedSMIMEURIsService.h index 5b49dcf5f8..dfce321f52 100644 --- a/mailnews/extensions/smime/src/nsEncryptedSMIMEURIsService.h +++ b/mailnews/extensions/smime/src/nsEncryptedSMIMEURIsService.h @@ -9,15 +9,14 @@ #include "nsTArray.h" #include "nsString.h" -class nsEncryptedSMIMEURIsService : public nsIEncryptedSMIMEURIsService -{ -public: +class nsEncryptedSMIMEURIsService : public nsIEncryptedSMIMEURIsService { + public: NS_DECL_ISUPPORTS NS_DECL_NSIENCRYPTEDSMIMEURISSERVICE nsEncryptedSMIMEURIsService(); -protected: + protected: virtual ~nsEncryptedSMIMEURIsService(); nsTArray mEncryptedURIs; }; diff --git a/mailnews/extensions/smime/src/nsMsgComposeSecure.cpp b/mailnews/extensions/smime/src/nsMsgComposeSecure.cpp index b5cb16c0f4..41f6ddc31c 100644 --- a/mailnews/extensions/smime/src/nsMsgComposeSecure.cpp +++ b/mailnews/extensions/smime/src/nsMsgComposeSecure.cpp @@ -41,10 +41,11 @@ using namespace mozilla::psm; // displayed only if the MUA doesn't support MIME. // We need to consider what to do in case the server doesn't support 8BITMIME. // In short, we can't use non-ASCII characters here. -static const char crypto_multipart_blurb[] = "This is a cryptographically signed message in MIME format."; +static const char crypto_multipart_blurb[] = + "This is a cryptographically signed message in MIME format."; -static void mime_crypto_write_base64 (void *closure, const char *buf, - unsigned long size); +static void mime_crypto_write_base64(void *closure, const char *buf, + unsigned long size); static nsresult mime_encoder_output_fn(const char *buf, int32_t size, void *closure); static nsresult mime_nested_encoder_output_fn(const char *buf, int32_t size, @@ -55,43 +56,35 @@ static nsresult make_multipart_signed_header_string(bool outer_p, int16_t hash_type); static char *mime_make_separator(const char *prefix); +static void GenerateGlobalRandomBytes(unsigned char *buf, int32_t len) { + static bool firstTime = true; -static void -GenerateGlobalRandomBytes(unsigned char *buf, int32_t len) -{ - static bool firstTime = true; - - if (firstTime) - { + if (firstTime) { // Seed the random-number generator with current time so that // the numbers will be different every time we run. - srand( (unsigned)PR_Now() ); + srand((unsigned)PR_Now()); firstTime = false; } - for( int32_t i = 0; i < len; i++ ) - buf[i] = rand() % 10; + for (int32_t i = 0; i < len; i++) buf[i] = rand() % 10; } -char -*mime_make_separator(const char *prefix) -{ +char *mime_make_separator(const char *prefix) { unsigned char rand_buf[13]; GenerateGlobalRandomBytes(rand_buf, 12); - return PR_smprintf("------------%s" - "%02X%02X%02X%02X" - "%02X%02X%02X%02X" - "%02X%02X%02X%02X", - prefix, - rand_buf[0], rand_buf[1], rand_buf[2], rand_buf[3], - rand_buf[4], rand_buf[5], rand_buf[6], rand_buf[7], - rand_buf[8], rand_buf[9], rand_buf[10], rand_buf[11]); + return PR_smprintf( + "------------%s" + "%02X%02X%02X%02X" + "%02X%02X%02X%02X" + "%02X%02X%02X%02X", + prefix, rand_buf[0], rand_buf[1], rand_buf[2], rand_buf[3], rand_buf[4], + rand_buf[5], rand_buf[6], rand_buf[7], rand_buf[8], rand_buf[9], + rand_buf[10], rand_buf[11]); } // end of copied code which needs fixed.... - ///////////////////////////////////////////////////////////////////////////////////////// // Implementation of nsMsgComposeSecure ///////////////////////////////////////////////////////////////////////////////////////// @@ -99,17 +92,15 @@ char NS_IMPL_ISUPPORTS(nsMsgComposeSecure, nsIMsgComposeSecure) nsMsgComposeSecure::nsMsgComposeSecure() -:mSignMessage(false), mAlwaysEncryptMessage(false) -{ + : mSignMessage(false), mAlwaysEncryptMessage(false) { /* member initializers and constructor code */ - mMultipartSignedBoundary = 0; + mMultipartSignedBoundary = 0; mBuffer = 0; mBufferedBytes = 0; mHashType = 0; } -nsMsgComposeSecure::~nsMsgComposeSecure() -{ +nsMsgComposeSecure::~nsMsgComposeSecure() { /* destructor code */ if (mEncryptionContext) { if (mBufferedBytes) { @@ -119,88 +110,76 @@ nsMsgComposeSecure::~nsMsgComposeSecure() mEncryptionContext->Finish(); } - delete [] mBuffer; + delete[] mBuffer; PR_FREEIF(mMultipartSignedBoundary); } -NS_IMETHODIMP nsMsgComposeSecure::SetSignMessage(bool value) -{ +NS_IMETHODIMP nsMsgComposeSecure::SetSignMessage(bool value) { mSignMessage = value; return NS_OK; } -NS_IMETHODIMP nsMsgComposeSecure::GetSignMessage(bool *_retval) -{ +NS_IMETHODIMP nsMsgComposeSecure::GetSignMessage(bool *_retval) { *_retval = mSignMessage; return NS_OK; } -NS_IMETHODIMP nsMsgComposeSecure::SetRequireEncryptMessage(bool value) -{ +NS_IMETHODIMP nsMsgComposeSecure::SetRequireEncryptMessage(bool value) { mAlwaysEncryptMessage = value; return NS_OK; } -NS_IMETHODIMP nsMsgComposeSecure::GetRequireEncryptMessage(bool *_retval) -{ +NS_IMETHODIMP nsMsgComposeSecure::GetRequireEncryptMessage(bool *_retval) { *_retval = mAlwaysEncryptMessage; return NS_OK; } - -NS_IMETHODIMP nsMsgComposeSecure::RequiresCryptoEncapsulation(nsIMsgIdentity * aIdentity, nsIMsgCompFields * aCompFields, bool * aRequiresEncryptionWork) -{ +NS_IMETHODIMP nsMsgComposeSecure::RequiresCryptoEncapsulation( + nsIMsgIdentity *aIdentity, nsIMsgCompFields *aCompFields, + bool *aRequiresEncryptionWork) { NS_ENSURE_ARG_POINTER(aRequiresEncryptionWork); *aRequiresEncryptionWork = false; bool alwaysEncryptMessages = false; bool signMessage = false; - nsresult rv = ExtractEncryptionState(aIdentity, aCompFields, &signMessage, &alwaysEncryptMessages); + nsresult rv = ExtractEncryptionState(aIdentity, aCompFields, &signMessage, + &alwaysEncryptMessages); NS_ENSURE_SUCCESS(rv, rv); - if (alwaysEncryptMessages || signMessage) - *aRequiresEncryptionWork = true; + if (alwaysEncryptMessages || signMessage) *aRequiresEncryptionWork = true; return NS_OK; } - nsresult nsMsgComposeSecure::GetSMIMEBundleString(const char16_t *name, - nsString &outString) -{ + nsString &outString) { outString.Truncate(); NS_ENSURE_ARG_POINTER(name); NS_ENSURE_TRUE(InitializeSMIMEBundle(), NS_ERROR_FAILURE); - return mSMIMEBundle->GetStringFromName(NS_ConvertUTF16toUTF8(name).get(), outString); + return mSMIMEBundle->GetStringFromName(NS_ConvertUTF16toUTF8(name).get(), + outString); } -nsresult -nsMsgComposeSecure:: -SMIMEBundleFormatStringFromName(const char *name, - const char16_t **params, - uint32_t numParams, - nsAString& outString) -{ +nsresult nsMsgComposeSecure::SMIMEBundleFormatStringFromName( + const char *name, const char16_t **params, uint32_t numParams, + nsAString &outString) { NS_ENSURE_ARG_POINTER(name); - if (!InitializeSMIMEBundle()) - return NS_ERROR_FAILURE; + if (!InitializeSMIMEBundle()) return NS_ERROR_FAILURE; return mSMIMEBundle->FormatStringFromName(name, params, numParams, outString); } -bool nsMsgComposeSecure::InitializeSMIMEBundle() -{ - if (mSMIMEBundle) - return true; +bool nsMsgComposeSecure::InitializeSMIMEBundle() { + if (mSMIMEBundle) return true; nsCOMPtr bundleService = - mozilla::services::GetStringBundleService(); + mozilla::services::GetStringBundleService(); nsresult rv = bundleService->CreateBundle(SMIME_STRBUNDLE_URL, getter_AddRefs(mSMIMEBundle)); NS_ENSURE_SUCCESS(rv, false); @@ -208,33 +187,28 @@ bool nsMsgComposeSecure::InitializeSMIMEBundle() return true; } -void nsMsgComposeSecure::SetError(nsIMsgSendReport *sendReport, const char16_t *bundle_string) -{ - if (!sendReport || !bundle_string) - return; +void nsMsgComposeSecure::SetError(nsIMsgSendReport *sendReport, + const char16_t *bundle_string) { + if (!sendReport || !bundle_string) return; - if (mErrorAlreadyReported) - return; + if (mErrorAlreadyReported) return; mErrorAlreadyReported = true; nsString errorString; nsresult res = GetSMIMEBundleString(bundle_string, errorString); - if (NS_SUCCEEDED(res) && !errorString.IsEmpty()) - { - sendReport->SetMessage(nsIMsgSendReport::process_Current, - errorString.get(), + if (NS_SUCCEEDED(res) && !errorString.IsEmpty()) { + sendReport->SetMessage(nsIMsgSendReport::process_Current, errorString.get(), true); } } -void nsMsgComposeSecure::SetErrorWithParam(nsIMsgSendReport *sendReport, const char *bundle_string, const char *param) -{ - if (!sendReport || !bundle_string || !param) - return; +void nsMsgComposeSecure::SetErrorWithParam(nsIMsgSendReport *sendReport, + const char *bundle_string, + const char *param) { + if (!sendReport || !bundle_string || !param) return; - if (mErrorAlreadyReported) - return; + if (mErrorAlreadyReported) return; mErrorAlreadyReported = true; @@ -243,25 +217,21 @@ void nsMsgComposeSecure::SetErrorWithParam(nsIMsgSendReport *sendReport, const c const char16_t *params[1]; NS_ConvertASCIItoUTF16 ucs2(param); - params[0]= ucs2.get(); + params[0] = ucs2.get(); - res = SMIMEBundleFormatStringFromName(bundle_string, - params, - 1, - errorString); + res = SMIMEBundleFormatStringFromName(bundle_string, params, 1, errorString); - if (NS_SUCCEEDED(res) && !errorString.IsEmpty()) - { - sendReport->SetMessage(nsIMsgSendReport::process_Current, - errorString.get(), + if (NS_SUCCEEDED(res) && !errorString.IsEmpty()) { + sendReport->SetMessage(nsIMsgSendReport::process_Current, errorString.get(), true); } } -nsresult nsMsgComposeSecure::ExtractEncryptionState(nsIMsgIdentity * aIdentity, nsIMsgCompFields * aComposeFields, bool * aSignMessage, bool * aEncrypt) -{ +nsresult nsMsgComposeSecure::ExtractEncryptionState( + nsIMsgIdentity *aIdentity, nsIMsgCompFields *aComposeFields, + bool *aSignMessage, bool *aEncrypt) { if (!aComposeFields && !aIdentity) - return NS_ERROR_FAILURE; // kick out...invalid args.... + return NS_ERROR_FAILURE; // kick out...invalid args.... NS_ENSURE_ARG_POINTER(aSignMessage); NS_ENSURE_ARG_POINTER(aEncrypt); @@ -274,9 +244,8 @@ nsresult nsMsgComposeSecure::ExtractEncryptionState(nsIMsgIdentity * aIdentity, // Select a hash algorithm to sign message // based on subject public key type and size. -static nsresult -GetSigningHashFunction(nsIX509Cert *aSigningCert, int16_t *hashType) -{ +static nsresult GetSigningHashFunction(nsIX509Cert *aSigningCert, + int16_t *hashType) { // Get the signing certificate CERTCertificate *scert = nullptr; if (aSigningCert) { @@ -327,7 +296,7 @@ GetSigningHashFunction(nsIX509Cert *aSigningCert, int16_t *hashType) // NSS only supports SHA-1, SHA-224, and SHA-256 for DSA signatures. // The S/MIME code does not support SHA-224. - if (siglen >= 512) { // 512-bit signature = 256-bit q parameter + if (siglen >= 512) { // 512-bit signature = 256-bit q parameter *hashType = nsICryptoHash::SHA256; } else { *hashType = nsICryptoHash::SHA1; @@ -336,11 +305,11 @@ GetSigningHashFunction(nsIX509Cert *aSigningCert, int16_t *hashType) // For ECDSA, siglen is twice the length of the field size. The security // strength of the key is half the length (in bits) of the field size. - if (siglen >= 1024) { // 1024-bit signature = 512-bit field size + if (siglen >= 1024) { // 1024-bit signature = 512-bit field size *hashType = nsICryptoHash::SHA512; - } else if (siglen >= 768) { // 768-bit signature = 384-bit field size + } else if (siglen >= 768) { // 768-bit signature = 384-bit field size *hashType = nsICryptoHash::SHA384; - } else if (siglen >= 512) { // 512-bit signature = 256-bit field size + } else if (siglen >= 512) { // 512-bit signature = 256-bit field size *hashType = nsICryptoHash::SHA256; } else { *hashType = nsICryptoHash::SHA1; @@ -353,20 +322,19 @@ GetSigningHashFunction(nsIX509Cert *aSigningCert, int16_t *hashType) return NS_OK; } -/* void beginCryptoEncapsulation (in nsOutputFileStream aStream, in boolean aEncrypt, in boolean aSign, in string aRecipeints, in boolean aIsDraft); */ -NS_IMETHODIMP nsMsgComposeSecure::BeginCryptoEncapsulation(nsIOutputStream * aStream, - const char * aRecipients, - nsIMsgCompFields * aCompFields, - nsIMsgIdentity * aIdentity, - nsIMsgSendReport *sendReport, - bool aIsDraft) -{ +/* void beginCryptoEncapsulation (in nsOutputFileStream aStream, in boolean + * aEncrypt, in boolean aSign, in string aRecipeints, in boolean aIsDraft); */ +NS_IMETHODIMP nsMsgComposeSecure::BeginCryptoEncapsulation( + nsIOutputStream *aStream, const char *aRecipients, + nsIMsgCompFields *aCompFields, nsIMsgIdentity *aIdentity, + nsIMsgSendReport *sendReport, bool aIsDraft) { mErrorAlreadyReported = false; nsresult rv = NS_OK; bool encryptMessages = false; bool signMessage = false; - ExtractEncryptionState(aIdentity, aCompFields, &signMessage, &encryptMessages); + ExtractEncryptionState(aIdentity, aCompFields, &signMessage, + &encryptMessages); if (!signMessage && !encryptMessages) return NS_ERROR_FAILURE; @@ -387,7 +355,8 @@ NS_IMETHODIMP nsMsgComposeSecure::BeginCryptoEncapsulation(nsIOutputStream * aSt aIdentity->GetUnicharAttribute("encryption_cert_name", mEncryptionCertName); aIdentity->GetCharAttribute("encryption_cert_dbkey", mEncryptionCertDBKey); - rv = MimeCryptoHackCerts(aRecipients, sendReport, encryptMessages, signMessage, aIdentity); + rv = MimeCryptoHackCerts(aRecipients, sendReport, encryptMessages, + signMessage, aIdentity); if (NS_FAILED(rv)) { goto FAIL; } @@ -397,30 +366,29 @@ NS_IMETHODIMP nsMsgComposeSecure::BeginCryptoEncapsulation(nsIOutputStream * aSt NS_ENSURE_SUCCESS(rv, rv); } - switch (mCryptoState) - { - case mime_crypto_clear_signed: - rv = MimeInitMultipartSigned(true, sendReport); - break; - case mime_crypto_opaque_signed: - PR_ASSERT(0); /* #### no api for this yet */ - rv = NS_ERROR_NOT_IMPLEMENTED; - break; - case mime_crypto_signed_encrypted: - rv = MimeInitEncryption(true, sendReport); - break; - case mime_crypto_encrypted: - rv = MimeInitEncryption(false, sendReport); - break; - case mime_crypto_none: - /* This can happen if mime_crypto_hack_certs() decided to turn off - encryption (by asking the user.) */ - // XXX 1 is not a valid nsresult - rv = static_cast(1); - break; - default: - PR_ASSERT(0); - break; + switch (mCryptoState) { + case mime_crypto_clear_signed: + rv = MimeInitMultipartSigned(true, sendReport); + break; + case mime_crypto_opaque_signed: + PR_ASSERT(0); /* #### no api for this yet */ + rv = NS_ERROR_NOT_IMPLEMENTED; + break; + case mime_crypto_signed_encrypted: + rv = MimeInitEncryption(true, sendReport); + break; + case mime_crypto_encrypted: + rv = MimeInitEncryption(false, sendReport); + break; + case mime_crypto_none: + /* This can happen if mime_crypto_hack_certs() decided to turn off + encryption (by asking the user.) */ + // XXX 1 is not a valid nsresult + rv = static_cast(1); + break; + default: + PR_ASSERT(0); + break; } FAIL: @@ -428,50 +396,50 @@ FAIL: } /* void finishCryptoEncapsulation (in boolean aAbort); */ -NS_IMETHODIMP nsMsgComposeSecure::FinishCryptoEncapsulation(bool aAbort, nsIMsgSendReport *sendReport) -{ +NS_IMETHODIMP nsMsgComposeSecure::FinishCryptoEncapsulation( + bool aAbort, nsIMsgSendReport *sendReport) { nsresult rv = NS_OK; if (!aAbort) { switch (mCryptoState) { - case mime_crypto_clear_signed: - rv = MimeFinishMultipartSigned (true, sendReport); - break; - case mime_crypto_opaque_signed: - PR_ASSERT(0); /* #### no api for this yet */ - rv = NS_ERROR_FAILURE; - break; - case mime_crypto_signed_encrypted: - rv = MimeFinishEncryption (true, sendReport); - break; - case mime_crypto_encrypted: - rv = MimeFinishEncryption (false, sendReport); - break; - default: - PR_ASSERT(0); - rv = NS_ERROR_FAILURE; - break; + case mime_crypto_clear_signed: + rv = MimeFinishMultipartSigned(true, sendReport); + break; + case mime_crypto_opaque_signed: + PR_ASSERT(0); /* #### no api for this yet */ + rv = NS_ERROR_FAILURE; + break; + case mime_crypto_signed_encrypted: + rv = MimeFinishEncryption(true, sendReport); + break; + case mime_crypto_encrypted: + rv = MimeFinishEncryption(false, sendReport); + break; + default: + PR_ASSERT(0); + rv = NS_ERROR_FAILURE; + break; } } return rv; } -nsresult nsMsgComposeSecure::MimeInitMultipartSigned(bool aOuter, nsIMsgSendReport *sendReport) -{ +nsresult nsMsgComposeSecure::MimeInitMultipartSigned( + bool aOuter, nsIMsgSendReport *sendReport) { /* First, construct and write out the multipart/signed MIME header data. */ nsresult rv = NS_OK; char *header = 0; uint32_t L; - rv = make_multipart_signed_header_string(aOuter, &header, - &mMultipartSignedBoundary, mHashType); + rv = make_multipart_signed_header_string( + aOuter, &header, &mMultipartSignedBoundary, mHashType); NS_ENSURE_SUCCESS(rv, rv); L = strlen(header); - if (aOuter){ + if (aOuter) { /* If this is the outer block, write it to the file. */ uint32_t n; rv = mStream->Write(header, L, &n); @@ -481,7 +449,7 @@ nsresult nsMsgComposeSecure::MimeInitMultipartSigned(bool aOuter, nsIMsgSendRepo } } else { /* If this is an inner block, feed it through the crypto stream. */ - rv = MimeCryptoWriteBlock (header, L); + rv = MimeCryptoWriteBlock(header, L); } PR_Free(header); @@ -491,22 +459,22 @@ nsresult nsMsgComposeSecure::MimeInitMultipartSigned(bool aOuter, nsIMsgSendRepo on the object which we are signing. */ - PR_SetError(0,0); + PR_SetError(0, 0); mDataHash = do_CreateInstance("@mozilla.org/security/hash;1", &rv); NS_ENSURE_SUCCESS(rv, rv); rv = mDataHash->Init(mHashType); NS_ENSURE_SUCCESS(rv, rv); - PR_SetError(0,0); + PR_SetError(0, 0); return rv; } -nsresult nsMsgComposeSecure::MimeInitEncryption(bool aSign, nsIMsgSendReport *sendReport) -{ +nsresult nsMsgComposeSecure::MimeInitEncryption(bool aSign, + nsIMsgSendReport *sendReport) { nsresult rv; nsCOMPtr bundleSvc = - mozilla::services::GetStringBundleService(); + mozilla::services::GetStringBundleService(); NS_ENSURE_TRUE(bundleSvc, NS_ERROR_UNEXPECTED); nsCOMPtr sMIMEBundle; @@ -514,34 +482,30 @@ nsresult nsMsgComposeSecure::MimeInitEncryption(bool aSign, nsIMsgSendReport *se bundleSvc->CreateBundle(SMIME_STRBUNDLE_URL, getter_AddRefs(sMIMEBundle)); - if (!sMIMEBundle) - return NS_ERROR_FAILURE; + if (!sMIMEBundle) return NS_ERROR_FAILURE; sMIMEBundle->GetStringFromName("mime_smimeEncryptedContentDesc", mime_smime_enc_content_desc); NS_ConvertUTF16toUTF8 enc_content_desc_utf8(mime_smime_enc_content_desc); nsCOMPtr mimeConverter = - do_GetService(NS_MIME_CONVERTER_CONTRACTID, &rv); + do_GetService(NS_MIME_CONVERTER_CONTRACTID, &rv); NS_ENSURE_SUCCESS(rv, rv); nsCString encodedContentDescription; - mimeConverter->EncodeMimePartIIStr_UTF8(enc_content_desc_utf8, false, - sizeof("Content-Description: "), - nsIMimeConverter::MIME_ENCODED_WORD_SIZE, - encodedContentDescription); + mimeConverter->EncodeMimePartIIStr_UTF8( + enc_content_desc_utf8, false, sizeof("Content-Description: "), + nsIMimeConverter::MIME_ENCODED_WORD_SIZE, encodedContentDescription); /* First, construct and write out the opaque-crypto-blob MIME header data. */ - char *s = - PR_smprintf("Content-Type: " APPLICATION_PKCS7_MIME - "; name=\"smime.p7m\"; smime-type=enveloped-data" CRLF - "Content-Transfer-Encoding: " ENCODING_BASE64 CRLF - "Content-Disposition: attachment" - "; filename=\"smime.p7m\"" CRLF - "Content-Description: %s" CRLF - CRLF, - encodedContentDescription.get()); + char *s = PR_smprintf("Content-Type: " APPLICATION_PKCS7_MIME + "; name=\"smime.p7m\"; smime-type=enveloped-data" CRLF + "Content-Transfer-Encoding: " ENCODING_BASE64 CRLF + "Content-Disposition: attachment" + "; filename=\"smime.p7m\"" CRLF + "Content-Description: %s" CRLF CRLF, + encodedContentDescription.get()); uint32_t L; if (!s) return NS_ERROR_OUT_OF_MEMORY; @@ -567,12 +531,11 @@ nsresult nsMsgComposeSecure::MimeInitEncryption(bool aSign, nsIMsgSendReport *se // Initialize the base64 encoder MOZ_ASSERT(!mCryptoEncoder, "Shouldn't have an encoder already"); - mCryptoEncoder = MimeEncoder::GetBase64Encoder(mime_encoder_output_fn, - this); + mCryptoEncoder = MimeEncoder::GetBase64Encoder(mime_encoder_output_fn, this); /* Initialize the encrypter (and add the sender's cert.) */ PR_ASSERT(mSelfEncryptionCert); - PR_SetError(0,0); + PR_SetError(0, 0); mEncryptionCinfo = do_CreateInstance(NS_CMSMESSAGE_CONTRACTID, &rv); if (NS_FAILED(rv)) return rv; rv = mEncryptionCinfo->CreateEncrypted(mCerts); @@ -586,13 +549,13 @@ nsresult nsMsgComposeSecure::MimeInitEncryption(bool aSign, nsIMsgSendReport *se if (!mBuffer) { mBuffer = new char[eBufferSize]; - if (!mBuffer) - return NS_ERROR_OUT_OF_MEMORY; + if (!mBuffer) return NS_ERROR_OUT_OF_MEMORY; } mBufferedBytes = 0; - rv = mEncryptionContext->Start(mEncryptionCinfo, mime_crypto_write_base64, mCryptoEncoder); + rv = mEncryptionContext->Start(mEncryptionCinfo, mime_crypto_write_base64, + mCryptoEncoder); if (NS_FAILED(rv)) { SetError(sendReport, u"ErrorEncryptMail"); goto FAIL; @@ -606,23 +569,25 @@ nsresult nsMsgComposeSecure::MimeInitEncryption(bool aSign, nsIMsgSendReport *se if (NS_FAILED(rv)) goto FAIL; } - FAIL: +FAIL: return rv; } -nsresult nsMsgComposeSecure::MimeFinishMultipartSigned (bool aOuter, nsIMsgSendReport *sendReport) -{ +nsresult nsMsgComposeSecure::MimeFinishMultipartSigned( + bool aOuter, nsIMsgSendReport *sendReport) { int status; nsresult rv; - nsCOMPtr cinfo = do_CreateInstance(NS_CMSMESSAGE_CONTRACTID, &rv); + nsCOMPtr cinfo = + do_CreateInstance(NS_CMSMESSAGE_CONTRACTID, &rv); NS_ENSURE_SUCCESS(rv, rv); - nsCOMPtr encoder = do_CreateInstance(NS_CMSENCODER_CONTRACTID, &rv); + nsCOMPtr encoder = + do_CreateInstance(NS_CMSENCODER_CONTRACTID, &rv); NS_ENSURE_SUCCESS(rv, rv); - char * header = nullptr; + char *header = nullptr; nsCOMPtr bundleSvc = - mozilla::services::GetStringBundleService(); + mozilla::services::GetStringBundleService(); NS_ENSURE_TRUE(bundleSvc, NS_ERROR_UNEXPECTED); nsCOMPtr sMIMEBundle; @@ -630,8 +595,7 @@ nsresult nsMsgComposeSecure::MimeFinishMultipartSigned (bool aOuter, nsIMsgSendR bundleSvc->CreateBundle(SMIME_STRBUNDLE_URL, getter_AddRefs(sMIMEBundle)); - if (!sMIMEBundle) - return NS_ERROR_FAILURE; + if (!sMIMEBundle) return NS_ERROR_FAILURE; sMIMEBundle->GetStringFromName("mime_smimeSignatureContentDesc", mime_smime_sig_content_desc); @@ -652,18 +616,13 @@ nsresult nsMsgComposeSecure::MimeFinishMultipartSigned (bool aOuter, nsIMsgSendR /* Write out the headers for the signature. */ uint32_t L; - header = - PR_smprintf(CRLF - "--%s" CRLF - "Content-Type: " APPLICATION_PKCS7_SIGNATURE - "; name=\"smime.p7s\"" CRLF - "Content-Transfer-Encoding: " ENCODING_BASE64 CRLF - "Content-Disposition: attachment; " - "filename=\"smime.p7s\"" CRLF - "Content-Description: %s" CRLF - CRLF, - mMultipartSignedBoundary, - sig_content_desc_utf8.get()); + header = PR_smprintf( + CRLF "--%s" CRLF "Content-Type: " APPLICATION_PKCS7_SIGNATURE + "; name=\"smime.p7s\"" CRLF + "Content-Transfer-Encoding: " ENCODING_BASE64 CRLF + "Content-Disposition: attachment; " + "filename=\"smime.p7s\"" CRLF "Content-Description: %s" CRLF CRLF, + mMultipartSignedBoundary, sig_content_desc_utf8.get()); if (!header) { rv = NS_ERROR_OUT_OF_MEMORY; @@ -681,7 +640,7 @@ nsresult nsMsgComposeSecure::MimeFinishMultipartSigned (bool aOuter, nsIMsgSendR } } else { /* If this is an inner block, feed it through the crypto stream. */ - rv = MimeCryptoWriteBlock (header, L); + rv = MimeCryptoWriteBlock(header, L); } PR_Free(header); @@ -691,12 +650,13 @@ nsresult nsMsgComposeSecure::MimeFinishMultipartSigned (bool aOuter, nsIMsgSendR NS_ASSERTION(mHashType, "Hash function for signature has not been set."); - PR_ASSERT (mSelfSigningCert); - PR_SetError(0,0); + PR_ASSERT(mSelfSigningCert); + PR_SetError(0, 0); rv = cinfo->CreateSigned(mSelfSigningCert, mSelfEncryptionCert, - (unsigned char*)hashString.get(), hashString.Length(), mHashType); - if (NS_FAILED(rv)) { + (unsigned char *)hashString.get(), + hashString.Length(), mHashType); + if (NS_FAILED(rv)) { SetError(sendReport, u"ErrorCanNotSignMail"); goto FAIL; } @@ -704,11 +664,11 @@ nsresult nsMsgComposeSecure::MimeFinishMultipartSigned (bool aOuter, nsIMsgSendR // Initialize the base64 encoder for the signature data. MOZ_ASSERT(!mSigEncoder, "Shouldn't already have a mSigEncoder"); mSigEncoder = MimeEncoder::GetBase64Encoder( - (aOuter ? mime_encoder_output_fn : mime_nested_encoder_output_fn), this); + (aOuter ? mime_encoder_output_fn : mime_nested_encoder_output_fn), this); /* Write out the signature. */ - PR_SetError(0,0); + PR_SetError(0, 0); rv = encoder->Start(cinfo, mime_crypto_write_base64, mSigEncoder); if (NS_FAILED(rv)) { SetError(sendReport, u"ErrorCanNotSignMail"); @@ -732,46 +692,44 @@ nsresult nsMsgComposeSecure::MimeFinishMultipartSigned (bool aOuter, nsIMsgSendR /* Now write out the terminating boundary. */ { - uint32_t L; - char *header = PR_smprintf(CRLF "--%s--" CRLF, - mMultipartSignedBoundary); - PR_Free(mMultipartSignedBoundary); - mMultipartSignedBoundary = 0; + uint32_t L; + char *header = PR_smprintf(CRLF "--%s--" CRLF, mMultipartSignedBoundary); + PR_Free(mMultipartSignedBoundary); + mMultipartSignedBoundary = 0; - if (!header) { - rv = NS_ERROR_OUT_OF_MEMORY; - goto FAIL; - } - L = strlen(header); - if (aOuter) { - /* If this is the outer block, write it to the file. */ - uint32_t n; - rv = mStream->Write(header, L, &n); - if (NS_FAILED(rv) || n < L) - // XXX This is -1, not an nsresult - rv = static_cast(MK_MIME_ERROR_WRITING_FILE); - } else { - /* If this is an inner block, feed it through the crypto stream. */ - rv = MimeCryptoWriteBlock (header, L); - } + if (!header) { + rv = NS_ERROR_OUT_OF_MEMORY; + goto FAIL; + } + L = strlen(header); + if (aOuter) { + /* If this is the outer block, write it to the file. */ + uint32_t n; + rv = mStream->Write(header, L, &n); + if (NS_FAILED(rv) || n < L) + // XXX This is -1, not an nsresult + rv = static_cast(MK_MIME_ERROR_WRITING_FILE); + } else { + /* If this is an inner block, feed it through the crypto stream. */ + rv = MimeCryptoWriteBlock(header, L); + } } FAIL: return rv; } - /* Helper function for mime_finish_crypto_encapsulation() to close off an opaque crypto object (for encrypted or signed-and-encrypted messages.) */ -nsresult nsMsgComposeSecure::MimeFinishEncryption (bool aSign, nsIMsgSendReport *sendReport) -{ +nsresult nsMsgComposeSecure::MimeFinishEncryption( + bool aSign, nsIMsgSendReport *sendReport) { nsresult rv; /* If this object is both encrypted and signed, close off the signature first (since it's inside.) */ if (aSign) { - rv = MimeFinishMultipartSigned (false, sendReport); + rv = MimeFinishMultipartSigned(false, sendReport); if (NS_FAILED(rv)) { goto FAIL; } @@ -812,10 +770,9 @@ nsresult nsMsgComposeSecure::MimeFinishEncryption (bool aSign, nsIMsgSendReport uint32_t n; rv = mStream->Write(CRLF, 2, &n); - if (NS_FAILED(rv) || n < 2) - rv = NS_ERROR_FAILURE; + if (NS_FAILED(rv) || n < 2) rv = NS_ERROR_FAILURE; - FAIL: +FAIL: return rv; } @@ -823,10 +780,8 @@ nsresult nsMsgComposeSecure::MimeFinishEncryption (bool aSign, nsIMsgSendReport */ nsresult nsMsgComposeSecure::MimeCryptoHackCerts(const char *aRecipients, nsIMsgSendReport *sendReport, - bool aEncrypt, - bool aSign, - nsIMsgIdentity *aIdentity) -{ + bool aEncrypt, bool aSign, + nsIMsgIdentity *aIdentity) { nsCOMPtr certdb = do_GetService(NS_X509CERTDB_CONTRACTID); nsresult res; @@ -854,19 +809,16 @@ nsresult nsMsgComposeSecure::MimeCryptoHackCerts(const char *aRecipients, res = certdb->FindCertByDBKey(mEncryptionCertDBKey, getter_AddRefs(mSelfEncryptionCert)); if (NS_SUCCEEDED(res) && mSelfEncryptionCert && - (certVerifier->VerifyCert(mSelfEncryptionCert->GetCert(), - certificateUsageEmailRecipient, - mozilla::pkix::Now(), - nullptr, nullptr, - builtChain, - // Only local checks can run on the main thread. - CertVerifier::FLAG_LOCAL_ONLY) - != mozilla::pkix::Success)) { + (certVerifier->VerifyCert( + mSelfEncryptionCert->GetCert(), certificateUsageEmailRecipient, + mozilla::pkix::Now(), nullptr, nullptr, builtChain, + // Only local checks can run on the main thread. + CertVerifier::FLAG_LOCAL_ONLY) != mozilla::pkix::Success)) { // not suitable for encryption, so unset cert and clear pref mSelfEncryptionCert = nullptr; mEncryptionCertDBKey.Truncate(); aIdentity->SetCharAttribute("encryption_cert_dbkey", - mEncryptionCertDBKey); + mEncryptionCertDBKey); } } @@ -875,14 +827,11 @@ nsresult nsMsgComposeSecure::MimeCryptoHackCerts(const char *aRecipients, res = certdb->FindCertByDBKey(mSigningCertDBKey, getter_AddRefs(mSelfSigningCert)); if (NS_SUCCEEDED(res) && mSelfSigningCert && - (certVerifier->VerifyCert(mSelfSigningCert->GetCert(), - certificateUsageEmailSigner, - mozilla::pkix::Now(), - nullptr, nullptr, - builtChain, - // Only local checks can run on the main thread. - CertVerifier::FLAG_LOCAL_ONLY) - != mozilla::pkix::Success)) { + (certVerifier->VerifyCert( + mSelfSigningCert->GetCert(), certificateUsageEmailSigner, + mozilla::pkix::Now(), nullptr, nullptr, builtChain, + // Only local checks can run on the main thread. + CertVerifier::FLAG_LOCAL_ONLY) != mozilla::pkix::Success)) { // not suitable for signing, so unset cert and clear pref mSelfSigningCert = nullptr; mSigningCertDBKey.Truncate(); @@ -901,7 +850,6 @@ nsresult nsMsgComposeSecure::MimeCryptoHackCerts(const char *aRecipients, return NS_ERROR_FAILURE; } - if (aEncrypt && mSelfEncryptionCert) { // Make sure self's configured cert is prepared for being used // as an email recipient cert. @@ -919,7 +867,7 @@ nsresult nsMsgComposeSecure::MimeCryptoHackCerts(const char *aRecipients, if (aEncrypt) { nsTArray mailboxes; ExtractEmails(EncodedHeader(nsDependentCString(aRecipients)), - UTF16ArrayAdapter<>(mailboxes)); + UTF16ArrayAdapter<>(mailboxes)); uint32_t count = mailboxes.Length(); bool already_added_self_cert = false; @@ -929,27 +877,26 @@ nsresult nsMsgComposeSecure::MimeCryptoHackCerts(const char *aRecipients, ToLowerCase(mailboxes[i], mailbox_lowercase); nsCOMPtr cert; // TODO: allow user to override and go ahead with an invalid certificate - res = FindCertByEmailAddress(mailbox_lowercase, true, getter_AddRefs(cert)); + res = + FindCertByEmailAddress(mailbox_lowercase, true, getter_AddRefs(cert)); if (NS_FAILED(res)) { // Failure to find a valid encryption cert is fatal. // Here I assume that mailbox is ascii rather than utf8. - SetErrorWithParam(sendReport, - "MissingRecipientEncryptionCert", + SetErrorWithParam(sendReport, "MissingRecipientEncryptionCert", mailboxes[i].get()); return res; } - /* #### see if recipient requests `signedData'. - if (...) no_clearsigning_p = true; - (This is the only reason we even bother looking up the certs - of the recipients if we're sending a signed-but-not-encrypted - message.) - */ + /* #### see if recipient requests `signedData'. + if (...) no_clearsigning_p = true; + (This is the only reason we even bother looking up the certs + of the recipients if we're sending a signed-but-not-encrypted + message.) + */ bool isSame; - if (NS_SUCCEEDED(cert->Equals(mSelfEncryptionCert, &isSame)) - && isSame) { + if (NS_SUCCEEDED(cert->Equals(mSelfEncryptionCert, &isSame)) && isSame) { already_added_self_cert = true; } @@ -963,8 +910,8 @@ nsresult nsMsgComposeSecure::MimeCryptoHackCerts(const char *aRecipients, return res; } -NS_IMETHODIMP nsMsgComposeSecure::MimeCryptoWriteBlock (const char *buf, int32_t size) -{ +NS_IMETHODIMP nsMsgComposeSecure::MimeCryptoWriteBlock(const char *buf, + int32_t size) { int status = 0; nsresult rv; @@ -978,9 +925,8 @@ NS_IMETHODIMP nsMsgComposeSecure::MimeCryptoWriteBlock (const char *buf, int32_t */ if (size >= 5 && buf[0] == 'F' && !strncmp(buf, "From ", 5)) { char mangle[] = ">"; - nsresult res = MimeCryptoWriteBlock (mangle, 1); - if (NS_FAILED(res)) - return res; + nsresult res = MimeCryptoWriteBlock(mangle, 1); + if (NS_FAILED(res)) return res; // This value will actually be cast back to an nsresult before use, so this // cast is reasonable under the circumstances. status = static_cast(res); @@ -989,13 +935,13 @@ NS_IMETHODIMP nsMsgComposeSecure::MimeCryptoWriteBlock (const char *buf, int32_t /* If we're signing, or signing-and-encrypting, feed this data into the computation of the hash. */ if (mDataHash) { - PR_SetError(0,0); - mDataHash->Update((const uint8_t*) buf, size); + PR_SetError(0, 0); + mDataHash->Update((const uint8_t *)buf, size); status = PR_GetError(); if (status < 0) goto FAIL; } - PR_SetError(0,0); + PR_SetError(0, 0); if (mEncryptionContext) { /* If we're encrypting, or signing-and-encrypting, write this data by filtering it through the crypto library. */ @@ -1006,9 +952,10 @@ NS_IMETHODIMP nsMsgComposeSecure::MimeCryptoWriteBlock (const char *buf, int32_t while (inputBytesLeft) { const uint32_t spaceLeftInBuffer = eBufferSize - mBufferedBytes; - const uint32_t bytesToAppend = std::min(inputBytesLeft, spaceLeftInBuffer); + const uint32_t bytesToAppend = + std::min(inputBytesLeft, spaceLeftInBuffer); - memcpy(mBuffer+mBufferedBytes, inputBytesIterator, bytesToAppend); + memcpy(mBuffer + mBufferedBytes, inputBytesIterator, bytesToAppend); mBufferedBytes += bytesToAppend; inputBytesIterator += bytesToAppend; @@ -1036,7 +983,7 @@ NS_IMETHODIMP nsMsgComposeSecure::MimeCryptoWriteBlock (const char *buf, int32_t return static_cast(MK_MIME_ERROR_WRITING_FILE); } } - FAIL: +FAIL: // XXX status sometimes has invalid nsresults like -1 or PR_GetError() // assigned to it return static_cast(status); @@ -1047,49 +994,44 @@ NS_IMETHODIMP nsMsgComposeSecure::MimeCryptoWriteBlock (const char *buf, int32_t of a multipart object. The boundary itself is also returned (so that the caller knows what to write to close it off.) */ -static nsresult -make_multipart_signed_header_string(bool outer_p, - char **header_return, - char **boundary_return, - int16_t hash_type) -{ +static nsresult make_multipart_signed_header_string(bool outer_p, + char **header_return, + char **boundary_return, + int16_t hash_type) { const char *hashStr; *header_return = 0; *boundary_return = mime_make_separator("ms"); - if (!*boundary_return) - return NS_ERROR_OUT_OF_MEMORY; + if (!*boundary_return) return NS_ERROR_OUT_OF_MEMORY; switch (hash_type) { - case nsICryptoHash::SHA1: - hashStr = PARAM_MICALG_SHA1; - break; - case nsICryptoHash::SHA256: - hashStr = PARAM_MICALG_SHA256; - break; - case nsICryptoHash::SHA384: - hashStr = PARAM_MICALG_SHA384; - break; - case nsICryptoHash::SHA512: - hashStr = PARAM_MICALG_SHA512; - break; - default: - return NS_ERROR_INVALID_ARG; + case nsICryptoHash::SHA1: + hashStr = PARAM_MICALG_SHA1; + break; + case nsICryptoHash::SHA256: + hashStr = PARAM_MICALG_SHA256; + break; + case nsICryptoHash::SHA384: + hashStr = PARAM_MICALG_SHA384; + break; + case nsICryptoHash::SHA512: + hashStr = PARAM_MICALG_SHA512; + break; + default: + return NS_ERROR_INVALID_ARG; } - *header_return = PR_smprintf( - "Content-Type: " MULTIPART_SIGNED "; " - "protocol=\"" APPLICATION_PKCS7_SIGNATURE "\"; " - "micalg=%s; " - "boundary=\"%s\"" CRLF - CRLF - "%s%s" - "--%s" CRLF, - hashStr, - *boundary_return, - (outer_p ? crypto_multipart_blurb : ""), - (outer_p ? CRLF CRLF : ""), - *boundary_return); + *header_return = PR_smprintf("Content-Type: " MULTIPART_SIGNED + "; " + "protocol=\"" APPLICATION_PKCS7_SIGNATURE + "\"; " + "micalg=%s; " + "boundary=\"%s\"" CRLF CRLF + "%s%s" + "--%s" CRLF, + hashStr, *boundary_return, + (outer_p ? crypto_multipart_blurb : ""), + (outer_p ? CRLF CRLF : ""), *boundary_return); if (!*header_return) { PR_Free(*boundary_return); @@ -1109,27 +1051,24 @@ make_multipart_signed_header_string(bool outer_p, it's used to write the encoded representation of the signature. The only difference is which MimeEncoder object is used. */ -static void -mime_crypto_write_base64 (void *closure, const char *buf, unsigned long size) -{ - MimeEncoder *encoder = (MimeEncoder *) closure; +static void mime_crypto_write_base64(void *closure, const char *buf, + unsigned long size) { + MimeEncoder *encoder = (MimeEncoder *)closure; nsresult rv = encoder->Write(buf, size); PR_SetError(NS_FAILED(rv) ? static_cast(rv) : 0, 0); } - /* Used as the output function of MimeEncoder -- when we have generated the signature for a multipart/signed object, this is used to write the base64-encoded representation of the signature to the file. */ // TODO: size should probably be converted to uint32_t -nsresult mime_encoder_output_fn(const char *buf, int32_t size, void *closure) -{ - nsMsgComposeSecure *state = (nsMsgComposeSecure *) closure; +nsresult mime_encoder_output_fn(const char *buf, int32_t size, void *closure) { + nsMsgComposeSecure *state = (nsMsgComposeSecure *)closure; nsCOMPtr stream; state->GetOutputStream(getter_AddRefs(stream)); uint32_t n; - nsresult rv = stream->Write((char *) buf, size, &n); + nsresult rv = stream->Write((char *)buf, size, &n); if (NS_FAILED(rv) || n < (uint32_t)size) return NS_ERROR_FAILURE; else @@ -1141,10 +1080,9 @@ nsresult mime_encoder_output_fn(const char *buf, int32_t size, void *closure) signature should be fed into the crypto engine, rather than being written directly to the file. */ -static nsresult -mime_nested_encoder_output_fn (const char *buf, int32_t size, void *closure) -{ - nsMsgComposeSecure *state = (nsMsgComposeSecure *) closure; +static nsresult mime_nested_encoder_output_fn(const char *buf, int32_t size, + void *closure) { + nsMsgComposeSecure *state = (nsMsgComposeSecure *)closure; // Copy to new null-terminated string so JS glue doesn't crash when // MimeCryptoWriteBlock() is implemented in JS. @@ -1154,10 +1092,9 @@ mime_nested_encoder_output_fn (const char *buf, int32_t size, void *closure) } NS_IMETHODIMP -nsMsgComposeSecure::FindCertByEmailAddress(const nsACString& aEmailAddress, +nsMsgComposeSecure::FindCertByEmailAddress(const nsACString &aEmailAddress, bool aRequireValidCert, - nsIX509Cert** _retval) -{ + nsIX509Cert **_retval) { nsresult rv = BlockUntilLoadableRootsLoaded(); if (NS_FAILED(rv)) { return rv; @@ -1166,40 +1103,34 @@ nsMsgComposeSecure::FindCertByEmailAddress(const nsACString& aEmailAddress, RefPtr certVerifier(GetDefaultCertVerifier()); NS_ENSURE_TRUE(certVerifier, NS_ERROR_UNEXPECTED); - const nsCString& flatEmailAddress = PromiseFlatCString(aEmailAddress); + const nsCString &flatEmailAddress = PromiseFlatCString(aEmailAddress); UniqueCERTCertList certlist( - PK11_FindCertsFromEmailAddress(flatEmailAddress.get(), nullptr)); - if (!certlist) - return NS_ERROR_FAILURE; + PK11_FindCertsFromEmailAddress(flatEmailAddress.get(), nullptr)); + if (!certlist) return NS_ERROR_FAILURE; // certlist now contains certificates with the right email address, // but they might not have the correct usage or might even be invalid if (CERT_LIST_END(CERT_LIST_HEAD(certlist), certlist)) - return NS_ERROR_FAILURE; // no certs found + return NS_ERROR_FAILURE; // no certs found CERTCertListNode *node; // search for a valid certificate - for (node = CERT_LIST_HEAD(certlist); - !CERT_LIST_END(node, certlist); + for (node = CERT_LIST_HEAD(certlist); !CERT_LIST_END(node, certlist); node = CERT_LIST_NEXT(node)) { UniqueCERTCertList unusedCertChain; - mozilla::pkix::Result result = - certVerifier->VerifyCert(node->cert, certificateUsageEmailRecipient, - mozilla::pkix::Now(), - nullptr /*XXX pinarg*/, - nullptr /*hostname*/, - unusedCertChain, - // Only local checks can run on the main thread. - CertVerifier::FLAG_LOCAL_ONLY); + mozilla::pkix::Result result = certVerifier->VerifyCert( + node->cert, certificateUsageEmailRecipient, mozilla::pkix::Now(), + nullptr /*XXX pinarg*/, nullptr /*hostname*/, unusedCertChain, + // Only local checks can run on the main thread. + CertVerifier::FLAG_LOCAL_ONLY); if (result == mozilla::pkix::Success) { break; } } - if (CERT_LIST_END(node, certlist)) { // no valid cert found - if (aRequireValidCert) - return NS_ERROR_FAILURE; + if (CERT_LIST_END(node, certlist)) { // no valid cert found + if (aRequireValidCert) return NS_ERROR_FAILURE; // Get the first non-valid then. node = CERT_LIST_HEAD(certlist); @@ -1208,8 +1139,7 @@ nsMsgComposeSecure::FindCertByEmailAddress(const nsACString& aEmailAddress, // |node| now contains the first valid (if aRequireValidCert true) // certificate with correct usage. RefPtr nssCert = nsNSSCertificate::Create(node->cert); - if (!nssCert) - return NS_ERROR_OUT_OF_MEMORY; + if (!nssCert) return NS_ERROR_OUT_OF_MEMORY; nssCert.forget(_retval); return NS_OK; diff --git a/mailnews/extensions/smime/src/nsMsgComposeSecure.h b/mailnews/extensions/smime/src/nsMsgComposeSecure.h index 79b317bb83..5d5611696c 100644 --- a/mailnews/extensions/smime/src/nsMsgComposeSecure.h +++ b/mailnews/extensions/smime/src/nsMsgComposeSecure.h @@ -23,7 +23,7 @@ namespace mozilla { namespace mailnews { class MimeEncoder; } -} +} // namespace mozilla typedef enum { mime_crypto_none, /* normal unencapsulated MIME message */ @@ -33,31 +33,36 @@ typedef enum { mime_crypto_signed_encrypted /* application/x-pkcs7-mime */ } mimeDeliveryCryptoState; -class nsMsgComposeSecure : public nsIMsgComposeSecure -{ -public: +class nsMsgComposeSecure : public nsIMsgComposeSecure { + public: NS_DECL_ISUPPORTS NS_DECL_NSIMSGCOMPOSESECURE nsMsgComposeSecure(); - void GetOutputStream(nsIOutputStream **stream) { NS_IF_ADDREF(*stream = mStream);} + void GetOutputStream(nsIOutputStream **stream) { + NS_IF_ADDREF(*stream = mStream); + } nsresult GetSMIMEBundleString(const char16_t *name, nsString &outString); -private: + private: virtual ~nsMsgComposeSecure(); typedef mozilla::mailnews::MimeEncoder MimeEncoder; nsresult MimeInitMultipartSigned(bool aOuter, nsIMsgSendReport *sendReport); nsresult MimeInitEncryption(bool aSign, nsIMsgSendReport *sendReport); - nsresult MimeFinishMultipartSigned (bool aOuter, nsIMsgSendReport *sendReport); - nsresult MimeFinishEncryption (bool aSign, nsIMsgSendReport *sendReport); - nsresult MimeCryptoHackCerts(const char *aRecipients, nsIMsgSendReport *sendReport, bool aEncrypt, bool aSign, nsIMsgIdentity *aIdentity); + nsresult MimeFinishMultipartSigned(bool aOuter, nsIMsgSendReport *sendReport); + nsresult MimeFinishEncryption(bool aSign, nsIMsgSendReport *sendReport); + nsresult MimeCryptoHackCerts(const char *aRecipients, + nsIMsgSendReport *sendReport, bool aEncrypt, + bool aSign, nsIMsgIdentity *aIdentity); bool InitializeSMIMEBundle(); nsresult SMIMEBundleFormatStringFromName(const char *name, const char16_t **params, uint32_t numParams, - nsAString& outString); - nsresult ExtractEncryptionState(nsIMsgIdentity * aIdentity, nsIMsgCompFields * aComposeFields, bool * aSignMessage, bool * aEncrypt); + nsAString &outString); + nsresult ExtractEncryptionState(nsIMsgIdentity *aIdentity, + nsIMsgCompFields *aComposeFields, + bool *aSignMessage, bool *aEncrypt); bool mSignMessage; bool mAlwaysEncryptMessage; @@ -81,13 +86,14 @@ private: nsAutoPtr mCryptoEncoder; bool mIsDraft; - enum {eBufferSize = 8192}; + enum { eBufferSize = 8192 }; char *mBuffer; uint32_t mBufferedBytes; bool mErrorAlreadyReported; void SetError(nsIMsgSendReport *sendReport, const char16_t *bundle_string); - void SetErrorWithParam(nsIMsgSendReport *sendReport, const char *bundle_string, const char *param); + void SetErrorWithParam(nsIMsgSendReport *sendReport, + const char *bundle_string, const char *param); }; #endif diff --git a/mailnews/extensions/smime/src/nsMsgSMIMECID.h b/mailnews/extensions/smime/src/nsMsgSMIMECID.h index 6ee59bd323..e6061f9b5d 100644 --- a/mailnews/extensions/smime/src/nsMsgSMIMECID.h +++ b/mailnews/extensions/smime/src/nsMsgSMIMECID.h @@ -6,25 +6,31 @@ #ifndef nsMsgSMIMECID_h__ #define nsMsgSMIMECID_h__ -#define NS_MSGCOMPOSESECURE_CID \ -{ /* 54976882-7421-4286-8ecc-46373f15d7b5 */ \ - 0x54976882, 0x7421, 0x4286, \ - {0x8e, 0xcc, 0x46, 0x37, 0x3f, 0x15, 0xd7, 0xb5 }} +#define NS_MSGCOMPOSESECURE_CID \ + { /* 54976882-7421-4286-8ecc-46373f15d7b5 */ \ + 0x54976882, 0x7421, 0x4286, { \ + 0x8e, 0xcc, 0x46, 0x37, 0x3f, 0x15, 0xd7, 0xb5 \ + } \ + } #define NS_SMIMEJSHELPER_CONTRACTID \ "@mozilla.org/messenger-smime/smimejshelper;1" -#define NS_SMIMEJSJELPER_CID \ -{ /* d57d928c-60e4-4f81-999d-5c762e611205 */ \ - 0xd57d928c, 0x60e4, 0x4f81, \ - {0x99, 0x9d, 0x5c, 0x76, 0x2e, 0x61, 0x12, 0x05 }} +#define NS_SMIMEJSJELPER_CID \ + { /* d57d928c-60e4-4f81-999d-5c762e611205 */ \ + 0xd57d928c, 0x60e4, 0x4f81, { \ + 0x99, 0x9d, 0x5c, 0x76, 0x2e, 0x61, 0x12, 0x05 \ + } \ + } -#define NS_SMIMEENCRYPTURISERVICE_CONTRACTID \ +#define NS_SMIMEENCRYPTURISERVICE_CONTRACTID \ "@mozilla.org/messenger-smime/smime-encrypted-uris-service;1" -#define NS_SMIMEENCRYPTURISERVICE_CID \ -{ /* a0134d58-018f-4d40-a099-fa079e5024a6 */ \ - 0xa0134d58, 0x018f, 0x4d40, \ - {0xa0, 0x99, 0xfa, 0x07, 0x9e, 0x50, 0x24, 0xa6 }} +#define NS_SMIMEENCRYPTURISERVICE_CID \ + { /* a0134d58-018f-4d40-a099-fa079e5024a6 */ \ + 0xa0134d58, 0x018f, 0x4d40, { \ + 0xa0, 0x99, 0xfa, 0x07, 0x9e, 0x50, 0x24, 0xa6 \ + } \ + } -#endif // nsMsgSMIMECID_h__ +#endif // nsMsgSMIMECID_h__ diff --git a/mailnews/extensions/smime/src/nsSMimeJSHelper.cpp b/mailnews/extensions/smime/src/nsSMimeJSHelper.cpp index 76880b26f3..110f95b558 100644 --- a/mailnews/extensions/smime/src/nsSMimeJSHelper.cpp +++ b/mailnews/extensions/smime/src/nsSMimeJSHelper.cpp @@ -20,24 +20,14 @@ using namespace mozilla::mailnews; NS_IMPL_ISUPPORTS(nsSMimeJSHelper, nsISMimeJSHelper) -nsSMimeJSHelper::nsSMimeJSHelper() -{ -} +nsSMimeJSHelper::nsSMimeJSHelper() {} -nsSMimeJSHelper::~nsSMimeJSHelper() -{ -} +nsSMimeJSHelper::~nsSMimeJSHelper() {} NS_IMETHODIMP nsSMimeJSHelper::GetRecipientCertsInfo( - nsIMsgCompFields *compFields, - uint32_t *count, - char16_t ***emailAddresses, - int32_t **certVerification, - char16_t ***certIssuedInfos, - char16_t ***certExpiresInfos, - nsIX509Cert ***certs, - bool *canEncrypt) -{ + nsIMsgCompFields *compFields, uint32_t *count, char16_t ***emailAddresses, + int32_t **certVerification, char16_t ***certIssuedInfos, + char16_t ***certExpiresInfos, nsIX509Cert ***certs, bool *canEncrypt) { NS_ENSURE_ARG_POINTER(count); *count = 0; @@ -62,29 +52,30 @@ NS_IMETHODIMP nsSMimeJSHelper::GetRecipientCertsInfo( *canEncrypt = false; rv = NS_OK; - if (mailbox_count) - { + if (mailbox_count) { nsCOMPtr composeSecure = - do_CreateInstance(NS_MSGCOMPOSESECURE_CONTRACTID, &rv); + do_CreateInstance(NS_MSGCOMPOSESECURE_CONTRACTID, &rv); NS_ENSURE_SUCCESS(rv, rv); - char16_t **outEA = static_cast(moz_xmalloc(mailbox_count * sizeof(char16_t *))); - int32_t *outCV = static_cast(moz_xmalloc(mailbox_count * sizeof(int32_t))); - char16_t **outCII = static_cast(moz_xmalloc(mailbox_count * sizeof(char16_t *))); - char16_t **outCEI = static_cast(moz_xmalloc(mailbox_count * sizeof(char16_t *))); - nsIX509Cert **outCerts = static_cast(moz_xmalloc(mailbox_count * sizeof(nsIX509Cert *))); + char16_t **outEA = static_cast( + moz_xmalloc(mailbox_count * sizeof(char16_t *))); + int32_t *outCV = + static_cast(moz_xmalloc(mailbox_count * sizeof(int32_t))); + char16_t **outCII = static_cast( + moz_xmalloc(mailbox_count * sizeof(char16_t *))); + char16_t **outCEI = static_cast( + moz_xmalloc(mailbox_count * sizeof(char16_t *))); + nsIX509Cert **outCerts = static_cast( + moz_xmalloc(mailbox_count * sizeof(nsIX509Cert *))); - if (!outEA || !outCV || !outCII || !outCEI || !outCerts) - { + if (!outEA || !outCV || !outCII || !outCEI || !outCerts) { free(outEA); free(outCV); free(outCII); free(outCEI); free(outCerts); rv = NS_ERROR_OUT_OF_MEMORY; - } - else - { + } else { char16_t **iEA = outEA; int32_t *iCV = outCV; char16_t **iCII = outCII; @@ -94,10 +85,8 @@ NS_IMETHODIMP nsSMimeJSHelper::GetRecipientCertsInfo( bool found_blocker = false; bool memory_failure = false; - for (uint32_t i = 0; - i < mailbox_count; - ++i, ++iEA, ++iCV, ++iCII, ++iCEI, ++iCert) - { + for (uint32_t i = 0; i < mailbox_count; + ++i, ++iEA, ++iCV, ++iCII, ++iCEI, ++iCert) { *iCert = nullptr; *iCV = 0; *iCII = nullptr; @@ -120,8 +109,7 @@ NS_IMETHODIMP nsSMimeJSHelper::GetRecipientCertsInfo( nsCOMPtr cert; if (NS_SUCCEEDED(composeSecure->FindCertByEmailAddress( - email_lowercase, false, getter_AddRefs(cert)))) - { + email_lowercase, false, getter_AddRefs(cert)))) { cert.forget(iCert); nsCOMPtr validity; @@ -130,8 +118,7 @@ NS_IMETHODIMP nsSMimeJSHelper::GetRecipientCertsInfo( if (NS_SUCCEEDED(rv)) { nsString id, ed; - if (NS_SUCCEEDED(validity->GetNotBeforeLocalDay(id))) - { + if (NS_SUCCEEDED(validity->GetNotBeforeLocalDay(id))) { *iCII = ToNewUnicode(id); if (!*iCII) { memory_failure = true; @@ -139,8 +126,7 @@ NS_IMETHODIMP nsSMimeJSHelper::GetRecipientCertsInfo( } } - if (NS_SUCCEEDED(validity->GetNotAfterLocalDay(ed))) - { + if (NS_SUCCEEDED(validity->GetNotAfterLocalDay(ed))) { *iCEI = ToNewUnicode(ed); if (!*iCEI) { memory_failure = true; @@ -148,9 +134,7 @@ NS_IMETHODIMP nsSMimeJSHelper::GetRecipientCertsInfo( } } } - } - else - { + } else { found_blocker = true; } } @@ -162,10 +146,8 @@ NS_IMETHODIMP nsSMimeJSHelper::GetRecipientCertsInfo( NS_FREE_XPCOM_ISUPPORTS_POINTER_ARRAY(mailbox_count, outCerts); free(outCV); rv = NS_ERROR_OUT_OF_MEMORY; - } - else { - if (mailbox_count > 0 && !found_blocker) - { + } else { + if (mailbox_count > 0 && !found_blocker) { *canEncrypt = true; } @@ -180,11 +162,9 @@ NS_IMETHODIMP nsSMimeJSHelper::GetRecipientCertsInfo( return rv; } -NS_IMETHODIMP nsSMimeJSHelper::GetNoCertAddresses( - nsIMsgCompFields *compFields, - uint32_t *count, - char16_t ***emailAddresses) -{ +NS_IMETHODIMP nsSMimeJSHelper::GetNoCertAddresses(nsIMsgCompFields *compFields, + uint32_t *count, + char16_t ***emailAddresses) { NS_ENSURE_ARG_POINTER(count); *count = 0; @@ -198,30 +178,26 @@ NS_IMETHODIMP nsSMimeJSHelper::GetNoCertAddresses( uint32_t mailbox_count = mailboxes.Length(); - if (!mailbox_count) - { + if (!mailbox_count) { *count = 0; *emailAddresses = nullptr; return NS_OK; } nsCOMPtr composeSecure = - do_CreateInstance(NS_MSGCOMPOSESECURE_CONTRACTID, &rv); + do_CreateInstance(NS_MSGCOMPOSESECURE_CONTRACTID, &rv); NS_ENSURE_SUCCESS(rv, rv); uint32_t missing_count = 0; bool *haveCert = new bool[mailbox_count]; - if (!haveCert) - { + if (!haveCert) { return NS_ERROR_OUT_OF_MEMORY; } rv = NS_OK; - if (mailbox_count) - { - for (uint32_t i = 0; i < mailbox_count; ++i) - { + if (mailbox_count) { + for (uint32_t i = 0; i < mailbox_count; ++i) { haveCert[i] = false; nsCString email_lowercase; @@ -229,37 +205,30 @@ NS_IMETHODIMP nsSMimeJSHelper::GetNoCertAddresses( nsCOMPtr cert; if (NS_SUCCEEDED(composeSecure->FindCertByEmailAddress( - email_lowercase, true, getter_AddRefs(cert)))) + email_lowercase, true, getter_AddRefs(cert)))) haveCert[i] = true; - if (!haveCert[i]) - ++missing_count; + if (!haveCert[i]) ++missing_count; } } *count = missing_count; - if (missing_count) - { - char16_t **outEA = static_cast(moz_xmalloc(missing_count * sizeof(char16_t *))); - if (!outEA ) - { + if (missing_count) { + char16_t **outEA = static_cast( + moz_xmalloc(missing_count * sizeof(char16_t *))); + if (!outEA) { rv = NS_ERROR_OUT_OF_MEMORY; - } - else - { + } else { char16_t **iEA = outEA; bool memory_failure = false; - for (uint32_t i = 0; i < mailbox_count; ++i) - { - if (!haveCert[i]) - { + for (uint32_t i = 0; i < mailbox_count; ++i) { + if (!haveCert[i]) { if (memory_failure) { *iEA = nullptr; - } - else { + } else { *iEA = ToNewUnicode(NS_ConvertUTF8toUTF16(mailboxes[i])); if (!*iEA) { memory_failure = true; @@ -272,45 +241,36 @@ NS_IMETHODIMP nsSMimeJSHelper::GetNoCertAddresses( if (memory_failure) { NS_FREE_XPCOM_ALLOCATED_POINTER_ARRAY(missing_count, outEA); rv = NS_ERROR_OUT_OF_MEMORY; - } - else { + } else { *emailAddresses = outEA; } } - } - else - { + } else { *emailAddresses = nullptr; } - delete [] haveCert; + delete[] haveCert; return rv; } nsresult nsSMimeJSHelper::getMailboxList(nsIMsgCompFields *compFields, - nsTArray &mailboxes) -{ - if (!compFields) - return NS_ERROR_INVALID_ARG; + nsTArray &mailboxes) { + if (!compFields) return NS_ERROR_INVALID_ARG; nsresult res; nsString to, cc, bcc, ng; res = compFields->GetTo(to); - if (NS_FAILED(res)) - return res; + if (NS_FAILED(res)) return res; res = compFields->GetCc(cc); - if (NS_FAILED(res)) - return res; + if (NS_FAILED(res)) return res; res = compFields->GetBcc(bcc); - if (NS_FAILED(res)) - return res; + if (NS_FAILED(res)) return res; res = compFields->GetNewsgroups(ng); - if (NS_FAILED(res)) - return res; + if (NS_FAILED(res)) return res; { nsCString all_recipients; @@ -330,11 +290,10 @@ nsresult nsSMimeJSHelper::getMailboxList(nsIMsgCompFields *compFields, all_recipients.Append(','); } - if (!ng.IsEmpty()) - all_recipients.Append(NS_ConvertUTF16toUTF8(ng)); + if (!ng.IsEmpty()) all_recipients.Append(NS_ConvertUTF16toUTF8(ng)); ExtractEmails(EncodedHeader(all_recipients), - UTF16ArrayAdapter<>(mailboxes)); + UTF16ArrayAdapter<>(mailboxes)); } return NS_OK; diff --git a/mailnews/extensions/smime/src/nsSMimeJSHelper.h b/mailnews/extensions/smime/src/nsSMimeJSHelper.h index 8606d75dbb..a769c4958e 100644 --- a/mailnews/extensions/smime/src/nsSMimeJSHelper.h +++ b/mailnews/extensions/smime/src/nsSMimeJSHelper.h @@ -8,18 +8,17 @@ #include "nsISMimeJSHelper.h" #include "nsIMsgCompFields.h" -class nsSMimeJSHelper : public nsISMimeJSHelper -{ -public: +class nsSMimeJSHelper : public nsISMimeJSHelper { + public: NS_DECL_ISUPPORTS NS_DECL_NSISMIMEJSHELPER nsSMimeJSHelper(); -private: + private: virtual ~nsSMimeJSHelper(); nsresult getMailboxList(nsIMsgCompFields *compFields, - nsTArray &mailboxes); + nsTArray &mailboxes); }; #endif