mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1003731 - Removal of transliteration. r=hsivonen 

IGNORE IDL

--HG--
extra : rebase_source : 823a82df0b5590799b7e3e010c9f0aff8141f091
extra : histedit_source : 7a16332b7c8955a4b097467829b4300a128c59f8
This commit is contained in:
Andy Pusch 2015-04-22 00:21:00 -04:00
Родитель 2a6275c310
Коммит e193f90a5f
12 изменённых файлов: 119 добавлений и 3230 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

52
intl/uconv/tests/unit/test_bug365345.js
Просмотреть файл

@ -1,52 +0,0 @@
/* Tests conversion of unrepresented characters that should be transliterated
* to spaces (bug 365345), and some others from transliterate.properties while
* I'm here
*/
const inSpace = "Hello Space";
const inEnSpace = "Hello\u2002EnSpace";
const inEmSpace = "Hello\u2003EmSpace";
const inEuro = "Hello\u20ACEuro";
const inTamil1000 = "Hello\u0BF2Tamil1000";
const inMonospace9 = "Hello\ud835\udfffMonospace9";
const expectedSpace = "Hello Space";
const expectedEnSpace = "Hello EnSpace";
const expectedEmSpace = "Hello EmSpace";
const expectedEuro = "HelloEUREuro";
const expectedTamil1000 = "Hello[1000]Tamil1000";
const expectedMonospace9 = "Hello9Monospace9";
const EntityAfterCharsetConv = 512;
const transliterate = 8;
const charset = "ISO-8859-2";
function run_test() {
var SaveAsCharset =
Components.Constructor("@mozilla.org/intl/saveascharset;1",
"nsISaveAsCharset",
"Init");
var converter = new SaveAsCharset(charset,
EntityAfterCharsetConv,
transliterate);
var outSpace = converter.Convert(inSpace);
do_check_eq(outSpace, expectedSpace);
var outEnSpace = converter.Convert(inEnSpace);
do_check_eq(outEnSpace, expectedEnSpace);
var outEmSpace = converter.Convert(inEmSpace);
do_check_eq(outEmSpace, expectedEmSpace);
var outEuro = converter.Convert(inEuro);
do_check_eq(outEuro, expectedEuro);
var outTamil1000 = converter.Convert(inTamil1000);
do_check_eq(outTamil1000, expectedTamil1000);
var outMonospace9 = converter.Convert(inMonospace9);
do_check_eq(outMonospace9, expectedMonospace9);
}

1
intl/uconv/tests/unit/xpcshell.ini
Просмотреть файл

@ -14,7 +14,6 @@ support-files =
[test_bug317216.js]
[test_bug321379.js]
[test_bug340714.js]
[test_bug365345.js]
[test_bug381412.Big5-HKSCS.js]
[test_bug381412.Big5.js]
[test_bug381412.euc-kr.js]

178
intl/unicharutil/nsEntityConverter.cpp
Просмотреть файл

@ -1,4 +1,4 @@
/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
@ -13,79 +13,56 @@
//
// implementation methods
//
nsEntityConverter::nsEntityConverter() :
mVersionList(nullptr),
mVersionListLength(0)
nsEntityConverter::nsEntityConverter() { }
nsEntityConverter::~nsEntityConverter() { }
nsIStringBundle*
nsEntityConverter:: GetVersionBundleInstance(uint32_t versionNumber)
{
}
nsEntityConverter::~nsEntityConverter()
{
if (mVersionList)
delete [] mVersionList;
}
NS_IMETHODIMP
nsEntityConverter::LoadVersionPropertyFile()
{
NS_NAMED_LITERAL_CSTRING(url, "resource://gre/res/entityTables/htmlEntityVersions.properties");
nsCOMPtr<nsIStringBundleService> bundleService =
mozilla::services::GetStringBundleService();
if (!bundleService)
return NS_ERROR_FAILURE;
nsCOMPtr<nsIStringBundle> entities;
nsresult rv = bundleService->CreateBundle(url.get(), getter_AddRefs(entities));
if (NS_FAILED(rv)) return rv;
nsresult result;
nsAutoString key;
nsXPIDLString value;
rv = entities->GetStringFromName(MOZ_UTF16("length"),
getter_Copies(value));
NS_ASSERTION(NS_SUCCEEDED(rv),"nsEntityConverter: malformed entity table\n");
if (NS_FAILED(rv)) return rv;
mVersionListLength = nsAutoString(value).ToInteger(&result);
NS_ASSERTION(32 >= mVersionListLength,"nsEntityConverter: malformed entity table\n");
if (32 < mVersionListLength) return NS_ERROR_FAILURE;
mVersionList = new nsEntityVersionList[mVersionListLength];
if (!mVersionList) return NS_ERROR_OUT_OF_MEMORY;
for (uint32_t i = 0; i < mVersionListLength && NS_SUCCEEDED(rv); i++) {
key.SetLength(0);
key.AppendInt(i+1, 10);
rv = entities->GetStringFromName(key.get(), getter_Copies(value));
uint32_t len = value.Length();
if (kVERSION_STRING_LEN < len) return NS_ERROR_UNEXPECTED;
memcpy(mVersionList[i].mEntityListName, value.get(), len*sizeof(char16_t));
mVersionList[i].mEntityListName[len] = 0;
mVersionList[i].mVersion = (1 << i);
switch(versionNumber){
case nsIEntityConverter::html40Latin1:
if (!mHTML40Latin1Bundle) {
mHTML40Latin1Bundle = LoadEntityBundle(kHTML40LATIN1);
MOZ_ASSERT(mHTML40Latin1Bundle, "LoadEntityBundle failed");
}
return mHTML40Latin1Bundle;
case nsIEntityConverter::html40Symbols:
if (!mHTML40SymbolsBundle) {
mHTML40SymbolsBundle = LoadEntityBundle(kHTML40SYMBOLS);
MOZ_ASSERT(mHTML40SymbolsBundle, "LoadEntityBundle failed");
}
return mHTML40SymbolsBundle;
case nsIEntityConverter::html40Special:
if (!mHTML40SpecialBundle) {
mHTML40SpecialBundle = LoadEntityBundle(kHTML40SPECIAL);
MOZ_ASSERT(mHTML40SpecialBundle, "LoadEntityBundle failed");
}
return mHTML40SpecialBundle;
case nsIEntityConverter::mathml20:
if (!mMathML20Bundle) {
mMathML20Bundle = LoadEntityBundle(kMATHML20);
MOZ_ASSERT(mMathML20Bundle, "LoadEntityBundle failed");
}
return mMathML20Bundle;
default:
return nullptr;
}
return NS_OK;
}
already_AddRefed<nsIStringBundle>
nsEntityConverter::LoadEntityBundle(uint32_t version)
nsEntityConverter:: LoadEntityBundle(const char *fileName)
{
nsAutoCString url(NS_LITERAL_CSTRING("resource://gre/res/entityTables/"));
NS_ENSURE_TRUE(fileName, nullptr);
nsAutoCString url("resource://gre/res/entityTables/");
nsresult rv;
nsCOMPtr<nsIStringBundleService> bundleService =
do_GetService(NS_STRINGBUNDLE_CONTRACTID, &rv);
NS_ENSURE_SUCCESS(rv, nullptr);
const char16_t *versionName = GetVersionName(version);
NS_ENSURE_TRUE(versionName, nullptr);
// all property file names are ASCII, like "html40Latin1" so this is safe
LossyAppendUTF16toASCII(versionName, url);
url.AppendLiteral(".properties");
url.Append(fileName);
nsCOMPtr<nsIStringBundle> bundle;
rv = bundleService->CreateBundle(url.get(), getter_AddRefs(bundle));
@ -94,50 +71,11 @@ nsEntityConverter::LoadEntityBundle(uint32_t version)
return bundle.forget();
}
const char16_t*
nsEntityConverter:: GetVersionName(uint32_t versionNumber)
{
for (uint32_t i = 0; i < mVersionListLength; i++) {
if (versionNumber == mVersionList[i].mVersion)
return mVersionList[i].mEntityListName;
}
return nullptr;
}
nsIStringBundle*
nsEntityConverter:: GetVersionBundleInstance(uint32_t versionNumber)
{
if (!mVersionList) {
// load the property file which contains available version names
// and generate a list of version/name pair
if (NS_FAILED(LoadVersionPropertyFile()))
return nullptr;
}
uint32_t i;
for (i = 0; i < mVersionListLength; i++) {
if (versionNumber == mVersionList[i].mVersion) {
if (!mVersionList[i].mEntities)
{ // not loaded
// load the property file
mVersionList[i].mEntities = LoadEntityBundle(versionNumber);
NS_ASSERTION(mVersionList[i].mEntities, "LoadEntityBundle failed");
}
return mVersionList[i].mEntities.get();
}
}
return nullptr;
}
//
// nsISupports methods
//
NS_IMPL_ISUPPORTS(nsEntityConverter,nsIEntityConverter)
//
// nsIEntityConverter
//
@ -151,18 +89,22 @@ NS_IMETHODIMP
nsEntityConverter::ConvertUTF32ToEntity(uint32_t character, uint32_t entityVersion, char **_retval)
{
NS_ASSERTION(_retval, "null ptr- _retval");
if(nullptr == _retval)
if (nullptr == _retval) {
return NS_ERROR_NULL_POINTER;
}
*_retval = nullptr;
for (uint32_t mask = 1, mask2 = 0xFFFFFFFFL; (0!=(entityVersion & mask2)); mask<<=1, mask2<<=1) {
if (0 == (entityVersion & mask))
if (0 == (entityVersion & mask)) {
continue;
nsIStringBundle* entities = GetVersionBundleInstance(entityVersion & mask);
NS_ASSERTION(entities, "Cannot get the property file");
}
if (!entities)
nsIStringBundle* entities = GetVersionBundleInstance(entityVersion & mask);
NS_ASSERTION(entities, "Cannot get the entity");
if (!entities) {
continue;
}
nsAutoString key(NS_LITERAL_STRING("entity."));
key.AppendInt(character,10);
@ -171,9 +113,6 @@ nsEntityConverter::ConvertUTF32ToEntity(uint32_t character, uint32_t entityVersi
nsresult rv = entities->GetStringFromName(key.get(), getter_Copies(value));
if (NS_SUCCEEDED(rv)) {
*_retval = ToNewCString(value);
if(nullptr == *_retval)
return NS_ERROR_OUT_OF_MEMORY;
else
return NS_OK;
}
}
@ -194,13 +133,10 @@ nsEntityConverter::ConvertToEntities(const char16_t *inString, uint32_t entityVe
uint32_t len = NS_strlen(inString);
for (uint32_t i = 0; i < len; i++) {
nsAutoString key(NS_LITERAL_STRING("entity."));
if (NS_IS_HIGH_SURROGATE(inString[i]) &&
i + 2 < len &&
NS_IS_LOW_SURROGATE(inString[i + 1])) {
if (NS_IS_HIGH_SURROGATE(inString[i]) && i + 2 < len && NS_IS_LOW_SURROGATE(inString[i + 1])) {
key.AppendInt(SURROGATE_TO_UCS4(inString[i], inString[i+1]), 10);
++i;
}
else {
} else {
key.AppendInt(inString[i],10);
}
@ -208,16 +144,17 @@ nsEntityConverter::ConvertToEntities(const char16_t *inString, uint32_t entityVe
const char16_t *entity = nullptr;
for (uint32_t mask = 1, mask2 = 0xFFFFFFFFL; (0!=(entityVersion & mask2)); mask<<=1, mask2<<=1) {
if (0 == (entityVersion & mask))
if (0 == (entityVersion & mask)) {
continue;
}
nsIStringBundle* entities = GetVersionBundleInstance(entityVersion & mask);
NS_ASSERTION(entities, "Cannot get the property file");
if (!entities)
if (!entities) {
continue;
}
nsresult rv = entities->GetStringFromName(key.get(),
getter_Copies(value));
nsresult rv = entities->GetStringFromName(key.get(), getter_Copies(value));
if (NS_SUCCEEDED(rv)) {
entity = value.get();
break;
@ -225,15 +162,12 @@ nsEntityConverter::ConvertToEntities(const char16_t *inString, uint32_t entityVe
}
if (entity) {
outString.Append(entity);
}
else {
} else {
outString.Append(&inString[i], 1);
}
}
*_retval = ToNewUnicode(outString);
if (!*_retval)
return NS_ERROR_OUT_OF_MEMORY;
return NS_OK;
}

37
intl/unicharutil/nsEntityConverter.h
Просмотреть файл

@ -10,22 +10,9 @@
#include "nsIStringBundle.h"
#include "nsCOMPtr.h"
#define kVERSION_STRING_LEN 128
class nsEntityVersionList
{
public:
nsEntityVersionList() {}
uint32_t mVersion;
char16_t mEntityListName[kVERSION_STRING_LEN+1];
nsCOMPtr<nsIStringBundle> mEntities;
};
class nsEntityConverter: public nsIEntityConverter
{
public:
//
// implementation methods
//
@ -36,31 +23,25 @@ public:
//
NS_DECL_ISUPPORTS
//
// nsIEntityConverter
//
NS_IMETHOD ConvertUTF32ToEntity(uint32_t character, uint32_t entityVersion, char **_retval) override;
NS_IMETHOD ConvertToEntity(char16_t character, uint32_t entityVersion, char **_retval) override;
NS_IMETHOD ConvertToEntities(const char16_t *inString, uint32_t entityVersion, char16_t **_retval) override;
protected:
// load a version property file and generate a version list (number/name pair)
NS_IMETHOD LoadVersionPropertyFile();
// map version number to version string
const char16_t* GetVersionName(uint32_t versionNumber);
// map version number to a string bundle
nsIStringBundle* GetVersionBundleInstance(uint32_t versionNumber);
// load a string bundle file
already_AddRefed<nsIStringBundle> LoadEntityBundle(uint32_t version);
already_AddRefed<nsIStringBundle> LoadEntityBundle(const char *fileName);
nsEntityVersionList *mVersionList; // array of version number/name pairs
uint32_t mVersionListLength; // number of supported versions
const char* kHTML40LATIN1 = "html40Latin1.properties";
const char* kHTML40SYMBOLS = "html40Symbols.properties";
const char* kHTML40SPECIAL = "html40Special.properties";
const char* kMATHML20 = "mathml20.properties";
nsCOMPtr<nsIStringBundle> mHTML40Latin1Bundle;
nsCOMPtr<nsIStringBundle> mHTML40SymbolsBundle;
nsCOMPtr<nsIStringBundle> mHTML40SpecialBundle;
nsCOMPtr<nsIStringBundle> mMathML20Bundle;
virtual ~nsEntityConverter();
};

2
intl/unicharutil/nsIEntityConverter.idl
Просмотреть файл

@ -18,7 +18,7 @@ interface nsIEntityConverter : nsISupports
const unsigned long html40Latin1 = 1;
const unsigned long html40Symbols = 2;
const unsigned long html40Special = 4; // excludes &quot, &amp, &lt, &gt
const unsigned long transliterate = 8;
const unsigned long transliterate = 8; // Obsolete
const unsigned long mathml20 = 16;
const unsigned long html32 = html40Latin1;
const unsigned long html40 = html40Latin1+html40Symbols+html40Special;

18
intl/unicharutil/tables/htmlEntityVersions.properties
Просмотреть файл

@ -1,18 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# LOCALIZATION NOTE: FILE
# This file associates internal names of entity lists to integers.
# Do not translate anything in this file
# list supported versions number/name pair
# length should not be greater than 32
length=5
1=html40Latin1
2=html40Symbols
3=html40Special
4=transliterate
5=mathml20

2
intl/unicharutil/tables/moz.build
Просмотреть файл

@ -5,12 +5,10 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
RESOURCE_FILES.entityTables = [
'htmlEntityVersions.properties',
'html40Latin1.properties',
'html40Symbols.properties',
'html40Special.properties',
'mathml20.properties',
'transliterate.properties',
]
if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'qt':

2893
intl/unicharutil/tables/transliterate.properties
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

2
intl/unicharutil/tests/moz.build Normal file → Executable file
Просмотреть файл

@ -4,8 +4,6 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
XPCSHELL_TESTS_MANIFESTS += ['unit/xpcshell.ini']
GeckoSimplePrograms([
'NormalizationTest',
'UnicharSelfTest',

56
intl/unicharutil/tests/unit/test_bug_427350_1.js
Просмотреть файл

@ -1,56 +0,0 @@
/* Tests transliteration of new characters in Unicode 5.1, 5.2, and 6.0
*/
const inTeluguFractions = "\u0C78\u0C79\u0C7A\u0C7B\u0C7C\u0C7D\u0C7E";
const inMalayalamNumbers = "\u0D70\u0D71\u0D72\u0D73\u0D74\u0D75";
/* MYANMAR SHAN DIGIT ONE,
SUNDANESE DIGIT TWO,
LEPCHA DIGIT THREE,
OL CHIKI DIGIT FOUR,
VAI DIGIT FIVE,
SAURASHTRA DIGIT SIX
KAYAH LI DIGIT SEVEN
CHAM DIGIT EIGHT
JAVANESE DIGIT NINE
MEETEI MAYEK DIGIT ZERO */
const inDigits = "\u1091\u1BB2\u1C43\u1C54\uA625\uA8D6\uA907\uAA58\uA9D9\uABF0";
const inRomanNumerals = "\u2185\u2186\u2187\u2188";
const inSuperSubscripts = "\u2C7C\u2C7D\u2095\u209C";
const expectedTeluguFractions = "[0][1][2][3][1][2][3]";
const expectedMalayalamNumbers = "[10][100][1000][1/4][1/2][3/4]";
const expectedDigits = "1234567890";
const expectedRomanNumerals = "[6][50][50000][100000]";
const expectedSuperSubscripts = "v(j)^(V)v(h)v(t)";
const EntityAfterCharsetConv = 512;
const transliterate = 8;
const charset = "ISO-8859-1";
function run_test() {
var SaveAsCharset =
Components.Constructor("@mozilla.org/intl/saveascharset;1",
"nsISaveAsCharset",
"Init");
var converter = new SaveAsCharset(charset,
EntityAfterCharsetConv,
transliterate);
var outTeluguFractions = converter.Convert(inTeluguFractions);
do_check_eq(outTeluguFractions, expectedTeluguFractions);
var outMalayalamNumbers = converter.Convert(inMalayalamNumbers);
do_check_eq(outMalayalamNumbers, expectedMalayalamNumbers);
var outDigits = converter.Convert(inDigits);
do_check_eq(outDigits, expectedDigits);
var outRomanNumerals = converter.Convert(inRomanNumerals);
do_check_eq(outRomanNumerals, expectedRomanNumerals);
var outSuperSubscripts = converter.Convert(inSuperSubscripts);
do_check_eq(outSuperSubscripts, expectedSuperSubscripts);
}

2
intl/unicharutil/tests/unit/xpcshell.ini Normal file → Executable file
Просмотреть файл

@ -2,5 +2,3 @@
head =
tail =
skip-if = toolkit == 'gonk'
[test_bug_427350_1.js]

2
widget/nsPrimitiveHelpers.cpp
Просмотреть файл

@ -160,7 +160,7 @@ nsPrimitiveHelpers :: ConvertUnicodeToPlatformPlainText ( char16_t* inUnicode, i
rv = converter->Init(platformCharset.get(),
nsISaveAsCharset::attr_EntityAfterCharsetConv +
nsISaveAsCharset::attr_FallbackQuestionMark,
nsIEntityConverter::transliterate);
0);
NS_ENSURE_SUCCESS(rv, rv);
rv = converter->Convert(inUnicode, outPlainTextData);