Bug 1129859 - Wrong dictionary chosen when lang attribute is en-US on some Linux distros. r=roc

--HG--
rename : extensions/spellcheck/hunspell/tests/unit/data/1463589_utf.aff => extensions/spellcheck/hunspell/tests/unit/data/1463589-utf.aff
rename : extensions/spellcheck/hunspell/tests/unit/data/1463589_utf.dic => extensions/spellcheck/hunspell/tests/unit/data/1463589-utf.dic
rename : extensions/spellcheck/hunspell/tests/unit/data/1463589_utf.sug => extensions/spellcheck/hunspell/tests/unit/data/1463589-utf.sug
rename : extensions/spellcheck/hunspell/tests/unit/data/1463589_utf.test => extensions/spellcheck/hunspell/tests/unit/data/1463589-utf.test
rename : extensions/spellcheck/hunspell/tests/unit/data/1463589_utf.wrong => extensions/spellcheck/hunspell/tests/unit/data/1463589-utf.wrong
rename : extensions/spellcheck/hunspell/tests/unit/data/allcaps_utf.aff => extensions/spellcheck/hunspell/tests/unit/data/allcaps-utf.aff
rename : extensions/spellcheck/hunspell/tests/unit/data/allcaps_utf.dic => extensions/spellcheck/hunspell/tests/unit/data/allcaps-utf.dic
rename : extensions/spellcheck/hunspell/tests/unit/data/allcaps_utf.good => extensions/spellcheck/hunspell/tests/unit/data/allcaps-utf.good
rename : extensions/spellcheck/hunspell/tests/unit/data/allcaps_utf.sug => extensions/spellcheck/hunspell/tests/unit/data/allcaps-utf.sug
rename : extensions/spellcheck/hunspell/tests/unit/data/allcaps_utf.test => extensions/spellcheck/hunspell/tests/unit/data/allcaps-utf.test
rename : extensions/spellcheck/hunspell/tests/unit/data/allcaps_utf.wrong => extensions/spellcheck/hunspell/tests/unit/data/allcaps-utf.wrong
rename : extensions/spellcheck/hunspell/tests/unit/data/base_utf.aff => extensions/spellcheck/hunspell/tests/unit/data/base-utf.aff
rename : extensions/spellcheck/hunspell/tests/unit/data/base_utf.dic => extensions/spellcheck/hunspell/tests/unit/data/base-utf.dic
rename : extensions/spellcheck/hunspell/tests/unit/data/base_utf.good => extensions/spellcheck/hunspell/tests/unit/data/base-utf.good
rename : extensions/spellcheck/hunspell/tests/unit/data/base_utf.sug => extensions/spellcheck/hunspell/tests/unit/data/base-utf.sug
rename : extensions/spellcheck/hunspell/tests/unit/data/base_utf.test => extensions/spellcheck/hunspell/tests/unit/data/base-utf.test
rename : extensions/spellcheck/hunspell/tests/unit/data/base_utf.wrong => extensions/spellcheck/hunspell/tests/unit/data/base-utf.wrong
rename : extensions/spellcheck/hunspell/tests/unit/data/colons_in_words.aff => extensions/spellcheck/hunspell/tests/unit/data/colons-in-words.aff
rename : extensions/spellcheck/hunspell/tests/unit/data/colons_in_words.dic => extensions/spellcheck/hunspell/tests/unit/data/colons-in-words.dic
rename : extensions/spellcheck/hunspell/tests/unit/data/colons_in_words.test => extensions/spellcheck/hunspell/tests/unit/data/colons-in-words.test
rename : extensions/spellcheck/hunspell/tests/unit/data/condition_utf.aff => extensions/spellcheck/hunspell/tests/unit/data/condition-utf.aff
rename : extensions/spellcheck/hunspell/tests/unit/data/condition_utf.dic => extensions/spellcheck/hunspell/tests/unit/data/condition-utf.dic
rename : extensions/spellcheck/hunspell/tests/unit/data/condition_utf.good => extensions/spellcheck/hunspell/tests/unit/data/condition-utf.good
rename : extensions/spellcheck/hunspell/tests/unit/data/condition_utf.test => extensions/spellcheck/hunspell/tests/unit/data/condition-utf.test
rename : extensions/spellcheck/hunspell/tests/unit/data/condition_utf.wrong => extensions/spellcheck/hunspell/tests/unit/data/condition-utf.wrong
rename : extensions/spellcheck/hunspell/tests/unit/data/digits_in_words.aff => extensions/spellcheck/hunspell/tests/unit/data/digits-in-words.aff
rename : extensions/spellcheck/hunspell/tests/unit/data/digits_in_words.dic => extensions/spellcheck/hunspell/tests/unit/data/digits-in-words.dic
rename : extensions/spellcheck/hunspell/tests/unit/data/digits_in_words.test => extensions/spellcheck/hunspell/tests/unit/data/digits-in-words.test
rename : extensions/spellcheck/hunspell/tests/unit/data/digits_in_words.wrong => extensions/spellcheck/hunspell/tests/unit/data/digits-in-words.wrong
rename : extensions/spellcheck/hunspell/tests/unit/data/ngram_utf_fix.aff => extensions/spellcheck/hunspell/tests/unit/data/ngram-utf-fix.aff
rename : extensions/spellcheck/hunspell/tests/unit/data/ngram_utf_fix.dic => extensions/spellcheck/hunspell/tests/unit/data/ngram-utf-fix.dic
rename : extensions/spellcheck/hunspell/tests/unit/data/ngram_utf_fix.good => extensions/spellcheck/hunspell/tests/unit/data/ngram-utf-fix.good
rename : extensions/spellcheck/hunspell/tests/unit/data/ngram_utf_fix.sug => extensions/spellcheck/hunspell/tests/unit/data/ngram-utf-fix.sug
rename : extensions/spellcheck/hunspell/tests/unit/data/ngram_utf_fix.test => extensions/spellcheck/hunspell/tests/unit/data/ngram-utf-fix.test
rename : extensions/spellcheck/hunspell/tests/unit/data/ngram_utf_fix.wrong => extensions/spellcheck/hunspell/tests/unit/data/ngram-utf-fix.wrong
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_cpdpat.aff => extensions/spellcheck/hunspell/tests/unit/data/opentaal-cpdpat.aff
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_cpdpat.dic => extensions/spellcheck/hunspell/tests/unit/data/opentaal-cpdpat.dic
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_cpdpat.good => extensions/spellcheck/hunspell/tests/unit/data/opentaal-cpdpat.good
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_cpdpat.test => extensions/spellcheck/hunspell/tests/unit/data/opentaal-cpdpat.test
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_cpdpat.wrong => extensions/spellcheck/hunspell/tests/unit/data/opentaal-cpdpat.wrong
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_cpdpat2.aff => extensions/spellcheck/hunspell/tests/unit/data/opentaal-cpdpat2.aff
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_cpdpat2.dic => extensions/spellcheck/hunspell/tests/unit/data/opentaal-cpdpat2.dic
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_cpdpat2.good => extensions/spellcheck/hunspell/tests/unit/data/opentaal-cpdpat2.good
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_cpdpat2.test => extensions/spellcheck/hunspell/tests/unit/data/opentaal-cpdpat2.test
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_cpdpat2.wrong => extensions/spellcheck/hunspell/tests/unit/data/opentaal-cpdpat2.wrong
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_forbiddenword1.aff => extensions/spellcheck/hunspell/tests/unit/data/opentaal-forbiddenword1.aff
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_forbiddenword1.dic => extensions/spellcheck/hunspell/tests/unit/data/opentaal-forbiddenword1.dic
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_forbiddenword1.good => extensions/spellcheck/hunspell/tests/unit/data/opentaal-forbiddenword1.good
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_forbiddenword1.sug => extensions/spellcheck/hunspell/tests/unit/data/opentaal-forbiddenword1.sug
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_forbiddenword1.test => extensions/spellcheck/hunspell/tests/unit/data/opentaal-forbiddenword1.test
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_forbiddenword1.wrong => extensions/spellcheck/hunspell/tests/unit/data/opentaal-forbiddenword1.wrong
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_forbiddenword2.aff => extensions/spellcheck/hunspell/tests/unit/data/opentaal-forbiddenword2.aff
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_forbiddenword2.dic => extensions/spellcheck/hunspell/tests/unit/data/opentaal-forbiddenword2.dic
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_forbiddenword2.good => extensions/spellcheck/hunspell/tests/unit/data/opentaal-forbiddenword2.good
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_forbiddenword2.sug => extensions/spellcheck/hunspell/tests/unit/data/opentaal-forbiddenword2.sug
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_forbiddenword2.test => extensions/spellcheck/hunspell/tests/unit/data/opentaal-forbiddenword2.test
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_forbiddenword2.wrong => extensions/spellcheck/hunspell/tests/unit/data/opentaal-forbiddenword2.wrong
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_keepcase.aff => extensions/spellcheck/hunspell/tests/unit/data/opentaal-keepcase.aff
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_keepcase.dic => extensions/spellcheck/hunspell/tests/unit/data/opentaal-keepcase.dic
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_keepcase.good => extensions/spellcheck/hunspell/tests/unit/data/opentaal-keepcase.good
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_keepcase.sug => extensions/spellcheck/hunspell/tests/unit/data/opentaal-keepcase.sug
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_keepcase.test => extensions/spellcheck/hunspell/tests/unit/data/opentaal-keepcase.test
rename : extensions/spellcheck/hunspell/tests/unit/data/opentaal_keepcase.wrong => extensions/spellcheck/hunspell/tests/unit/data/opentaal-keepcase.wrong
rename : extensions/spellcheck/hunspell/tests/unit/data/utf8_bom.aff => extensions/spellcheck/hunspell/tests/unit/data/utf8-bom.aff
rename : extensions/spellcheck/hunspell/tests/unit/data/utf8_bom.dic => extensions/spellcheck/hunspell/tests/unit/data/utf8-bom.dic
rename : extensions/spellcheck/hunspell/tests/unit/data/utf8_bom.good => extensions/spellcheck/hunspell/tests/unit/data/utf8-bom.good
rename : extensions/spellcheck/hunspell/tests/unit/data/utf8_bom.test => extensions/spellcheck/hunspell/tests/unit/data/utf8-bom.test
rename : extensions/spellcheck/hunspell/tests/unit/data/utf8_bom2.aff => extensions/spellcheck/hunspell/tests/unit/data/utf8-bom2.aff
rename : extensions/spellcheck/hunspell/tests/unit/data/utf8_bom2.dic => extensions/spellcheck/hunspell/tests/unit/data/utf8-bom2.dic
rename : extensions/spellcheck/hunspell/tests/unit/data/utf8_bom2.good => extensions/spellcheck/hunspell/tests/unit/data/utf8-bom2.good
rename : extensions/spellcheck/hunspell/tests/unit/data/utf8_bom2.test => extensions/spellcheck/hunspell/tests/unit/data/utf8-bom2.test
rename : extensions/spellcheck/hunspell/tests/unit/data/utf8_nonbmp.aff => extensions/spellcheck/hunspell/tests/unit/data/utf8-nonbmp.aff
rename : extensions/spellcheck/hunspell/tests/unit/data/utf8_nonbmp.dic => extensions/spellcheck/hunspell/tests/unit/data/utf8-nonbmp.dic
rename : extensions/spellcheck/hunspell/tests/unit/data/utf8_nonbmp.good => extensions/spellcheck/hunspell/tests/unit/data/utf8-nonbmp.good
rename : extensions/spellcheck/hunspell/tests/unit/data/utf8_nonbmp.sug => extensions/spellcheck/hunspell/tests/unit/data/utf8-nonbmp.sug
rename : extensions/spellcheck/hunspell/tests/unit/data/utf8_nonbmp.test => extensions/spellcheck/hunspell/tests/unit/data/utf8-nonbmp.test
rename : extensions/spellcheck/hunspell/tests/unit/data/utf8_nonbmp.wrong => extensions/spellcheck/hunspell/tests/unit/data/utf8-nonbmp.wrong
extra : rebase_source : 09ef9975419ce11e104ce2792c2f0f3364f6db8d

editor/composer/nsEditorSpellCheck.cpp

@@ -105,23 +105,6 @@ GetLoadContext(nsIEditor* aEditor)
   return loadContext.forget();
 }
 
-/**
- * Helper function for converting underscore to dash in dictionary name,
- * ie. en_CA to en-CA. This is required for some Linux distributions which
- * use underscore as separator in system-wide installed dictionaries.
- * We use it for nsStyleUtil::DashMatchCompare.
- */
-static nsString
-GetDictNameWithDash(const nsAString& aDictName)
-{
-  nsString dictNameWithDash(aDictName);
-  int32_t underScore = dictNameWithDash.FindChar('_');
-  if (underScore != -1) {
-    dictNameWithDash.Replace(underScore, 1, '-');
-  }
-  return dictNameWithDash;
-}
-
 /**
  * Fetches the dictionary stored in content prefs and maintains state during the
  * fetch, which is asynchronous.
@@ -621,7 +604,7 @@ nsEditorSpellCheck::SetCurrentDictionary(const nsAString& aDictionary)
       langCode.Assign(aDictionary);
     }
     if (mPreferredLang.IsEmpty() ||
-        !nsStyleUtil::DashMatchCompare(GetDictNameWithDash(mPreferredLang), langCode, comparator)) {
+        !nsStyleUtil::DashMatchCompare(mPreferredLang, langCode, comparator)) {
       // When user sets dictionary manually, we store this value associated
       // with editor url.
       StoreCurrentDictionary(mEditor, aDictionary);
@@ -806,7 +789,7 @@ nsEditorSpellCheck::DictionaryFetched(DictionaryFetcher* aFetcher)
       // try dictionary.spellchecker preference if it starts with langCode (and
       // if we haven't tried it already)
       if (!preferedDict.IsEmpty() && !dictName.Equals(preferedDict) &&
-          nsStyleUtil::DashMatchCompare(GetDictNameWithDash(preferedDict), langCode, comparator)) {
+          nsStyleUtil::DashMatchCompare(preferedDict, langCode, comparator)) {
         rv = SetCurrentDictionary(preferedDict);
       }
 
@@ -834,7 +817,7 @@ nsEditorSpellCheck::DictionaryFetched(DictionaryFetcher* aFetcher)
             // We have already tried it
             continue;
           }
-          if (nsStyleUtil::DashMatchCompare(GetDictNameWithDash(dictStr), langCode, comparator) &&
+          if (nsStyleUtil::DashMatchCompare(dictStr, langCode, comparator) &&
               NS_SUCCEEDED(SetCurrentDictionary(dictStr))) {
               break;
           }
@@ -859,10 +842,7 @@ nsEditorSpellCheck::DictionaryFetched(DictionaryFetcher* aFetcher)
         if (dot_pos != -1) {
           lang = Substring(lang, 0, dot_pos);
         }
-        // Some Linux distributions use '_' as lang/dialect separator.
-        rv = SetCurrentDictionary(lang);
         if (NS_FAILED(rv)) {
-          // Replace '_' with '-' when dictionary with underscore not found.
           int32_t underScore = lang.FindChar('_');
           if (underScore != -1) {
             lang.Replace(underScore, 1, '-');
@@ -873,14 +853,10 @@ nsEditorSpellCheck::DictionaryFetched(DictionaryFetcher* aFetcher)
       if (NS_FAILED(rv)) {
         rv = SetCurrentDictionary(NS_LITERAL_STRING("en-US"));
         if (NS_FAILED(rv)) {
-          // Some Linux distributions are using '_' as separator for dictionaries.
-          rv = SetCurrentDictionary(NS_LITERAL_STRING("en_US"));
-          if (NS_FAILED(rv)) {
-            nsTArray<nsString> dictList;
-            rv = mSpellChecker->GetDictionaryList(&dictList);
-            if (NS_SUCCEEDED(rv) && dictList.Length() > 0) {
-              SetCurrentDictionary(dictList[0]);
-            }
+          nsTArray<nsString> dictList;
+          rv = mSpellChecker->GetDictionaryList(&dictList);
+          if (NS_SUCCEEDED(rv) && dictList.Length() > 0) {
+            SetCurrentDictionary(dictList[0]);
           }
         }
       }

extensions/spellcheck/hunspell/src/mozHunspell.cpp

@@ -490,6 +490,9 @@ mozHunspell::LoadDictionariesFromDir(nsIFile* aDir)
     printf("Adding dictionary: %s\n", NS_ConvertUTF16toUTF8(dict).get());
 #endif
 
+    // Replace '_' separator with '-'
+    dict.ReplaceChar("_", '-');
+
     mDictionaries.Put(dict, file);
   }
 

extensions/spellcheck/hunspell/tests/unit/test_hunspell.js

@@ -8,11 +8,11 @@ const Ci = Components.interfaces;
 const tests = [
     ["affixes", "iso-8859-1"],
     ["condition", "iso-8859-1"],
-    ["condition_utf", "UTF-8"],
+    ["condition-utf", "UTF-8"],
     ["base", "iso-8859-1"],
-    ["base_utf", "UTF-8"],
+    ["base-utf", "UTF-8"],
     ["allcaps", "iso-8859-1"],
-    ["allcaps_utf", "UTF-8"],
+    ["allcaps-utf", "UTF-8"],
     ["allcaps2", "iso-8859-1"],
     ["allcaps3", "iso-8859-1"],
     ["keepcase", "iso-8859-1"],
@@ -48,9 +48,9 @@ const tests = [
     ["conditionalprefix", "iso-8859-1"],
     ["zeroaffix", "iso-8859-1"],
     ["utf8", "UTF-8"],
-    ["utf8_bom", "UTF-8", {1: "todo"}],
-    ["utf8_bom2", "UTF-8", {1: "todo"}],
-    ["utf8_nonbmp", "UTF-8", {1: "todo", 2: "todo", 3: "todo", 4: "todo"}],
+    ["utf8-bom", "UTF-8", {1: "todo"}],
+    ["utf8-bom2", "UTF-8", {1: "todo"}],
+    ["utf8-nonbmp", "UTF-8", {1: "todo", 2: "todo", 3: "todo", 4: "todo"}],
     ["compoundflag", "iso-8859-1"],
     ["compoundrule", "iso-8859-1"],
     ["compoundrule2", "iso-8859-1"],
@@ -91,14 +91,14 @@ const tests = [
     ["1592880", "iso-8859-1"],
     ["1695964", "iso-8859-1"],
     ["1463589", "iso-8859-1"],
-    ["1463589_utf", "UTF-8"],
+    ["1463589-utf", "UTF-8"],
     ["IJ", "iso-8859-1"],
     ["i68568", "iso-8859-1"],
     ["i68568utf", "UTF-8"],
     ["1706659", "iso-8859-1"],
-    ["digits_in_words", "iso-8859-1"],
-//    ["colons_in_words", "iso-8859-1"], Suggestion test only
-    ["ngram_utf_fix", "UTF-8"],
+    ["digits-in-words", "iso-8859-1"],
+//    ["colons-in-words", "iso-8859-1"], Suggestion test only
+    ["ngram-utf-fix", "UTF-8"],
     ["morph", "us-ascii",
      {11: "todo", 12: "todo", 13: "todo", 14: "todo", 15: "todo", 16: "todo",
       17: "todo", 18: "todo", 19: "todo", 20: "todo", 21: "todo", 22: "todo",
@@ -109,15 +109,15 @@ const tests = [
     ["oconv", "UTF-8"],
     ["encoding", "iso-8859-1", {1: "todo", 3: "todo"}],
     ["korean", "UTF-8"],
-    ["opentaal_forbiddenword1", "UTF-8"],
-    ["opentaal_forbiddenword2", "UTF-8"],
-    ["opentaal_keepcase", "UTF-8"],
+    ["opentaal-forbiddenword1", "UTF-8"],
+    ["opentaal-forbiddenword2", "UTF-8"],
+    ["opentaal-keepcase", "UTF-8"],
     ["arabic", "UTF-8"],
     ["2970240", "iso-8859-1"],
     ["2970242", "iso-8859-1"],
     ["breakoff", "iso-8859-1"],
-    ["opentaal_cpdpat", "iso-8859-1"],
-    ["opentaal_cpdpat2", "iso-8859-1"],
+    ["opentaal-cpdpat", "iso-8859-1"],
+    ["opentaal-cpdpat2", "iso-8859-1"],
     ["2999225", "iso-8859-1"],
     ["onlyincompound2", "iso-8859-1"],
     ["forceucase", "iso-8859-1"],

extensions/spellcheck/tests/chrome/test_add_remove_dictionaries.xul

@@ -68,16 +68,16 @@ function RunTest() {
 
     // test that base and map dictionaries are available
     var dicts = getDictionaryList(editor);
-    isnot(dicts.indexOf("base_utf"), -1, "base is available");
+    isnot(dicts.indexOf("base-utf"), -1, "base is available");
     isnot(dicts.indexOf("maputf"), -1, "map is available");
 
     // select base dictionary
-    setCurrentDictionary(editor, "base_utf");
+    setCurrentDictionary(editor, "base-utf");
 
     onSpellCheck(textbox, function () {
       // test that base dictionary is in use
       is(getMisspelledWords(editor), "Frühstück" + "qwertyu", "base misspellings");
-      is(getCurrentDictionary(editor), "base_utf", "current dictionary");
+      is(getCurrentDictionary(editor), "base-utf", "current dictionary");
 
       // select map dictionary
       setCurrentDictionary(editor, "maputf");
@@ -97,7 +97,7 @@ function RunTest() {
 
           // test that base dictionary is available and map dictionary is unavailable
           var dicts = getDictionaryList(editor);
-          isnot(dicts.indexOf("base_utf"), -1, "base is available");
+          isnot(dicts.indexOf("base-utf"), -1, "base is available");
           is(dicts.indexOf("maputf"), -1, "map is unavailable");
 
           // uninstall base dictionary