Bug 1543077 part 4 - Have only one item for Japanese in the Text Encoding menu. r=emk,Gijs

Differential Revision: https://phabricator.services.mozilla.com/D28634

--HG--
extra : moz-landing-system : lando

browser/base/content/browser.js

@@ -6620,6 +6620,9 @@ function handleDroppedLink(event, urlOrLinks, nameOrTriggeringPrincipal, trigger
 
 function BrowserSetForcedCharacterSet(aCharset) {
   if (aCharset) {
+    if (aCharset == "Japanese") {
+      aCharset = "Shift_JIS";
+    }
     gBrowser.selectedBrowser.characterSet = aCharset;
     // Save the forced character-set
     PlacesUIUtils.setCharsetForPage(getWebNavigation().currentURI,
@@ -6634,7 +6637,8 @@ function BrowserCharsetReload() {
 }
 
 function UpdateCurrentCharset(target) {
-  let selectedCharset = CharsetMenu.foldCharset(gBrowser.selectedBrowser.characterSet);
+  let selectedCharset = CharsetMenu.foldCharset(gBrowser.selectedBrowser.characterSet,
+                                                gBrowser.selectedBrowser.charsetAutodetected);
   for (let menuItem of target.getElementsByTagName("menuitem")) {
     let isSelected = menuItem.getAttribute("charset") === selectedCharset;
     menuItem.setAttribute("checked", isSelected);

browser/components/customizableui/CustomizableWidgets.jsm

@@ -358,7 +358,8 @@ const CustomizableWidgets = [
     },
     updateCurrentCharset(aDocument) {
       let currentCharset = aDocument.defaultView.gBrowser.selectedBrowser.characterSet;
-      currentCharset = CharsetMenu.foldCharset(currentCharset);
+      let {charsetAutodetected} = aDocument.defaultView.gBrowser.selectedBrowser;
+      currentCharset = CharsetMenu.foldCharset(currentCharset, charsetAutodetected);
 
       let pinnedContainer = aDocument.getElementById("PanelUI-characterEncodingView-pinned");
       let charsetContainer = aDocument.getElementById("PanelUI-characterEncodingView-charsets");

docshell/base/nsDocShell.cpp

@@ -1275,7 +1275,6 @@ nsDocShell::GatherCharsetMenuTelemetry() {
     case kCharsetFromDocTypeDefault:
     case kCharsetFromCache:
     case kCharsetFromParentFrame:
-    case kCharsetFromHintPrevDoc:
       // Changing charset on an unlabeled doc.
       if (isFileURL) {
         Telemetry::AccumulateCategorical(
@@ -1891,6 +1890,26 @@ nsDocShell::GetMayEnableCharacterEncodingMenu(
   return NS_OK;
 }
 
+NS_IMETHODIMP
+nsDocShell::GetCharsetAutodetected(bool* aCharsetAutodetected) {
+  *aCharsetAutodetected = false;
+  if (!mContentViewer) {
+    return NS_OK;
+  }
+  Document* doc = mContentViewer->GetDocument();
+  if (!doc) {
+    return NS_OK;
+  }
+  int32_t source = doc->GetDocumentCharacterSetSource();
+
+  if (source == kCharsetFromAutoDetection ||
+      source == kCharsetFromUserForcedAutoDetection) {
+    *aCharsetAutodetected = true;
+  }
+
+  return NS_OK;
+}
+
 NS_IMETHODIMP
 nsDocShell::GetDocShellEnumerator(int32_t aItemType,
                                   DocShellEnumeratorDirection aDirection,
@@ -8365,11 +8384,11 @@ nsresult nsDocShell::SetupNewViewer(nsIContentViewer* aNewViewer) {
 
   const Encoding* forceCharset = nullptr;
   const Encoding* hintCharset = nullptr;
-  int32_t hintCharsetSource;
-  float textZoom;
-  float pageZoom;
-  float overrideDPPX;
-  bool styleDisabled;
+  int32_t hintCharsetSource = kCharsetUninitialized;
+  float textZoom = 1.0;
+  float pageZoom = 1.0;
+  float overrideDPPX = 1.0;
+  bool styleDisabled = false;
   // |newMUDV| also serves as a flag to set the data from the above vars
   nsCOMPtr<nsIContentViewer> newCv;
 
@@ -10181,6 +10200,8 @@ nsresult nsDocShell::DoURILoad(nsDocShellLoadState* aLoadState,
     MOZ_ASSERT(NS_SUCCEEDED(rv));
   }
 
+  Unused << rv;  // Keep Coverity happy
+
   nsCOMPtr<nsIWritablePropertyBag2> props(do_QueryInterface(channel));
   if (props) {
     // save true referrer for those who need it (e.g. xpinstall whitelisting)

docshell/base/nsIDocShell.idl

@@ -878,6 +878,11 @@ interface nsIDocShell : nsIDocShellTreeItem
    */
   [infallible] readonly attribute boolean mayEnableCharacterEncodingMenu;
 
+  /**
+   * Indicates that the character encoding was autodetected.
+   */
+  [infallible] readonly attribute boolean charsetAutodetected;
+
            attribute  nsIEditor editor;
   readonly attribute  boolean   editable;             /* this docShell is editable */
   readonly attribute  boolean   hasEditingSession;    /* this docShell has an editing session */

dom/interfaces/base/nsIBrowser.idl

@@ -93,6 +93,12 @@ interface nsIBrowser : nsISupports
    */
   attribute boolean mayEnableCharacterEncodingMenu;
 
+  /**
+   * Whether or not the character encoding was detected by analyzing
+   * content (as opposed to reading a protocol label).
+   */
+  attribute boolean charsetAutodetected;
+
   /**
    * Called by Gecko to update the browser when its state changes.
    *

dom/ipc/BrowserChild.cpp

@@ -3504,6 +3504,8 @@ NS_IMETHODIMP BrowserChild::OnStateChange(nsIWebProgress* aWebProgress,
     stateChangeData->isNavigating() = docShell->GetIsNavigating();
     stateChangeData->mayEnableCharacterEncodingMenu() =
         docShell->GetMayEnableCharacterEncodingMenu();
+    stateChangeData->charsetAutodetected() =
+        docShell->GetCharsetAutodetected();
 
     if (document && aStateFlags & nsIWebProgressListener::STATE_STOP) {
       document->GetContentType(stateChangeData->contentType());

dom/ipc/BrowserParent.cpp

@@ -2364,6 +2364,8 @@ mozilla::ipc::IPCResult BrowserParent::RecvOnStateChange(
     Unused << browser->SetIsNavigating(aStateChangeData->isNavigating());
     Unused << browser->SetMayEnableCharacterEncodingMenu(
         aStateChangeData->mayEnableCharacterEncodingMenu());
+    Unused << browser->SetCharsetAutodetected(
+        aStateChangeData->charsetAutodetected());
     Unused << browser->UpdateForStateChange(aStateChangeData->charset(),
                                             aStateChangeData->documentURI(),
                                             aStateChangeData->contentType());

dom/ipc/PBrowser.ipdl

@@ -122,6 +122,7 @@ struct WebProgressStateChangeData
 {
   bool isNavigating;
   bool mayEnableCharacterEncodingMenu;
+  bool charsetAutodetected;
 
   // The following fields are only set when the aStateFlags param passed with
   // this struct is |nsIWebProgress.STATE_STOP|.

intl/Encoding.h

@@ -252,6 +252,14 @@ class Encoding final {
     return encoding_is_ascii_compatible(this);
   }
 
+  /**
+   * Checks whether this is a Japanese legacy encoding.
+   */
+  inline bool IsJapaneseLegacy() const {
+    return this == SHIFT_JIS_ENCODING || this == EUC_JP_ENCODING ||
+           this == ISO_2022_JP_ENCODING;
+  }
+
   /**
    * Returns the _output encoding_ of this encoding. This is UTF-8 for
    * UTF-16BE, UTF-16LE and replacement and the encoding itself otherwise.

modules/libpref/init/StaticPrefList.h

@@ -3116,6 +3116,15 @@ VARCACHE_PREF(
   RelaxedAtomicBool, false
 )
 
+// Whether ISO-2022-JP is a permitted content-based encoding detection
+// outcome.
+VARCACHE_PREF(
+  Live,
+  "intl.charset.detector.iso2022jp.allowed",
+   intl_charset_detector_iso2022jp_allowed,
+  bool, true
+)
+
 //---------------------------------------------------------------------------
 // Graphic Layers prefs
 //---------------------------------------------------------------------------

parser/html/nsHtml5StreamParser.cpp

@@ -180,7 +180,8 @@ nsHtml5StreamParser::nsHtml5StreamParser(nsHtml5TreeOpExecutor* aExecutor,
       mEventTarget(nsHtml5Module::GetStreamParserThread()->SerialEventTarget()),
       mExecutorFlusher(new nsHtml5ExecutorFlusher(aExecutor)),
       mLoadFlusher(new nsHtml5LoadFlusher(aExecutor)),
-      mJapaneseDetector(mozilla::JapaneseDetector::Create(true)),
+      mJapaneseDetector(mozilla::JapaneseDetector::Create(
+          StaticPrefs::intl_charset_detector_iso2022jp_allowed())),
       mInitialEncodingWasFromParentFrame(false),
       mHasHadErrors(false),
       mDecodingLocalFileAsUTF8(false),
@@ -296,32 +297,35 @@ void nsHtml5StreamParser::FeedJapaneseDetector(Span<const uint8_t> aBuffer,
     return;
   }
   mFeedChardet = false;
+  int32_t source = kCharsetFromAutoDetection;
+  if (mCharsetSource == kCharsetFromParentForced ||
+      mCharsetSource == kCharsetFromUserForced) {
+    source = kCharsetFromUserForcedAutoDetection;
+  }
   if (detected == mEncoding) {
-    MOZ_ASSERT(mCharsetSource < kCharsetFromAutoDetection,
-               "Why are we running chardet at all?");
-    mCharsetSource = kCharsetFromAutoDetection;
+    MOZ_ASSERT(mCharsetSource < source, "Why are we running chardet at all?");
+    mCharsetSource = source;
     mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource);
   } else if (HasDecoder()) {
     // We've already committed to a decoder. Request a reload from the
     // docshell.
-    mTreeBuilder->NeedsCharsetSwitchTo(WrapNotNull(detected),
-                                       kCharsetFromAutoDetection, 0);
+    mTreeBuilder->NeedsCharsetSwitchTo(WrapNotNull(detected), source, 0);
     FlushTreeOpsAndDisarmTimer();
     Interrupt();
   } else {
     // Got a confident answer from the sniffing buffer. That code will
     // take care of setting up the decoder.
     mEncoding = WrapNotNull(detected);
-    mCharsetSource = kCharsetFromAutoDetection;
+    mCharsetSource = source;
     mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource);
   }
 }
 
 void nsHtml5StreamParser::FeedDetector(Span<const uint8_t> aBuffer,
                                        bool aLast) {
-  if (mEncoding == SHIFT_JIS_ENCODING) {
+  if (mEncoding->IsJapaneseLegacy()) {
     FeedJapaneseDetector(aBuffer, aLast);
-  } else if (mEncoding == WINDOWS_1251_ENCODING) {
+  } else if (mEncoding == WINDOWS_1251_ENCODING && mChardet) {
     if (!aBuffer.IsEmpty()) {
       bool dontFeed = false;
       mozilla::Unused << mChardet->DoIt((const char*)aBuffer.Elements(),
@@ -379,6 +383,11 @@ nsHtml5StreamParser::SetupDecodingAndWriteSniffingBufferAndCurrentSegment(
     MOZ_ASSERT(mEncoding != UTF_8_ENCODING);
     mUnicodeDecoder = UTF_8_ENCODING->NewDecoderWithBOMRemoval();
   } else {
+    if (mCharsetSource >= kCharsetFromAutoDetection &&
+        !(mCharsetSource == kCharsetFromUserForced ||
+          mCharsetSource == kCharsetFromParentForced)) {
+      mFeedChardet = false;
+    }
     mDecodingLocalFileAsUTF8 = false;
     mUnicodeDecoder = mEncoding->NewDecoderWithBOMRemoval();
   }
@@ -523,12 +532,38 @@ static void HandleProcessingInstruction(void* aUserData,
   XML_StopParser(ud->mExpat, false);
 }
 
+void nsHtml5StreamParser::FinalizeSniffingWithDetector(
+    Span<const uint8_t> aFromSegment, uint32_t aCountToSniffingLimit,
+    bool aEof) {
+  if (mSniffingBuffer) {
+    FeedDetector(MakeSpan(mSniffingBuffer.get(), mSniffingLength), false);
+  }
+  if (mFeedChardet && !aFromSegment.IsEmpty()) {
+    // Avoid buffer boundary-dependent behavior when
+    // reparsing is forbidden. If reparse is forbidden,
+    // act as if we only saw the first 1024 bytes.
+    // When reparsing isn't forbidden, buffer boundaries
+    // can have an effect on whether the page is loaded
+    // once or twice. :-(
+    FeedDetector(mReparseForbidden ? aFromSegment.To(aCountToSniffingLimit)
+                                   : aFromSegment,
+                 false);
+  }
+  if (mFeedChardet && aEof &&
+      (!mReparseForbidden || aCountToSniffingLimit == aFromSegment.Length())) {
+    // Don't signal EOF if reparse is forbidden and we didn't pass all input
+    // to the detector above.
+    mFeedChardet = false;
+    FeedDetector(Span<const uint8_t>(), true);
+  }
+}
+
 nsresult nsHtml5StreamParser::FinalizeSniffing(Span<const uint8_t> aFromSegment,
                                                uint32_t aCountToSniffingLimit,
                                                bool aEof) {
-  NS_ASSERTION(IsParserThread(), "Wrong thread!");
-  NS_ASSERTION(mCharsetSource < kCharsetFromParentForced,
-               "Should not finalize sniffing when using forced charset.");
+  MOZ_ASSERT(IsParserThread(), "Wrong thread!");
+  MOZ_ASSERT(mCharsetSource < kCharsetFromUserForcedAutoDetection,
+             "Should not finalize sniffing with strong decision already made.");
   if (mMode == VIEW_SOURCE_XML) {
     static const XML_Memory_Handling_Suite memsuite = {
         (void* (*)(size_t))moz_xmalloc, (void* (*)(void*, size_t))moz_xrealloc,
@@ -591,38 +626,15 @@ nsresult nsHtml5StreamParser::FinalizeSniffing(Span<const uint8_t> aFromSegment,
   }
 
   // meta scan failed.
-  if (mCharsetSource >= kCharsetFromHintPrevDoc) {
-    mFeedChardet = false;
-    return SetupDecodingAndWriteSniffingBufferAndCurrentSegment(aFromSegment);
+  if (mCharsetSource < kCharsetFromMetaPrescan) {
+    // Check for BOMless UTF-16 with Basic
+    // Latin content for compat with IE. See bug 631751.
+    SniffBOMlessUTF16BasicLatin(aFromSegment.To(aCountToSniffingLimit));
   }
-  // Check for BOMless UTF-16 with Basic
-  // Latin content for compat with IE. See bug 631751.
-  SniffBOMlessUTF16BasicLatin(aFromSegment.To(aCountToSniffingLimit));
   // the charset may have been set now
   // maybe try chardet now;
   if (mFeedChardet && !mDecodingLocalFileAsUTF8) {
-    if (mSniffingBuffer) {
-      FeedDetector(MakeSpan(mSniffingBuffer.get(), mSniffingLength), false);
-    }
-    if (mFeedChardet && !aFromSegment.IsEmpty()) {
-        // Avoid buffer boundary-dependent behavior when
-        // reparsing is forbidden. If reparse is forbidden,
-        // act as if we only saw the first 1024 bytes.
-        // When reparsing isn't forbidden, buffer boundaries
-        // can have an effect on whether the page is loaded
-        // once or twice. :-(
-        FeedDetector(mReparseForbidden ? aFromSegment.To(aCountToSniffingLimit)
-                                       : aFromSegment,
-                     false);
-    }
-    if (mFeedChardet && aEof &&
-        (!mReparseForbidden ||
-         aCountToSniffingLimit == aFromSegment.Length())) {
-      // Don't signal EOF if reparse is forbidden and we didn't pass all input
-      // to the detector above.
-      mFeedChardet = false;
-      FeedDetector(Span<const uint8_t>(), true);
-    }
+    FinalizeSniffingWithDetector(aFromSegment, aCountToSniffingLimit, aEof);
     // fall thru; callback may have changed charset
   }
   if (mCharsetSource == kCharsetUninitialized) {
@@ -719,7 +731,6 @@ nsresult nsHtml5StreamParser::SniffStreamBytes(
     // earlier call to SetDocumentCharset(), since we didn't find a BOM and
     // overwrite mEncoding. (Note that if the user has overridden the charset,
     // we don't come here but check <meta> for XSS-dangerous charsets first.)
-    mFeedChardet = false;
     mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource);
     return SetupDecodingAndWriteSniffingBufferAndCurrentSegment(aFromSegment);
   }
@@ -751,12 +762,16 @@ nsresult nsHtml5StreamParser::SniffStreamBytes(
             (encoding->IsAsciiCompatible() ||
              encoding == ISO_2022_JP_ENCODING)) {
           // Honor override
+          if (mEncoding->IsJapaneseLegacy()) {
+            mFeedChardet = true;
+            FinalizeSniffingWithDetector(aFromSegment, countToSniffingLimit,
+                                         false);
+          }
           return SetupDecodingAndWriteSniffingBufferAndCurrentSegment(
               aFromSegment);
         }
         mEncoding = WrapNotNull(encoding);
         mCharsetSource = kCharsetFromMetaPrescan;
-        mFeedChardet = false;
         mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource);
         return SetupDecodingAndWriteSniffingBufferAndCurrentSegment(
             aFromSegment);
@@ -765,6 +780,10 @@ nsresult nsHtml5StreamParser::SniffStreamBytes(
     if (mCharsetSource == kCharsetFromParentForced ||
         mCharsetSource == kCharsetFromUserForced) {
       // meta not found, honor override
+      if (mEncoding->IsJapaneseLegacy()) {
+        mFeedChardet = true;
+        FinalizeSniffingWithDetector(aFromSegment, countToSniffingLimit, false);
+      }
       return SetupDecodingAndWriteSniffingBufferAndCurrentSegment(aFromSegment);
     }
     return FinalizeSniffing(aFromSegment, countToSniffingLimit, false);
@@ -793,7 +812,6 @@ nsresult nsHtml5StreamParser::SniffStreamBytes(
       }
       mEncoding = WrapNotNull(encoding);
       mCharsetSource = kCharsetFromMetaPrescan;
-      mFeedChardet = false;
       mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource);
       return SetupDecodingAndWriteSniffingBufferAndCurrentSegment(aFromSegment);
     }
@@ -1063,7 +1081,9 @@ nsresult nsHtml5StreamParser::OnStartRequest(nsIRequest* aRequest) {
     mInitialEncodingWasFromParentFrame = true;
   }
 
-  if (mCharsetSource >= kCharsetFromAutoDetection) {
+  if (mCharsetSource >= kCharsetFromAutoDetection &&
+      !(mCharsetSource == kCharsetFromParentForced ||
+        mCharsetSource == kCharsetFromUserForced)) {
     mFeedChardet = false;
   }
 
@@ -1073,9 +1093,9 @@ nsresult nsHtml5StreamParser::OnStartRequest(nsIRequest* aRequest) {
     return NS_OK;
   }
 
-  // We are reloading a document.open()ed doc or loading JSON/WebVTT/etc. into
-  // a browsing context. In the latter case, there's no need to remove the
-  // BOM manually here, because the UTF-8 decoder removes it.
+  // We are loading JSON/WebVTT/etc. into a browsing context.
+  // There's no need to remove the BOM manually here, because
+  // the UTF-8 decoder removes it.
   mReparseForbidden = true;
   mFeedChardet = false;
 

parser/html/nsHtml5StreamParser.h

@@ -293,6 +293,12 @@ class nsHtml5StreamParser final : public nsICharsetDetectionObserver {
    */
   void SniffBOMlessUTF16BasicLatin(mozilla::Span<const uint8_t> aFromSegment);
 
+  /**
+   * Write the start of the stream to detector.
+   */
+  void FinalizeSniffingWithDetector(mozilla::Span<const uint8_t> aFromSegment,
+                                    uint32_t aCountToSniffingLimit, bool aEof);
+
   /**
    * <meta charset> scan failed. Try chardet if applicable. After this, the
    * the parser will have some encoding even if a last resolt fallback.

parser/nsCharsetSource.h

@@ -14,14 +14,14 @@
 #define kCharsetFromCache 5
 #define kCharsetFromParentFrame 6
 #define kCharsetFromAutoDetection 7
-#define kCharsetFromHintPrevDoc 8
-#define kCharsetFromMetaPrescan 9  // this one and smaller: HTML5 Tentative
-#define kCharsetFromMetaTag 10     // this one and greater: HTML5 Confident
-#define kCharsetFromIrreversibleAutoDetection 11
-#define kCharsetFromChannel 12
-#define kCharsetFromOtherComponent 13
-#define kCharsetFromParentForced 14  // propagates to child frames
-#define kCharsetFromUserForced 15    // propagates to child frames
+#define kCharsetFromMetaPrescan 8  // this one and smaller: HTML5 Tentative
+#define kCharsetFromMetaTag 9      // this one and greater: HTML5 Confident
+#define kCharsetFromIrreversibleAutoDetection 10
+#define kCharsetFromChannel 11
+#define kCharsetFromOtherComponent 12
+#define kCharsetFromParentForced 13  // propagates to child frames
+#define kCharsetFromUserForced 14    // propagates to child frames
+#define kCharsetFromUserForcedAutoDetection 15
 #define kCharsetFromByteOrderMark 16
 #define kCharsetFromUtf8OnlyMime 17  // For JSON, WebVTT and such
 #define kCharsetFromBuiltIn 18       // resource: URLs

toolkit/content/widgets/browser-custom-element.js

@@ -240,6 +240,8 @@ class MozBrowser extends MozElements.MozElementMixin(XULFrameElement) {
 
     this._mayEnableCharacterEncodingMenu = null;
 
+    this._charsetAutodetected = false;
+
     this._contentPrincipal = null;
 
     this._csp = null;
@@ -616,6 +618,16 @@ class MozBrowser extends MozElements.MozElementMixin(XULFrameElement) {
     }
   }
 
+  get charsetAutodetected() {
+    return this.isRemoteBrowser ? this._charsetAutodetected : this.docShell.charsetAutodetected;
+  }
+
+  set charsetAutodetected(aAutodetected) {
+    if (this.isRemoteBrowser) {
+      this._charsetAutodetected = aAutodetected;
+    }
+  }
+
   get contentPrincipal() {
     return this.isRemoteBrowser ? this._contentPrincipal : this.contentDocument.nodePrincipal;
   }
@@ -1694,6 +1706,7 @@ class MozBrowser extends MozElements.MozElementMixin(XULFrameElement) {
         "_contentTitle",
         "_characterSet",
         "_mayEnableCharacterEncodingMenu",
+        "_charsetAutodetected",
         "_contentPrincipal",
         "_imageDocument",
         "_fullZoom",

toolkit/locales/en-US/chrome/global/charsetMenu.properties

@@ -31,8 +31,6 @@ charsetMenuAutodet = Auto-Detect
 charsetMenuAutodet.key = D
 charsetMenuAutodet.off = (off)
 charsetMenuAutodet.off.key = o
-charsetMenuAutodet.ja = Japanese
-charsetMenuAutodet.ja.key = J
 charsetMenuAutodet.ru = Russian
 charsetMenuAutodet.ru.key = R
 charsetMenuAutodet.uk = Ukrainian
@@ -104,25 +102,8 @@ windows-1255     = Hebrew
 # sorts right after that one in the collation order for your locale.
 ISO-8859-8       = Hebrew, Visual
 
-# Japanese
-Shift_JIS.key    = J
-Shift_JIS        = Japanese (Shift_JIS)
-EUC-JP.key       =   p
-EUC-JP           = Japanese (EUC-JP)
-ISO-2022-JP.key  =     n
-ISO-2022-JP      = Japanese (ISO-2022-JP)
-
-# UI string in anticipation of bug 1543077; deliberately not in use yet
-
-# LOCALIZATION NOTE (Japanese.key): If taken into use, this string will appear
-# instead of the string for Shift_JIS.key, so the use of the same
-# accelerator is deliberate.
+# Japanese (NOT AN ENCODING NAME)
 Japanese.key     = J
-# LOCALIZATION NOTE (Japanese): If taken into use, this string will appear
-# as a single item in place of the strings for the three items Shift_JIS,
-# EUC-JP, and ISO-2022-JP, so this string does not need to make sense together
-# with those strings and should be translated the way those were
-# but omitting the part in parentheses.
 Japanese         = Japanese
 
 # Korean

toolkit/modules/CharsetMenu.jsm

@@ -16,18 +16,16 @@ ChromeUtils.defineModuleGetter(this, "Deprecated",
 
 const kAutoDetectors = [
   ["off", ""],
-  ["ja", "ja_parallel_state_machine"],
   ["ru", "ruprob"],
   ["uk", "ukprob"],
 ];
 
 /**
  * This set contains encodings that are in the Encoding Standard, except:
- *  - XSS-dangerous encodings (except ISO-2022-JP which is assumed to be
- *    too common not to be included).
+ *  - Japanese encodings are represented by one autodetection item
  *  - x-user-defined, which practically never makes sense as an end-user-chosen
  *    override.
- *  - Encodings that IE11 doesn't have in its correspoding menu.
+ *  - Encodings that IE11 doesn't have in its corresponding menu.
  */
 const kEncodings = new Set([
   // Globally relevant
@@ -60,10 +58,8 @@ const kEncodings = new Set([
   // Hebrew
   "windows-1255",
   "ISO-8859-8",
-  // Japanese
-  "Shift_JIS",
-  "EUC-JP",
-  "ISO-2022-JP",
+  // Japanese (NOT AN ENCODING NAME)
+  "Japanese",
   // Korean
   "EUC-KR",
   // Thai
@@ -95,8 +91,7 @@ function CharsetComparator(a, b) {
   // happens to make the less frequently-used items first.
   let titleA = a.label.replace(/\(.*/, "") + b.value;
   let titleB = b.label.replace(/\(.*/, "") + a.value;
-  // Secondarily reverse sort by encoding name to sort "windows" or
-  // "shift_jis" first.
+  // Secondarily reverse sort by encoding name to sort "windows"
   return titleA.localeCompare(titleB) || b.value.localeCompare(a.value);
 }
 
@@ -239,7 +234,17 @@ var CharsetMenu = {
    * For substantially similar encodings, treat two encodings as the same
    * for the purpose of the check mark.
    */
-  foldCharset(charset) {
+  foldCharset(charset, isAutodetected) {
+    if (isAutodetected) {
+      switch (charset) {
+        case "Shift_JIS":
+        case "EUC-JP":
+        case "ISO-2022-JP":
+          return "Japanese";
+        default:
+          // fall through
+      }
+    }
     switch (charset) {
       case "ISO-8859-8-I":
         return "windows-1255";
@@ -252,8 +257,11 @@ var CharsetMenu = {
     }
   },
 
+  /**
+   * This method is for comm-central callers only.
+   */
   update(parent, charset) {
-    let menuitem = parent.getElementsByAttribute("charset", this.foldCharset(charset)).item(0);
+    let menuitem = parent.getElementsByAttribute("charset", this.foldCharset(charset, false)).item(0);
     if (menuitem) {
       menuitem.setAttribute("checked", "true");
     }

toolkit/modules/RemoteWebProgress.jsm

@@ -197,6 +197,7 @@ class RemoteWebProgressManager {
       if (json.charset) {
         this._browser._characterSet = json.charset;
         this._browser._mayEnableCharacterEncodingMenu = json.mayEnableCharacterEncodingMenu;
+        this._browser._charsetAutodetected = json.charsetAutodetected;
       }
     }
 

toolkit/modules/WebProgressChild.jsm

@@ -116,6 +116,7 @@ class WebProgressChild {
       json.title = this.mm.content.document.title;
       json.charset = this.mm.content.document.characterSet;
       json.mayEnableCharacterEncodingMenu = this.mm.docShell.mayEnableCharacterEncodingMenu;
+      json.charsetAutodetected = this.mm.docShell.charsetAutodetected;
       json.principal = this.mm.content.document.nodePrincipal;
       let csp = this.mm.content.document.csp;
       json.csp = E10SUtils.serializeCSP(csp);