fix for bug 110531, r=dougt, jag, sr=darin - moving ConverterInputStream into uconv, and changing the one in XPCOM into an UTF8-only converter

content/html/style/src/nsCSSLoader.cpp

@@ -1244,7 +1244,7 @@ CSSLoaderImpl::LoadSheet(URLKey& aKey, SheetLoadData* aData)
         if (NS_SUCCEEDED(result)) {
           // Translate the input using the argument character set id into unicode
           nsIUnicharInputStream* uin;
-          result = NS_NewConverterStream(&uin, nsnull, in);
+          result = NS_NewUTF8ConverterStream(&uin, in, 0);
           if (NS_SUCCEEDED(result)) {
             mLoadingSheets.Put(&aKey, aData);
             PRBool completed;

htmlparser/src/nsExpatTokenizer.cpp

@@ -51,7 +51,7 @@
 #include "prlog.h"
 
 #include "prmem.h"
-#include "nsIUnicharInputStream.h"
+#include "nsIConverterInputStream.h"
 #include "nsNetUtil.h"
 #include "nsIServiceManager.h"
 #include "nsCOMPtr.h"
@@ -822,9 +822,9 @@ IsLoadableDTD(nsCOMPtr<nsIURI>* aDTD)
       res = dtdURL->GetFileName(getter_Copies(fileName));
       if (NS_SUCCEEDED(res) && fileName) {
         nsSpecialSystemDirectory dtdPath(nsSpecialSystemDirectory::OS_CurrentProcessDirectory);
-        nsString path; path.AssignWithConversion(kDTDDirectory);        
+        nsCAutoString path(kDTDDirectory);        
-        path.AppendWithConversion(fileName.get());
+        path.Append(fileName);
-        dtdPath += path;
+        dtdPath += path.get();
         if (dtdPath.Exists()) {
           // The DTD was found in the local DTD directory.
           // Set aDTD to a file: url pointing to the local DTD
@@ -878,14 +878,10 @@ nsresult nsExpatTokenizer::LoadStream(nsIInputStream* in,
   // read it
   PRUint32               aCount = 1024,
                          bufsize = aCount*sizeof(PRUnichar);  
-  nsIUnicharInputStream *uniIn = nsnull;
+  nsCOMPtr<nsIUnicharInputStream> uniIn;
-  nsAutoString utf8; utf8.AssignWithConversion("UTF-8");
 
-  nsresult res = NS_NewConverterStream(&uniIn,
+  nsresult res = NS_NewUTF8ConverterStream(getter_AddRefs(uniIn),
-                                       nsnull,
+                                           in, aCount);
-                                       in,
-                                       aCount,
-                                       &utf8);
   if (NS_FAILED(res)) return res;
 
   PRUint32 aReadCount = 0;
@@ -909,7 +905,6 @@ nsresult nsExpatTokenizer::LoadStream(nsIInputStream* in,
   uniBuf = (PRUnichar *) PR_Malloc(retLen*sizeof(PRUnichar));
   nsCRT::memcpy(uniBuf, aBuf, sizeof(PRUnichar) * retLen);
   PR_FREEIF(aBuf);      
-  NS_RELEASE(uniIn);
 
   return res;
 }

layout/style/nsCSSLoader.cpp

@@ -1244,7 +1244,7 @@ CSSLoaderImpl::LoadSheet(URLKey& aKey, SheetLoadData* aData)
         if (NS_SUCCEEDED(result)) {
           // Translate the input using the argument character set id into unicode
           nsIUnicharInputStream* uin;
-          result = NS_NewConverterStream(&uin, nsnull, in);
+          result = NS_NewUTF8ConverterStream(&uin, in, 0);
           if (NS_SUCCEEDED(result)) {
             mLoadingSheets.Put(&aKey, aData);
             PRBool completed;

parser/htmlparser/src/nsExpatTokenizer.cpp

@@ -51,7 +51,7 @@
 #include "prlog.h"
 
 #include "prmem.h"
-#include "nsIUnicharInputStream.h"
+#include "nsIConverterInputStream.h"
 #include "nsNetUtil.h"
 #include "nsIServiceManager.h"
 #include "nsCOMPtr.h"
@@ -822,9 +822,9 @@ IsLoadableDTD(nsCOMPtr<nsIURI>* aDTD)
       res = dtdURL->GetFileName(getter_Copies(fileName));
       if (NS_SUCCEEDED(res) && fileName) {
         nsSpecialSystemDirectory dtdPath(nsSpecialSystemDirectory::OS_CurrentProcessDirectory);
-        nsString path; path.AssignWithConversion(kDTDDirectory);        
+        nsCAutoString path(kDTDDirectory);        
-        path.AppendWithConversion(fileName.get());
+        path.Append(fileName);
-        dtdPath += path;
+        dtdPath += path.get();
         if (dtdPath.Exists()) {
           // The DTD was found in the local DTD directory.
           // Set aDTD to a file: url pointing to the local DTD
@@ -878,14 +878,10 @@ nsresult nsExpatTokenizer::LoadStream(nsIInputStream* in,
   // read it
   PRUint32               aCount = 1024,
                          bufsize = aCount*sizeof(PRUnichar);  
-  nsIUnicharInputStream *uniIn = nsnull;
+  nsCOMPtr<nsIUnicharInputStream> uniIn;
-  nsAutoString utf8; utf8.AssignWithConversion("UTF-8");
 
-  nsresult res = NS_NewConverterStream(&uniIn,
+  nsresult res = NS_NewUTF8ConverterStream(getter_AddRefs(uniIn),
-                                       nsnull,
+                                           in, aCount);
-                                       in,
-                                       aCount,
-                                       &utf8);
   if (NS_FAILED(res)) return res;
 
   PRUint32 aReadCount = 0;
@@ -909,7 +905,6 @@ nsresult nsExpatTokenizer::LoadStream(nsIInputStream* in,
   uniBuf = (PRUnichar *) PR_Malloc(retLen*sizeof(PRUnichar));
   nsCRT::memcpy(uniBuf, aBuf, sizeof(PRUnichar) * retLen);
   PR_FREEIF(aBuf);      
-  NS_RELEASE(uniIn);
 
   return res;
 }

xpcom/ds/nsIPersistentProperties2.idl

@@ -53,12 +53,8 @@
 
 [scriptable, uuid(283EE646-1AEF-11D4-98B3-00C04fA0CE9A)]
 interface nsIPropertyElement : nsISupports {
-
+  attribute wstring key;
-  [noscript]  void        SetKey(in nsStringPtr aKey);
+  attribute wstring value;
-  [noscript]  void        SetValue(in nsStringPtr aValue);
-              wstring     getKey();
-              wstring     getValue();
-
 };
 
 

xpcom/ds/nsPersistentProperties.cpp

@@ -111,14 +111,8 @@ NS_IMETHODIMP
 nsPersistentProperties::Load(nsIInputStream *aIn)
 {
   PRInt32  c;
-  nsresult ret = NS_ERROR_FAILURE;
+  nsresult ret = NS_NewUTF8ConverterStream(&mIn, aIn, 0);
-
+  
-  nsAutoString uesc;
-  uesc.AssignWithConversion("UTF-8");
-
-#ifndef XPCOM_STANDALONE
-  ret = NS_NewConverterStream(&mIn, nsnull, aIn, 0, &uesc);
-#endif /* XPCOM_STANDALONE */
   if (ret != NS_OK) {
 #ifdef NS_DEBUG
       printf("NS_NewConverterStream failed\n");
@@ -298,17 +292,14 @@ PR_STATIC_CALLBACK(PRIntn)
 AddElemToArray(PLHashEntry* he, PRIntn i, void* arg)
 {
   nsISupportsArray	*propArray = (nsISupportsArray *) arg;
-
+  
-  nsString* keyStr = new nsString((PRUnichar*) he->key);
+  nsPropertyElement *element =
-  nsString* valueStr = new nsString((PRUnichar*) he->value);
+      new nsPropertyElement((PRUnichar*)he->key,
-
+                            (PRUnichar*)he->value);
-  nsPropertyElement *element = new nsPropertyElement();
   if (!element)
      return HT_ENUMERATE_STOP;
 
   NS_ADDREF(element);
-  element->SetKey(keyStr);
-  element->SetValue(valueStr);
   propArray->InsertElementAt(element, i);
 
   return HT_ENUMERATE_NEXT;
@@ -450,20 +441,7 @@ nsPersistentProperties::Has(const char* prop, PRBool *result)
 // PropertyElement
 ////////////////////////////////////////////////////////////////////////////////
 
-nsPropertyElement::nsPropertyElement()
-{
-    NS_INIT_REFCNT();
-    mKey = nsnull;
-    mValue = nsnull;
-}
 
-nsPropertyElement::~nsPropertyElement()
-{
-	if (mKey)
-		delete mKey;
-	if (mValue)
-		delete mValue;
-}
 NS_METHOD
 nsPropertyElement::Create(nsISupports *aOuter, REFNSIID aIID, void **aResult)
 {
@@ -485,7 +463,7 @@ nsPropertyElement::GetKey(PRUnichar **aReturnKey)
 {
   if (aReturnKey)
   {
-    *aReturnKey = ToNewUnicode(*mKey);
+    *aReturnKey = ToNewUnicode(mKey);
     return NS_OK;
   }
 
@@ -497,7 +475,7 @@ nsPropertyElement::GetValue(PRUnichar **aReturnValue)
 {
   if (aReturnValue)
   {
-    *aReturnValue = ToNewUnicode(*mValue);
+    *aReturnValue = ToNewUnicode(mValue);
     return NS_OK;
   }
 
@@ -505,18 +483,16 @@ nsPropertyElement::GetValue(PRUnichar **aReturnValue)
 }
 
 NS_IMETHODIMP
-nsPropertyElement::SetKey(nsString* aKey)
+nsPropertyElement::SetKey(const PRUnichar* aKey)
 {
   mKey = aKey;
-
   return NS_OK;
 }
 
 NS_IMETHODIMP
-nsPropertyElement::SetValue(nsString* aValue)
+nsPropertyElement::SetValue(const PRUnichar* aValue)
 {
   mValue = aValue;
-
   return NS_OK;
 }
 

xpcom/ds/nsPersistentProperties.h

@@ -74,6 +74,8 @@ public:
 
 protected:
   nsIUnicharInputStream* mIn;
+  PRUint32 mBufferPos;
+  PRUint32 mBufferLength;
   nsIPersistentProperties* mSubclass;
   struct PLHashTable*    mTable;
 };
@@ -81,8 +83,10 @@ protected:
 class nsPropertyElement : public nsIPropertyElement 
 {
 public:
-  nsPropertyElement();
+  nsPropertyElement() {};
-  virtual ~nsPropertyElement();
+  nsPropertyElement(const PRUnichar *aKey,
+                    const PRUnichar *aValue): mKey(aKey), mValue(aValue) {}
+  virtual ~nsPropertyElement() {};
 
   NS_DECL_ISUPPORTS
 
@@ -92,12 +96,12 @@ public:
   // nsIPropertyElement methods:
   NS_IMETHOD GetKey(PRUnichar **aReturnKey);
   NS_IMETHOD GetValue(PRUnichar **aReturnValue);
-  NS_IMETHOD SetKey(nsString* aKey);
+  NS_IMETHOD SetKey(const PRUnichar* aKey);
-  NS_IMETHOD SetValue(nsString* aValue);
+  NS_IMETHOD SetValue(const PRUnichar* aValue);
 
 protected:
-  nsString* mKey;
+  nsString mKey;
-  nsString* mValue;
+  nsString mValue;
 };
 
 #endif /* nsPersistentProperties_h___ */

xpcom/io/nsIUnicharInputStream.h

@@ -76,10 +76,8 @@ extern NS_COM nsresult
  * NS_INPUTSTREAM_NO_CONVERTER.
  */
 extern NS_COM nsresult
-  NS_NewConverterStream(nsIUnicharInputStream** aInstancePtrResult,
+  NS_NewUTF8ConverterStream(nsIUnicharInputStream** aInstancePtrResult,
-                        nsISupports* aOuter,
+                            nsIInputStream* aStreamToWrap,
-                        nsIInputStream* aStreamToWrap,
+                            PRInt32 aBufferSize = 0);
-                        PRInt32 aBufferSize = 0,
-                        nsString* aCharSet = nsnull);
 
 #endif /* nsUnicharInputStream_h___ */

xpcom/io/nsUnicharInputStream.cpp

@@ -35,15 +35,11 @@
  *
  * ***** END LICENSE BLOCK ***** */
 
-#ifndef XPCOM_STANDALONE
 
-#define NS_IMPL_IDS
 #include "nsIUnicharInputStream.h"
 #include "nsIByteBuffer.h"
 #include "nsIUnicharBuffer.h"
 #include "nsIServiceManager.h"
-#include "nsICharsetConverterManager.h"
-#include "nsIUnicodeDecoder.h"
 #include "nsString.h"
 #include "nsCRT.h"
 #include <fcntl.h>
@@ -53,8 +49,6 @@
 #include <unistd.h>
 #endif
 
-static NS_DEFINE_CID(kCharsetConverterManagerCID, NS_ICHARSETCONVERTERMANAGER_CID);
-
 class StringUnicharInputStream : public nsIUnicharInputStream {
 public:
   StringUnicharInputStream(nsString* aString);
@@ -142,44 +136,11 @@ NS_NewStringUnicharInputStream(nsIUnicharInputStream** aInstancePtrResult,
 
 //----------------------------------------------------------------------
 
-/**
+class UTF8InputStream : public nsIUnicharInputStream {
- * This function used to be public, with the NS_COM declaration. I am 
- * changing it right now into a module private visibility because there are
- * better and more xpcom-like ways to get a Converter.
- */
-
-nsresult NS_NewB2UConverter(nsIUnicodeDecoder** aInstancePtrResult, nsISupports* aOuter, nsString* aCharSet);
-nsresult
-NS_NewB2UConverter(nsIUnicodeDecoder** aInstancePtrResult,
-                   nsISupports* aOuter,
-                   nsString* aCharSet)
-{
-  if (nsnull != aOuter) {
-    return NS_ERROR_NO_AGGREGATION;
-  }
-
-  // Create converter
-  nsresult res;
-  nsAutoString defaultCharset;
-  defaultCharset.AssignWithConversion("ISO-8859-1");
-
-  if (aCharSet == nsnull) aCharSet = &defaultCharset;
-
-  nsCOMPtr<nsICharsetConverterManager> ccm = 
-           do_GetService(kCharsetConverterManagerCID, &res);
-  if (NS_FAILED(res)) return res;
-
-  return ccm->GetUnicodeDecoder(aCharSet, aInstancePtrResult);
-}
-
-//----------------------------------------------------------------------
-
-class ConverterInputStream : public nsIUnicharInputStream {
 public:
-  ConverterInputStream(nsIInputStream* aStream,
+  UTF8InputStream(nsIInputStream* aStream,
-                       nsIUnicodeDecoder* aConverter,
+                  PRUint32 aBufSize);
-                       PRUint32 aBufSize);
+  virtual ~UTF8InputStream();
-  virtual ~ConverterInputStream();
 
   NS_DECL_ISUPPORTS
   NS_IMETHOD Read(PRUnichar* aBuf,
@@ -191,70 +152,55 @@ public:
 protected:
   PRInt32 Fill(nsresult * aErrorCode);
 
-  nsIInputStream* mInput;
+  static PRInt32 CountValidUTF8Bytes(const char *aBuf, PRInt32 aMaxBytes);
-  nsIUnicodeDecoder* mConverter;
+
-  nsIByteBuffer* mByteData;
+  nsCOMPtr<nsIInputStream> mInput;
+  nsCOMPtr<nsIByteBuffer> mByteData;
+  nsCOMPtr<nsIUnicharBuffer> mUnicharData;
+  
   PRUint32 mByteDataOffset;
-  nsIUnicharBuffer* mUnicharData;
   PRUint32 mUnicharDataOffset;
   PRUint32 mUnicharDataLength;
 };
 
-ConverterInputStream::ConverterInputStream(nsIInputStream* aStream,
+UTF8InputStream::UTF8InputStream(nsIInputStream* aStream,
-                                           nsIUnicodeDecoder* aConverter,
+                                 PRUint32 aBufferSize) :
-                                           PRUint32 aBufferSize)
+  mInput(aStream)
 {
   NS_INIT_REFCNT();
-  mInput = aStream;
-  NS_ADDREF(aStream);
-  mConverter = aConverter;
-  NS_ADDREF(aConverter);
   if (aBufferSize == 0) {
     aBufferSize = 8192;
   }
 
   // XXX what if these fail?
-  NS_NewByteBuffer(&mByteData, nsnull, aBufferSize);
+  NS_NewByteBuffer(getter_AddRefs(mByteData), nsnull, aBufferSize);
-  NS_NewUnicharBuffer(&mUnicharData, nsnull, aBufferSize);
+  NS_NewUnicharBuffer(getter_AddRefs(mUnicharData), nsnull, aBufferSize);
 
   mByteDataOffset = 0;
   mUnicharDataOffset = 0;
   mUnicharDataLength = 0;
 }
 
-NS_IMPL_ISUPPORTS1(ConverterInputStream,nsIUnicharInputStream)
+NS_IMPL_ISUPPORTS1(UTF8InputStream,nsIUnicharInputStream)
 
-ConverterInputStream::~ConverterInputStream()
+UTF8InputStream::~UTF8InputStream()
 {
   Close();
 }
 
-nsresult ConverterInputStream::Close()
+nsresult UTF8InputStream::Close()
 {
-  if (nsnull != mInput) {
+  mInput = nsnull;
-    NS_RELEASE(mInput);
+  mByteData = nsnull;
-    mInput = nsnull;
+  mUnicharData = nsnull;
-  }
-  if (nsnull != mConverter) {
-    NS_RELEASE(mConverter);
-    mConverter = nsnull;
-  }
-  if (nsnull != mByteData) {
-    NS_RELEASE(mByteData);
-    mByteData = nsnull;
-  }
-  if (nsnull != mUnicharData) {
-    NS_RELEASE(mUnicharData);
-    mUnicharData = nsnull;
-  }
 
   return NS_OK;
 }
 
-nsresult ConverterInputStream::Read(PRUnichar* aBuf,
+nsresult UTF8InputStream::Read(PRUnichar* aBuf,
-                                    PRUint32 aOffset,
+                               PRUint32 aOffset,
-                                    PRUint32 aCount,
+                               PRUint32 aCount,
-                                    PRUint32 *aReadCount)
+                               PRUint32 *aReadCount)
 {
   NS_ASSERTION(mUnicharDataLength >= mUnicharDataOffset, "unsigned madness");
   PRUint32 rv = mUnicharDataLength - mUnicharDataOffset;
@@ -277,7 +223,7 @@ nsresult ConverterInputStream::Read(PRUnichar* aBuf,
   return NS_OK;
 }
 
-PRInt32 ConverterInputStream::Fill(nsresult * aErrorCode)
+PRInt32 UTF8InputStream::Fill(nsresult * aErrorCode)
 {
   if (nsnull == mInput) {
     // We already closed the stream!
@@ -300,43 +246,92 @@ PRInt32 ConverterInputStream::Fill(nsresult * aErrorCode)
   NS_ASSERTION(remainder + nb == mByteData->GetLength(), "bad nb");
 
   // Now convert as much of the byte buffer to unicode as possible
-  PRInt32 dstLen = mUnicharData->GetBufferSize();
+  PRInt32 srcLen = CountValidUTF8Bytes(mByteData->GetBuffer(),remainder + nb);
-  PRInt32 srcLen = remainder + nb;
+
-  *aErrorCode = mConverter->Convert(mByteData->GetBuffer(), &srcLen,
+  NS_ConvertUTF8toUCS2
-                                    mUnicharData->GetBuffer(), &dstLen);
+    unicodeValue(Substring(mByteData->GetBuffer(),
+                           mByteData->GetBuffer() + srcLen));
+  PRInt32 dstLen = unicodeValue.Length();
+  
+  // the number of UCS2 characters should always be <= the number of
+  // UTF8 chars
+  NS_ASSERTION(dstLen <= mUnicharData->GetBufferSize(),
+               "Ouch. I would overflow my buffer if I wasn't so careful.");
+  if (dstLen > mUnicharData->GetBufferSize()) return 0;
+  
+  nsCRT::memcpy((void *)mUnicharData->GetBuffer(),
+                (void *)unicodeValue.get(), dstLen*sizeof(PRUnichar));
+
   mUnicharDataOffset = 0;
   mUnicharDataLength = dstLen;
   mByteDataOffset += srcLen;
+  
   return dstLen;
 }
 
-// XXX hook up auto-detect here (do we need more info, like the url?)
+PRInt32
-NS_COM nsresult
+UTF8InputStream::CountValidUTF8Bytes(const char* aBuffer, PRInt32 aMaxBytes)
-NS_NewConverterStream(nsIUnicharInputStream** aInstancePtrResult,
-                      nsISupports* aOuter,
-                      nsIInputStream* aStreamToWrap,
-                      PRInt32 aBufferSize,
-                      nsString* aCharSet)
 {
-  if (nsnull != aOuter) {
+  const char *c = aBuffer;
-    return NS_ERROR_NO_AGGREGATION;
+  const char *lastchar = aBuffer;
+  
+  PRInt32 bytes = 0;
+  while (*c && bytes <= aMaxBytes) {
+    lastchar = c;
+    if (UTF8traits::isASCII(*c)) {
+      c++;
+      bytes++;
+    }
+    
+    else if (UTF8traits::is2byte(*c)) {
+      c += 2;
+      bytes += 2;
+    }
+      
+    else if (UTF8traits::is3byte(*c)) {
+      c += 3;
+      bytes += 3;
+    }
+    
+    else if (UTF8traits::is4byte(*c)) {
+      c += 4;
+      bytes += 4;
+    }
+    
+    else if (UTF8traits::is5byte(*c)) {
+      c += 5;
+      bytes += 5;
+    }
+    else if (UTF8traits::is6byte(*c)) {
+      c+=6;
+      bytes +=6;
+    }
+    
+    else
+      NS_WARNING("Unrecognized UTF8 string in UTF8InputStream::CountValidUTF8Bytes()");
   }
 
-  // Create converter
+  // if we skipped pas the end of the buffer, back up to the last character
-  nsIUnicodeDecoder* converter;
+  if (bytes > aMaxBytes) {
-  nsresult rv = NS_NewB2UConverter(&converter, nsnull, aCharSet);
+    c = lastchar;
-  if (NS_OK != rv) {
+    bytes = (c-aBuffer);
-    return rv;
   }
+  
+  return bytes;
+}
 
+NS_COM nsresult
+NS_NewUTF8ConverterStream(nsIUnicharInputStream** aInstancePtrResult,
+                          nsIInputStream* aStreamToWrap,
+                          PRInt32 aBufferSize)
+{
   // Create converter input stream
-  ConverterInputStream* it =
+  UTF8InputStream* it =
-    new ConverterInputStream(aStreamToWrap, converter, aBufferSize);
+    new UTF8InputStream(aStreamToWrap, aBufferSize);
-  NS_RELEASE(converter);
+  
   if (nsnull == it) {
     return NS_ERROR_OUT_OF_MEMORY;
   }
   return it->QueryInterface(NS_GET_IID(nsIUnicharInputStream), 
                             (void **) aInstancePtrResult);
 }
-#endif /* XPCOM_STANDALONE */

xpcom/tests/CvtURL.cpp

@@ -36,7 +36,7 @@
  * ***** END LICENSE BLOCK ***** */
 #include <stdio.h>
 #include "nscore.h"
-#include "nsIUnicharInputStream.h"
+#include "nsIConverterInputStream.h"
 #include "nsIURL.h"
 #include "nsNetUtil.h"
 #include "nsCRT.h"
@@ -52,7 +52,7 @@ static nsString* ConvertCharacterSetName(const char* aName)
 int main(int argc, char** argv)
 {
   if (3 != argc) {
-    printf("usage: CvtURL url character-set-name\n");
+    printf("usage: CvtURL url utf8\n");
     return -1;
   }
 
@@ -82,9 +82,12 @@ int main(int argc, char** argv)
     return -1;
   }
 
-  // Translate the input using the argument character set id into unicode
+  // Translate the input using the argument character set id into
-  nsIUnicharInputStream* uin;
+  // unicode
-  rv = NS_NewConverterStream(&uin, nsnull, in, 0, cset);
+  nsCOMPtr<nsIConverterInputStream> uin =
+    do_CreateInstance("@mozilla.org/intl/converter-input-stream;1", &rv);
+  if (NS_SUCCEEDED(rv))
+    uin->Init(in, cset->get(), nsnull);
   if (NS_OK != rv) {
     printf("can't create converter input stream: %d\n", rv);
     return -1;
@@ -117,7 +120,6 @@ int main(int argc, char** argv)
 
   // Release the objects
   in->Release();
-  uin->Release();
   url->Release();
 
   return 0;