bug 125359, by default the ASN.1 en/decoder should treat all numbers as signed. But many source/target items desire unsigned integers (specifically, bignums in the crypto stuff), so implement an siUnsignedInteger type which notifies the en/decoder to handle the conversion.

r=nelsonb

security/nss/lib/cryptohi/dsautil.c

@@ -149,8 +149,10 @@ DSAU_EncodeDerSig(SECItem *dest, SECItem *src)
     ** prepend with leading zero.  
     ** Must remove all but one leading zero byte from numbers.
     */
+    sig.r.type = siUnsignedInteger;
     sig.r.data = signedR;
     sig.r.len  = sizeof signedR;
+    sig.s.type = siUnsignedInteger;
     sig.s.data = signedS;
     sig.s.len  = sizeof signedR;
 
@@ -193,6 +195,8 @@ DSAU_DecodeDerSig(SECItem *item)
     if (result->data == NULL)
 	goto loser;
 
+    sig.r.type = siUnsignedInteger;
+    sig.s.type = siUnsignedInteger;
     status = SEC_ASN1DecodeItem(NULL, &sig, DSA_SignatureTemplate, item);
     if (status != SECSuccess)
 	goto loser;

security/nss/lib/cryptohi/seckey.c

@@ -143,6 +143,48 @@ SEC_ASN1_CHOOSER_IMPLEMENT(SECKEY_DSAPublicKeyTemplate)
 SEC_ASN1_CHOOSER_IMPLEMENT(SECKEY_RSAPublicKeyTemplate)
 SEC_ASN1_CHOOSER_IMPLEMENT(CERT_SubjectPublicKeyInfoTemplate)
 
+/*
+ * See bugzilla bug 125359
+ * Since NSS (via PKCS#11) wants to handle big integers as unsigned ints,
+ * all of the templates above that en/decode into integers must be converted
+ * from ASN.1's signed integer type.  This is done by marking either the
+ * source or destination (encoding or decoding, respectively) type as
+ * siUnsignedInteger.
+ */
+static void
+prepare_rsa_pub_key_for_asn1(SECKEYPublicKey *pubk)
+{
+    pubk->u.rsa.modulus.type = siUnsignedInteger;
+    pubk->u.rsa.publicExponent.type = siUnsignedInteger;
+}
+
+static void
+prepare_dsa_pub_key_for_asn1(SECKEYPublicKey *pubk)
+{
+    pubk->u.dsa.publicValue.type = siUnsignedInteger;
+}
+
+static void
+prepare_pqg_params_for_asn1(SECKEYPQGParams *params)
+{
+    params->prime.type = siUnsignedInteger;
+    params->subPrime.type = siUnsignedInteger;
+    params->base.type = siUnsignedInteger;
+}
+
+static void
+prepare_dh_pub_key_for_asn1(SECKEYPublicKey *pubk)
+{
+    pubk->u.dh.prime.type = siUnsignedInteger;
+    pubk->u.dh.base.type = siUnsignedInteger;
+    pubk->u.dh.publicValue.type = siUnsignedInteger;
+}
+
+static void
+prepare_kea_pub_key_for_asn1(SECKEYPublicKey *pubk)
+{
+    pubk->u.kea.publicValue.type = siUnsignedInteger;
+}
 
 /* Create an RSA key pair is any slot able to do so.
 ** The created keys are "session" (temporary), not "token" (permanent), 
@@ -509,6 +551,7 @@ SECKEY_FortezzaDecodePQGtoOld(PRArenaPool *arena, SECKEYPublicKey *pubk,
             /* PQG params are in the standard format */
 
 	    /* Store DSA PQG parameters */
+	    prepare_pqg_params_for_asn1(&pubk->u.fortezza.params);
             rv = SEC_ASN1DecodeItem(arena, &pubk->u.fortezza.params,
                               SECKEY_PQGParamsTemplate,
                               params);
@@ -628,6 +671,7 @@ SECKEY_DSADecodePQG(PRArenaPool *arena, SECKEYPublicKey *pubk, SECItem *params)
         (params->data[0] != 0xa0)) {
     
          /* PQG params are in the standard format */
+         prepare_pqg_params_for_asn1(&pubk->u.dsa.params);
          rv = SEC_ASN1DecodeItem(arena, &pubk->u.dsa.params,
                              SECKEY_PQGParamsTemplate,
                              params);
@@ -875,6 +919,7 @@ seckey_ExtractPublicKey(CERTSubjectPublicKeyInfo *spki)
       case SEC_OID_X500_RSA_ENCRYPTION:
       case SEC_OID_PKCS1_RSA_ENCRYPTION:
 	pubk->keyType = rsaKey;
+	prepare_rsa_pub_key_for_asn1(pubk);
 	rv = SEC_ASN1DecodeItem(arena, pubk, SECKEY_RSAPublicKeyTemplate, &os);
 	if (rv == SECSuccess)
 	    return pubk;
@@ -882,6 +927,7 @@ seckey_ExtractPublicKey(CERTSubjectPublicKeyInfo *spki)
       case SEC_OID_ANSIX9_DSA_SIGNATURE:
       case SEC_OID_SDN702_DSA_SIGNATURE:
 	pubk->keyType = dsaKey;
+	prepare_dsa_pub_key_for_asn1(pubk);
 	rv = SEC_ASN1DecodeItem(arena, pubk, SECKEY_DSAPublicKeyTemplate, &os);
 	if (rv != SECSuccess) break;
 
@@ -892,6 +938,7 @@ seckey_ExtractPublicKey(CERTSubjectPublicKeyInfo *spki)
 	break;
       case SEC_OID_X942_DIFFIE_HELMAN_KEY:
 	pubk->keyType = dhKey;
+	prepare_dh_pub_key_for_asn1(pubk);
 	rv = SEC_ASN1DecodeItem(arena, pubk, SECKEY_DHPublicKeyTemplate, &os);
 	if (rv != SECSuccess) break;
 
@@ -914,6 +961,7 @@ seckey_ExtractPublicKey(CERTSubjectPublicKeyInfo *spki)
       case SEC_OID_MISSI_KEA:
 	pubk->keyType = keaKey;
 
+	prepare_kea_pub_key_for_asn1(pubk);
         rv = SEC_ASN1DecodeItem(arena, pubk,
                                 SECKEY_KEAPublicKeyTemplate, &os);
         if (rv != SECSuccess) break;
@@ -1269,6 +1317,7 @@ SECKEY_CreateSubjectPublicKeyInfo(SECKEYPublicKey *pubk)
 		/*
 		 * DER encode the public key into the subjectPublicKeyInfo.
 		 */
+		prepare_rsa_pub_key_for_asn1(pubk);
 		rv_item = SEC_ASN1EncodeItem(arena, &spki->subjectPublicKey,
 					     pubk, SECKEY_RSAPublicKeyTemplate);
 		if (rv_item != NULL) {
@@ -1286,6 +1335,7 @@ SECKEY_CreateSubjectPublicKeyInfo(SECKEYPublicKey *pubk)
 	    break;
 	  case dsaKey:
 	    /* DER encode the params. */
+	    prepare_pqg_params_for_asn1(&pubk->u.dsa.params);
 	    rv_item = SEC_ASN1EncodeItem(arena, &params, &pubk->u.dsa.params,
 					 SECKEY_PQGParamsTemplate);
 	    if (rv_item != NULL) {
@@ -1296,6 +1346,7 @@ SECKEY_CreateSubjectPublicKeyInfo(SECKEYPublicKey *pubk)
 		    /*
 		     * DER encode the public key into the subjectPublicKeyInfo.
 		     */
+		    prepare_dsa_pub_key_for_asn1(pubk);
 		    rv_item = SEC_ASN1EncodeItem(arena, &spki->subjectPublicKey,
 						 pubk,
 						 SECKEY_DSAPublicKeyTemplate);
@@ -1390,6 +1441,7 @@ SECKEY_DecodeDERPublicKey(SECItem *pubkder)
 	pubk->arena = arena;
 	pubk->pkcs11Slot = NULL;
 	pubk->pkcs11ID = 0;
+	prepare_rsa_pub_key_for_asn1(pubk);
 	rv = SEC_ASN1DecodeItem(arena, pubk, SECKEY_RSAPublicKeyTemplate,
 				pubkder);
 	if (rv == SECSuccess)
@@ -1732,14 +1784,17 @@ SECKEY_ImportDERPublicKey(SECItem *derKey, CK_KEY_TYPE type)
 
     switch( type ) {
       case CKK_RSA:
+	prepare_rsa_pub_key_for_asn1(pubk);
         rv = SEC_ASN1DecodeItem(NULL, pubk, SECKEY_RSAPublicKeyTemplate,derKey);
         pubk->keyType = rsaKey;
         break;
       case CKK_DSA:
+	prepare_dsa_pub_key_for_asn1(pubk);
         rv = SEC_ASN1DecodeItem(NULL, pubk, SECKEY_DSAPublicKeyTemplate,derKey);
         pubk->keyType = dsaKey;
         break;
       case CKK_DH:
+	prepare_dh_pub_key_for_asn1(pubk);
         rv = SEC_ASN1DecodeItem(NULL, pubk, SECKEY_DHPublicKeyTemplate, derKey);
         pubk->keyType = dhKey;
         break;

security/nss/lib/pk11wrap/pk11pk12.c

@@ -185,6 +185,45 @@ SEC_ASN1_CHOOSER_IMPLEMENT(SECKEY_PointerToEncryptedPrivateKeyInfoTemplate)
 SEC_ASN1_CHOOSER_IMPLEMENT(SECKEY_PrivateKeyInfoTemplate)
 SEC_ASN1_CHOOSER_IMPLEMENT(SECKEY_PointerToPrivateKeyInfoTemplate)
 
+/*
+ * See bugzilla bug 125359
+ * Since NSS (via PKCS#11) wants to handle big integers as unsigned ints,
+ * all of the templates above that en/decode into integers must be converted
+ * from ASN.1's signed integer type.  This is done by marking either the
+ * source or destination (encoding or decoding, respectively) type as
+ * siUnsignedInteger.
+ */
+
+static void
+prepare_rsa_priv_key_export_for_asn1(SECKEYRawPrivateKey *key)
+{
+    key->u.rsa.modulus.type = siUnsignedInteger;
+    key->u.rsa.publicExponent.type = siUnsignedInteger;
+    key->u.rsa.privateExponent.type = siUnsignedInteger;
+    key->u.rsa.prime1.type = siUnsignedInteger;
+    key->u.rsa.prime2.type = siUnsignedInteger;
+    key->u.rsa.exponent1.type = siUnsignedInteger;
+    key->u.rsa.exponent2.type = siUnsignedInteger;
+    key->u.rsa.coefficient.type = siUnsignedInteger;
+}
+
+static void
+prepare_dsa_priv_key_export_for_asn1(SECKEYRawPrivateKey *key)
+{
+    key->u.dsa.privateValue.type = siUnsignedInteger;
+    key->u.dsa.params.prime.type = siUnsignedInteger;
+    key->u.dsa.params.subPrime.type = siUnsignedInteger;
+    key->u.dsa.params.base.type = siUnsignedInteger;
+}
+
+static void
+prepare_dh_priv_key_export_for_asn1(SECKEYRawPrivateKey *key)
+{
+    key->u.dh.privateValue.type = siUnsignedInteger;
+    key->u.dh.prime.type = siUnsignedInteger;
+    key->u.dh.base.type = siUnsignedInteger;
+}
+
 
 SECStatus
 PK11_ImportDERPrivateKeyInfo(PK11SlotInfo *slot, SECItem *derPKI, 
@@ -437,6 +476,7 @@ PK11_ImportPrivateKeyInfoAndReturnKey(PK11SlotInfo *slot,
 
     switch(SECOID_GetAlgorithmTag(&pki->algorithm)) {
 	case SEC_OID_PKCS1_RSA_ENCRYPTION:
+	    prepare_rsa_priv_key_export_for_asn1(lpk);
 	    keyTemplate = SECKEY_RSAPrivateKeyExportTemplate;
 	    paramTemplate = NULL;
 	    paramDest = NULL;
@@ -444,6 +484,7 @@ PK11_ImportPrivateKeyInfoAndReturnKey(PK11SlotInfo *slot,
 	    keyType = CKK_RSA;
 	    break;
 	case SEC_OID_ANSIX9_DSA_SIGNATURE:
+	    prepare_dsa_priv_key_export_for_asn1(lpk);
 	    keyTemplate = SECKEY_DSAPrivateKeyExportTemplate;
 	    paramTemplate = SECKEY_PQGParamsTemplate;
 	    paramDest = &(lpk->u.dsa.params);
@@ -454,6 +495,7 @@ PK11_ImportPrivateKeyInfoAndReturnKey(PK11SlotInfo *slot,
 	    if(!publicValue) {
 		goto loser;
 	    }
+	    prepare_dh_priv_key_export_for_asn1(lpk);
 	    keyTemplate = SECKEY_DHPrivateKeyExportTemplate;
 	    paramTemplate = NULL;
 	    paramDest = NULL;

security/nss/lib/softoken/keydb.c

@@ -32,7 +32,7 @@
  *
  * Private Key Database code
  *
- * $Id: keydb.c,v 1.12 2001-12-07 01:36:17 relyea%netscape.com Exp $
+ * $Id: keydb.c,v 1.13 2002-02-21 22:41:37 ian.mcgreer%sun.com Exp $
  */
 
 #include "lowkeyi.h"
@@ -1579,6 +1579,7 @@ seckey_encrypt_private_key(
     /* Encode the key, and set the algorithm (with params) */
     switch (pk->keyType) {
       case NSSLOWKEYRSAKey:
+        prepare_low_rsa_priv_key_for_asn1(pk);
 	dummy = SEC_ASN1EncodeItem(temparena, &(pki->privateKey), pk, 
 				   nsslowkey_RSAPrivateKeyTemplate);
 	if (dummy == NULL) {
@@ -1594,6 +1595,7 @@ seckey_encrypt_private_key(
 	
 	break;
       case NSSLOWKEYDSAKey:
+        prepare_low_dsa_priv_key_for_asn1(pk);
 	dummy = SEC_ASN1EncodeItem(temparena, &(pki->privateKey), pk,
 				   nsslowkey_DSAPrivateKeyTemplate);
 	if (dummy == NULL) {
@@ -1601,6 +1603,7 @@ seckey_encrypt_private_key(
 	    goto loser;
 	}
 	
+        prepare_low_pqg_params_for_asn1(&pk->u.dsa.params);
 	dummy = SEC_ASN1EncodeItem(temparena, NULL, &pk->u.dsa.params,
 				   nsslowkey_PQGParamsTemplate);
 	if (dummy == NULL) {
@@ -1616,6 +1619,7 @@ seckey_encrypt_private_key(
 	
 	break;
       case NSSLOWKEYDHKey:
+        prepare_low_dh_priv_key_for_asn1(pk);
 	dummy = SEC_ASN1EncodeItem(temparena, &(pki->privateKey), pk,
 				   nsslowkey_DHPrivateKeyTemplate);
 	if (dummy == NULL) {
@@ -1860,23 +1864,27 @@ seckey_decrypt_private_key(NSSLOWKEYEncryptedPrivateKeyInfo *epki,
 	      case SEC_OID_X500_RSA_ENCRYPTION:
 	      case SEC_OID_PKCS1_RSA_ENCRYPTION:
 		pk->keyType = NSSLOWKEYRSAKey;
+		prepare_low_rsa_priv_key_for_asn1(pk);
 		rv = SEC_ASN1DecodeItem(permarena, pk,
 					nsslowkey_RSAPrivateKeyTemplate,
 					&pki->privateKey);
 		break;
 	      case SEC_OID_ANSIX9_DSA_SIGNATURE:
 		pk->keyType = NSSLOWKEYDSAKey;
+		prepare_low_dsa_priv_key_for_asn1(pk);
 		rv = SEC_ASN1DecodeItem(permarena, pk,
 					nsslowkey_DSAPrivateKeyTemplate,
 					&pki->privateKey);
 		if (rv != SECSuccess)
 		    goto loser;
+		prepare_low_pqg_params_for_asn1(&pk->u.dsa.params);
 		rv = SEC_ASN1DecodeItem(permarena, &pk->u.dsa.params,
 					nsslowkey_PQGParamsTemplate,
 					&pki->algorithm.parameters);
 		break;
 	      case SEC_OID_X942_DIFFIE_HELMAN_KEY:
 		pk->keyType = NSSLOWKEYDHKey;
+		prepare_low_dh_priv_key_for_asn1(pk);
 		rv = SEC_ASN1DecodeItem(permarena, pk,
 					nsslowkey_DHPrivateKeyTemplate,
 					&pki->privateKey);

security/nss/lib/softoken/lowcert.c

@@ -34,7 +34,7 @@
 /*
  * Certificate handling code
  *
- * $Id: lowcert.c,v 1.5 2002-01-17 00:20:52 ian.mcgreer%sun.com Exp $
+ * $Id: lowcert.c,v 1.6 2002-02-21 22:41:37 ian.mcgreer%sun.com Exp $
  */
 
 #include "seccomon.h"
@@ -117,6 +117,39 @@ const SEC_ASN1Template nsslowcert_DHPublicKeyTemplate[] = {
     { 0, }
 };
 
+/*
+ * See bugzilla bug 125359
+ * Since NSS (via PKCS#11) wants to handle big integers as unsigned ints,
+ * all of the templates above that en/decode into integers must be converted
+ * from ASN.1's signed integer type.  This is done by marking either the
+ * source or destination (encoding or decoding, respectively) type as
+ * siUnsignedInteger.
+ */
+
+static void
+prepare_low_rsa_pub_key_for_asn1(NSSLOWKEYPublicKey *pubk)
+{
+    pubk->u.rsa.modulus.type = siUnsignedInteger;
+    pubk->u.rsa.publicExponent.type = siUnsignedInteger;
+}
+
+static void
+prepare_low_dsa_pub_key_for_asn1(NSSLOWKEYPublicKey *pubk)
+{
+    pubk->u.dsa.publicValue.type = siUnsignedInteger;
+    pubk->u.dsa.params.prime.type = siUnsignedInteger;
+    pubk->u.dsa.params.subPrime.type = siUnsignedInteger;
+    pubk->u.dsa.params.base.type = siUnsignedInteger;
+}
+
+static void
+prepare_low_dh_pub_key_for_asn1(NSSLOWKEYPublicKey *pubk)
+{
+    pubk->u.dh.prime.type = siUnsignedInteger;
+    pubk->u.dh.base.type = siUnsignedInteger;
+    pubk->u.dh.publicValue.type = siUnsignedInteger;
+}
+
 /*
  * Allow use of default cert database, so that apps(such as mozilla) don't
  * have to pass the handle all over the place.
@@ -339,45 +372,23 @@ nsslowcert_FixupEmailAddr(char *emailAddr)
     return(retaddr);
 }
 
-/* NSS has traditionally keyed certificate entries in the cert database
- * by (serial number, DER_ISSUER).  The serial number may have a leading zero
- * in order to make it a signed integer.  However, the ASN.1 decoder now 
- * strips the leading zero, treating any INTEGER as unsigned.  In order to
- * be compatible with version 7 of the database, it is necessary to reapply
- * that leading zero to the serial number when needed, before computing the
- * database key.
- */
 static SECStatus
 nsslowcert_KeyFromIssuerAndSN(PRArenaPool *arena, SECItem *issuer, SECItem *sn,
 			SECItem *key)
 {
-    PRBool leadingZero = PR_FALSE;
-    int start;
 
     key->len = sn->len + issuer->len;
 
-    if (sn->data[0] & 0x80) {
-	leadingZero = PR_TRUE;
-	key->len++;
-    }
-    
     key->data = (unsigned char*)PORT_ArenaAlloc(arena, key->len);
     if ( !key->data ) {
 	goto loser;
     }
 
-    if (leadingZero) {
-	key->data[0] = 0;
-	start = 1;
-    } else {
-	start = 0;
-    }
-
     /* copy the serialNumber */
-    PORT_Memcpy(key->data + start, sn->data, sn->len);
+    PORT_Memcpy(key->data, sn->data, sn->len);
 
     /* copy the issuer */
-    PORT_Memcpy(&key->data[start + sn->len], issuer->data, issuer->len);
+    PORT_Memcpy(&key->data[sn->len], issuer->data, issuer->len);
 
     return(SECSuccess);
 
@@ -453,6 +464,7 @@ nsslowcert_ExtractPublicKey(NSSLOWCERTCertificate *cert)
       case SEC_OID_X500_RSA_ENCRYPTION:
       case SEC_OID_PKCS1_RSA_ENCRYPTION:
         pubk->keyType = NSSLOWKEYRSAKey;
+        prepare_low_rsa_pub_key_for_asn1(pubk);
         rv = SEC_ASN1DecodeItem(arena, pubk, 
 				nsslowcert_RSAPublicKeyTemplate, &os);
         if (rv == SECSuccess)
@@ -460,12 +472,14 @@ nsslowcert_ExtractPublicKey(NSSLOWCERTCertificate *cert)
         break;
       case SEC_OID_ANSIX9_DSA_SIGNATURE:
         pubk->keyType = NSSLOWKEYDSAKey;
+        prepare_low_dsa_pub_key_for_asn1(pubk);
         rv = SEC_ASN1DecodeItem(arena, pubk,
 				 nsslowcert_DSAPublicKeyTemplate, &os);
         if (rv == SECSuccess) return pubk;
         break;
       case SEC_OID_X942_DIFFIE_HELMAN_KEY:
         pubk->keyType = NSSLOWKEYDHKey;
+        prepare_low_dh_pub_key_for_asn1(pubk);
         rv = SEC_ASN1DecodeItem(arena, pubk,
 				 nsslowcert_DHPublicKeyTemplate, &os);
         if (rv == SECSuccess) return pubk;

security/nss/lib/softoken/lowkey.c

@@ -83,6 +83,61 @@ const SEC_ASN1Template nsslowkey_DHPrivateKeyTemplate[] = {
     { 0, }
 };
 
+/*
+ * See bugzilla bug 125359
+ * Since NSS (via PKCS#11) wants to handle big integers as unsigned ints,
+ * all of the templates above that en/decode into integers must be converted
+ * from ASN.1's signed integer type.  This is done by marking either the
+ * source or destination (encoding or decoding, respectively) type as
+ * siUnsignedInteger.
+ */
+
+void
+prepare_low_rsa_priv_key_for_asn1(NSSLOWKEYPrivateKey *key)
+{
+    key->u.rsa.modulus.type = siUnsignedInteger;
+    key->u.rsa.publicExponent.type = siUnsignedInteger;
+    key->u.rsa.privateExponent.type = siUnsignedInteger;
+    key->u.rsa.prime1.type = siUnsignedInteger;
+    key->u.rsa.prime2.type = siUnsignedInteger;
+    key->u.rsa.exponent1.type = siUnsignedInteger;
+    key->u.rsa.exponent2.type = siUnsignedInteger;
+    key->u.rsa.coefficient.type = siUnsignedInteger;
+}
+
+void
+prepare_low_pqg_params_for_asn1(PQGParams *params)
+{
+    params->prime.type = siUnsignedInteger;
+    params->subPrime.type = siUnsignedInteger;
+    params->base.type = siUnsignedInteger;
+}
+
+void
+prepare_low_dsa_priv_key_for_asn1(NSSLOWKEYPrivateKey *key)
+{
+    key->u.dsa.publicValue.type = siUnsignedInteger;
+    key->u.dsa.privateValue.type = siUnsignedInteger;
+    key->u.dsa.params.prime.type = siUnsignedInteger;
+    key->u.dsa.params.subPrime.type = siUnsignedInteger;
+    key->u.dsa.params.base.type = siUnsignedInteger;
+}
+
+void
+prepare_low_dsa_priv_key_export_for_asn1(NSSLOWKEYPrivateKey *key)
+{
+    key->u.dsa.privateValue.type = siUnsignedInteger;
+}
+
+void
+prepare_low_dh_priv_key_for_asn1(NSSLOWKEYPrivateKey *key)
+{
+    key->u.dh.prime.type = siUnsignedInteger;
+    key->u.dh.base.type = siUnsignedInteger;
+    key->u.dh.publicValue.type = siUnsignedInteger;
+    key->u.dh.privateValue.type = siUnsignedInteger;
+}
+
 void
 nsslowkey_DestroyPrivateKey(NSSLOWKEYPrivateKey *privk)
 {

security/nss/lib/softoken/lowkeyi.h

@@ -32,7 +32,7 @@
  *
  * key.h - public data structures and prototypes for the private key library
  *
- * $Id: lowkeyi.h,v 1.4 2001-12-07 01:36:18 relyea%netscape.com Exp $
+ * $Id: lowkeyi.h,v 1.5 2002-02-21 22:41:38 ian.mcgreer%sun.com Exp $
  */
 
 #ifndef _LOWKEYI_H_
@@ -46,6 +46,20 @@
 
 SEC_BEGIN_PROTOS
 
+/*
+ * See bugzilla bug 125359
+ * Since NSS (via PKCS#11) wants to handle big integers as unsigned ints,
+ * all of the templates above that en/decode into integers must be converted
+ * from ASN.1's signed integer type.  This is done by marking either the
+ * source or destination (encoding or decoding, respectively) type as
+ * siUnsignedInteger.
+ */
+extern void prepare_low_rsa_priv_key_for_asn1(NSSLOWKEYPrivateKey *key);
+extern void prepare_low_pqg_params_for_asn1(PQGParams *params);
+extern void prepare_low_dsa_priv_key_for_asn1(NSSLOWKEYPrivateKey *key);
+extern void prepare_low_dsa_priv_key_export_for_asn1(NSSLOWKEYPrivateKey *key);
+extern void prepare_low_dh_priv_key_for_asn1(NSSLOWKEYPrivateKey *key);
+
 typedef char * (* NSSLOWKEYDBNameFunc)(void *arg, int dbVersion);
     
 /*

security/nss/lib/softoken/pkcs11c.c

@@ -3327,13 +3327,16 @@ static SECItem *pk11_PackagePrivateKey(PK11Object *key)
     param = NULL;
     switch(lk->keyType) {
 	case NSSLOWKEYRSAKey:
+	    prepare_low_rsa_priv_key_for_asn1(lk);
 	    dummy = SEC_ASN1EncodeItem(arena, &pki->privateKey, lk,
 				       nsslowkey_RSAPrivateKeyTemplate);
 	    algorithm = SEC_OID_PKCS1_RSA_ENCRYPTION;
 	    break;
 	case NSSLOWKEYDSAKey:
+            prepare_low_dsa_priv_key_export_for_asn1(lk);
 	    dummy = SEC_ASN1EncodeItem(arena, &pki->privateKey, lk,
 				       nsslowkey_DSAPrivateKeyExportTemplate);
+	    prepare_low_pqg_params_for_asn1(&lk->u.dsa.params);
 	    param = SEC_ASN1EncodeItem(NULL, NULL, &(lk->u.dsa.params),
 				       nsslowkey_PQGParamsTemplate);
 	    algorithm = SEC_OID_ANSIX9_DSA_SIGNATURE;
@@ -3530,12 +3533,15 @@ pk11_unwrapPrivateKey(PK11Object *key, SECItem *bpki)
 	    paramTemplate = NULL;
 	    paramDest = NULL;
 	    lpk->keyType = NSSLOWKEYRSAKey;
+	    prepare_low_rsa_priv_key_for_asn1(lpk);
 	    break;
 	case SEC_OID_ANSIX9_DSA_SIGNATURE:
 	    keyTemplate = nsslowkey_DSAPrivateKeyExportTemplate;
 	    paramTemplate = nsslowkey_PQGParamsTemplate;
 	    paramDest = &(lpk->u.dsa.params);
 	    lpk->keyType = NSSLOWKEYDSAKey;
+	    prepare_low_dsa_priv_key_export_for_asn1(lpk);
+	    prepare_low_pqg_params_for_asn1(&lpk->u.dsa.params);
 	    break;
 	/* case NSSLOWKEYDHKey: */
 	default:

security/nss/lib/util/secasn1d.c

@@ -35,7 +35,7 @@
  * Support for DEcoding ASN.1 data based on BER/DER (Basic/Distinguished
  * Encoding Rules).
  *
- * $Id: secasn1d.c,v 1.14 2002-01-14 23:20:42 ian.mcgreer%sun.com Exp $
+ * $Id: secasn1d.c,v 1.15 2002-02-21 22:41:42 ian.mcgreer%sun.com Exp $
  */
 
 #include "secasn1.h"
@@ -1338,9 +1338,10 @@ sec_asn1d_parse_leaf (sec_asn1d_state *state,
 
     item = (SECItem *)(state->dest);
     if (item != NULL && item->data != NULL) {
-	/* Strip leading zeroes */
+	/* Strip leading zeroes when target is unsigned integer */
 	if (state->underlying_kind == SEC_ASN1_INTEGER && /* INTEGER   */
-	    item->len == 0)                               /* MSB       */
+	    item->len == 0 &&                             /* MSB       */
+	    item->type == siUnsignedInteger)              /* unsigned  */
 	{
 	    while (len > 1 && buf[0] == 0) {              /* leading 0 */
 		buf++;

security/nss/lib/util/secasn1e.c

@@ -35,7 +35,7 @@
  * Support for ENcoding ASN.1 data based on BER/DER (Basic/Distinguished
  * Encoding Rules).
  *
- * $Id: secasn1e.c,v 1.6 2002-01-22 22:48:26 ian.mcgreer%sun.com Exp $
+ * $Id: secasn1e.c,v 1.7 2002-02-21 22:41:42 ian.mcgreer%sun.com Exp $
  */
 
 #include "secasn1.h"
@@ -685,17 +685,18 @@ sec_asn1e_contents_length (const SEC_ASN1Template *theTemplate, void *src,
 	break;
 
       case SEC_ASN1_INTEGER:
-	/* ASN.1 INTEGERs are signed.  PKCS#11 BigIntegers are unsigned.  NSS
+	/* ASN.1 INTEGERs are signed.
-	 * will treat numbers going in and out of the ASN.1 encoder as
+	 * If the source is an unsigned integer, the encoder will need 
-	 * unsigned, so the encoder must handle the conversion.
+	 * to handle the conversion here.
 	 */
 	{
 	    unsigned char *buf = ((SECItem *)src)->data;
+	    SECItemType integerType = ((SECItem *)src)->type;
 	    len = ((SECItem *)src)->len;
 	    while (len > 0) {
 		if (*buf != 0) {
-		    if (*buf & 0x80) {
+		    if (*buf & 0x80 && integerType == siUnsignedInteger) {
-			len++; /* leading zero needed */
+			len++; /* leading zero needed to make number signed */
 		    }
 		    break; /* reached beginning of number */
 		}
@@ -1007,17 +1008,18 @@ sec_asn1e_write_contents (sec_asn1e_state *state,
 	    goto process_string;
 
 	  case SEC_ASN1_INTEGER:
-	   /* ASN.1 INTEGERs are signed.  PKCS#11 BigIntegers are unsigned.  
+	   /* ASN.1 INTEGERs are signed.  If the source is an unsigned
-	    * NSS will treat numbers going in and out of the ASN.1 encoder as
+	    * integer, the encoder will need to handle the conversion here.
-	    * unsigned, so the encoder must handle the conversion.
 	    */
 	    {
 		unsigned int blen;
 		unsigned char *buf;
+		SECItemType integerType;
 		blen = ((SECItem *)state->src)->len;
 		buf = ((SECItem *)state->src)->data;
+		integerType = ((SECItem *)state->src)->type;
 		while (blen > 0) {
-		    if (*buf & 0x80) {
+		    if (*buf & 0x80 && integerType == siUnsignedInteger) {
 			char zero = 0; /* write a leading 0 */
 			sec_asn1e_write_contents_bytes(state, &zero, 1);
 			/* and then the remaining buffer */
@@ -1025,10 +1027,16 @@ sec_asn1e_write_contents (sec_asn1e_state *state,
 			                               (char *)buf, blen); 
 			break;
 		    } 
-		    if (*buf != 0 || blen == 1) {
+		    /* Check three possibilities:
-			/* no leading zeros, msb of MSB is not 1, so write
+		     * 1.  No leading zeros, msb of MSB is not 1;
-			 * the remaining buffer (0 itself also goes here)
+		     * 2.  The number is zero itself;
-			 */
+		     * 3.  Encoding a signed integer with a leading zero,
+		     *     keep the zero so that the number is positive.
+		     */
+		    if (*buf != 0 || 
+		         blen == 1 || 
+		         (buf[1] & 0x80 && integerType != siUnsignedInteger) ) 
+		    {
 			sec_asn1e_write_contents_bytes(state, 
 			                               (char *)buf, blen); 
 			break;

security/nss/lib/util/seccomon.h

@@ -38,7 +38,7 @@
  * for security libraries.  It should not be dependent on any other
  * headers, and should not require linking with any libraries.
  *
- * $Id: seccomon.h,v 1.2 2001-01-18 16:36:43 wtc%netscape.com Exp $
+ * $Id: seccomon.h,v 1.3 2002-02-21 22:41:44 ian.mcgreer%sun.com Exp $
  */
 
 #ifndef _SECCOMMON_H_
@@ -67,7 +67,8 @@ typedef enum {
     siEncodedNameBuffer = 6,
     siAsciiNameString = 7,
     siAsciiString = 8,
-    siDEROID = 9
+    siDEROID = 9,
+    siUnsignedInteger = 10
 } SECItemType;
 
 typedef struct SECItemStr SECItem;

security/nss/lib/util/secitem.c

@@ -34,7 +34,7 @@
 /*
  * Support routines for SECItem data structure.
  *
- * $Id: secitem.c,v 1.5 2001-12-07 01:36:25 relyea%netscape.com Exp $
+ * $Id: secitem.c,v 1.6 2002-02-21 22:41:44 ian.mcgreer%sun.com Exp $
  */
 
 #include "seccomon.h"
@@ -196,6 +196,7 @@ SECITEM_DupItem(const SECItem *from)
 SECStatus
 SECITEM_CopyItem(PRArenaPool *arena, SECItem *to, const SECItem *from)
 {
+    to->type = from->type;
     if (from->data && from->len) {
 	if ( arena ) {
 	    to->data = (unsigned char*) PORT_ArenaAlloc(arena, from->len);