Fix name constraints code to pass NIST PKITS test 38. r=wtc. Bug 231223.

security/nss/lib/certdb/genname.c

@@ -1135,17 +1135,29 @@ compareURIN2C(const SECItem *name, const SECItem *constraint)
     return SECFailure;
 }
 
-/* for DNSnames, the constraint matches any string to which it matches the 
-** rightmost characters in that string.
-**  Constraint            Name             Result
-** ------------      ---------------      --------
-**  foo.bar.com          foo.bar.com      matches
-**  foo.bar.com          FoO.bAr.CoM      matches
-**  foo.bar.com      www.foo.bar.com      matches
-**  foo.bar.com        nofoo.bar.com      MATCHES
-** .foo.bar.com      www.foo.bar.com      matches
-** .foo.bar.com          foo.bar.com      no match
-** .foo.bar.com     www..foo.bar.com      matches
+/* for DNSname constraints, RFC 3280 says, (section 4.2.1.11, page 38)
+**
+** DNS name restrictions are expressed as foo.bar.com.  Any DNS name
+** that can be constructed by simply adding to the left hand side of the
+** name satisfies the name constraint.  For example, www.foo.bar.com
+** would satisfy the constraint but foo1.bar.com would not.
+**
+** But NIST's PKITS test suite requires that the constraint be treated
+** as a domain name, and requires that any name added to the left hand
+** side end in a dot ".".  Sensible, but not strictly following the RFC.
+**
+**  Constraint            Name            RFC 3280  NIST PKITS
+** ------------      ---------------      --------  ----------
+**  foo.bar.com          foo.bar.com      matches    matches
+**  foo.bar.com          FoO.bAr.CoM      matches    matches
+**  foo.bar.com      www.foo.bar.com      matches    matches
+**  foo.bar.com        nofoo.bar.com      MATCHES    NO MATCH
+** .foo.bar.com      www.foo.bar.com      matches    matches? disallowed?
+** .foo.bar.com          foo.bar.com      no match   no match
+** .foo.bar.com     www..foo.bar.com      matches    probably not 
+**
+** We will try to conform to NIST's PKITS tests, and the unstated 
+** rules they imply.
 */
 static SECStatus
 compareDNSN2C(const SECItem *name, const SECItem *constraint)
@@ -1161,7 +1173,10 @@ compareDNSN2C(const SECItem *name, const SECItem *constraint)
     offset = name->len - constraint->len;
     if (PL_strncasecmp(name->data + offset, constraint->data, constraint->len))
         return SECFailure;
+    if (!offset || 
+        (name->data[offset - 1] == '.') + (constraint->data[0] == '.') == 1)
 	return SECSuccess;
+    return SECFailure;
 }
 
 /* Returns SECSuccess if name matches constraint per RFC 3280 rules for