r=mccabe

bug #22866
Support unicode line/paragraph separators.

js/src/jsscan.c

@@ -61,6 +61,10 @@
 #include "jsregexp.h"
 #include "jsscan.h"
 
+/* Unicode separators that are treated as line terminators, in addition to \n, \r */
+#define LINE_SEPARATOR  (0x2028)
+#define PARA_SEPARATOR  (0x2029)
+
 #define RESERVE_JAVA_KEYWORDS
 #define RESERVE_ECMA_KEYWORDS
 
@@ -318,8 +322,14 @@ GetChar(JSTokenStream *ts)
                                     &ts->listenerTSData, ts->listenerData);
 	        /*
 	         * Any one of \n, \r, or \r\n ends a line (longest match wins).
+                 * Also allow the Unicode line and paragraph separators.
 	         */
 	        for (nl = ts->userbuf.ptr; nl < ts->userbuf.limit; nl++) {
+                    /*
+                    * Try to prevent value-testing on most characters by
+                    * filtering out characters that aren't 000x or 202x.
+                    */
+                    if ((*nl & 0xDFD0) == 0) {
 		        if (*nl == '\n')
 		            break;
 		        if (*nl == '\r') {
@@ -327,6 +337,9 @@ GetChar(JSTokenStream *ts)
 			        nl++;
 		            break;
 		        }
+                        if ((*nl == LINE_SEPARATOR) || (*nl == PARA_SEPARATOR))
+		            break;
+                    }
                 }
 
 	        /*
@@ -376,6 +389,9 @@ GetChar(JSTokenStream *ts)
 			    JS_ASSERT(ts->linebuf.base[len] == '\n');
 			    ts->linebuf.base[len-1] = '\n';
 		        }
+		    } else if ((*nl == LINE_SEPARATOR) ||
+                               (*nl == PARA_SEPARATOR)) {
+                        ts->linebuf.base[len-1] = '\n';
 		    }
 	        }
 
@@ -716,9 +732,9 @@ retry:
 	RETURN(TOK_EOF);
 
     hadUnicodeEscape = JS_FALSE;
-    if (JS_ISIDENT_START(c) 
-                || ((c == '\\') 
-                        && (c = getUnicodeEscape(ts), 
+    if (JS_ISIDENT_START(c) ||
+        ((c == '\\') &&
+            (c = getUnicodeEscape(ts), 
             hadUnicodeEscape = JS_ISIDENT_START(c)))) {
 	INIT_TOKENBUF(&ts->tokenbuf);
         for (;;) {