From 2248e04ae4d4feac9c36e38475eddb916721961c Mon Sep 17 00:00:00 2001
From: Jason Davies <jason@jasondavies.com>
Date: Sat, 12 Mar 2011 20:43:15 +0000
Subject: [PATCH 1/2] Fix CSV parsing of \r\n and \r

---
 Makefile                 |  1 +
 d3.csv.js                |  4 ++--
 d3.csv.min.js            |  2 +-
 src/csv/parse.js         |  4 ++--
 tests/test-csv-parse.js  | 18 ++++++++++++++++++
 tests/test-csv-parse.out |  9 +++++++++
 6 files changed, 33 insertions(+), 5 deletions(-)
 create mode 100644 tests/test-csv-parse.js
 create mode 100644 tests/test-csv-parse.out

diff --git a/Makefile b/Makefile
index 9ce15c7d..58d8a2e8 100644
--- a/Makefile
+++ b/Makefile
@@ -137,6 +137,7 @@ tests: \
 	tests/test-append.test \
 	tests/test-attr.test \
 	tests/test-call.test \
+	tests/test-csv-parse.test \
 	tests/test-format.test \
 	tests/test-transition.test \
 	tests/test-scale-linear.test \
diff --git a/d3.csv.js b/d3.csv.js
index 7b4015d2..677d20a3 100644
--- a/d3.csv.js
+++ b/d3.csv.js
@@ -21,7 +21,7 @@ d3.csv.parseRows = function(text, f) {
   var EOL = {}, // sentinel value for end-of-line
       EOF = {}, // sentinel value for end-of-file
       rows = [], // output rows
-      re = /[,\n]/g, // field separator regex
+      re = /\r\n|[,\r\n]/g, // field separator regex
       n = 0, // the current line number
       t, // the current token
       eol; // is the current token followed by EOL?
@@ -49,7 +49,7 @@ d3.csv.parseRows = function(text, f) {
     // common case
     var m = re.exec(text);
     if (m) {
-      if (m[0] == "\n") eol = true;
+      if (m[0] == "\n" || m[0] == "\r" || m[0] == "\r\n") eol = true;
       return text.substring(j, m.index);
     }
     re.lastIndex = text.length;
diff --git a/d3.csv.min.js b/d3.csv.min.js
index 1f027a74..da803c88 100644
--- a/d3.csv.min.js
+++ b/d3.csv.min.js
@@ -1,2 +1,2 @@
 (function(){function m(a){return a.map(n).join(",")}function n(a){return/[",\n]/.test(a)?'"'+a.replace(/\"/g,'""')+'"':a}d3.csv=function(a,c){d3.text(a,"text/csv",function(d){c(d&&d3.csv.parse(d))})};d3.csv.parse=function(a){var c;return d3.csv.parseRows(a,function(d,j){if(j){for(var f={},g=-1,e=c.length;++g<e;)f[c[g]]=d[g];return f}else{c=d;return null}})};d3.csv.parseRows=function(a,c){function d(){if(e.lastIndex==a.length)return f;if(k){k=false;return j}var h=e.lastIndex;if(a.charCodeAt(h)==34){for(var b=
-h;b++<a.length;)if(a.charCodeAt(b)==34){if(a.charCodeAt(b+1)!=34)break;b++}if(a.charCodeAt(b+1)==10)k=true;e.lastIndex=b+2;return a.substring(h+1,b).replace(/""/g,'"')}if(b=e.exec(a)){if(b[0]=="\n")k=true;return a.substring(h,b.index)}e.lastIndex=a.length;return a.substring(h)}for(var j={},f={},g=[],e=/[,\n]/g,o=0,i,k;(i=d())!==f;){for(var l=[];i!==j&&i!==f;){l.push(i);i=d()}c&&!(l=c(l,o++))||g.push(l)}return g};d3.csv.format=function(a){return a.map(m).join("\n")}})();
+h;b++<a.length;)if(a.charCodeAt(b)==34){if(a.charCodeAt(b+1)!=34)break;b++}if(a.charCodeAt(b+1)==10)k=true;e.lastIndex=b+2;return a.substring(h+1,b).replace(/""/g,'"')}if(b=e.exec(a)){if(b[0]=="\n"||b[0]=="\r"||b[0]=="\r\n")k=true;return a.substring(h,b.index)}e.lastIndex=a.length;return a.substring(h)}for(var j={},f={},g=[],e=/\r\n|[,\r\n]/g,o=0,i,k;(i=d())!==f;){for(var l=[];i!==j&&i!==f;){l.push(i);i=d()}c&&!(l=c(l,o++))||g.push(l)}return g};d3.csv.format=function(a){return a.map(m).join("\n")}})();
diff --git a/src/csv/parse.js b/src/csv/parse.js
index f959751e..315af58f 100644
--- a/src/csv/parse.js
+++ b/src/csv/parse.js
@@ -16,7 +16,7 @@ d3.csv.parseRows = function(text, f) {
   var EOL = {}, // sentinel value for end-of-line
       EOF = {}, // sentinel value for end-of-file
       rows = [], // output rows
-      re = /[,\n]/g, // field separator regex
+      re = /\r\n|[,\r\n]/g, // field separator regex
       n = 0, // the current line number
       t, // the current token
       eol; // is the current token followed by EOL?
@@ -44,7 +44,7 @@ d3.csv.parseRows = function(text, f) {
     // common case
     var m = re.exec(text);
     if (m) {
-      if (m[0] == "\n") eol = true;
+      if (m[0] == "\n" || m[0] == "\r" || m[0] == "\r\n") eol = true;
       return text.substring(j, m.index);
     }
     re.lastIndex = text.length;
diff --git a/tests/test-csv-parse.js b/tests/test-csv-parse.js
new file mode 100644
index 00000000..dbe12d40
--- /dev/null
+++ b/tests/test-csv-parse.js
@@ -0,0 +1,18 @@
+require("./../lib/env-js/envjs/node");
+require("./../d3");
+require("./../d3.csv");
+
+var newlines = {
+  UNIX: "\n",
+  DOS: "\r\n",
+  Mac: "\r"
+};
+
+for (os in newlines) {
+  console.log("parse " + os + " newlines:");
+  var rows = d3.csv.parse(["a,b,c", "1,2,3", "4,5,6"].join(newlines[os]));
+  console.log("  " + rows.map(function(row) {
+    return row.a + ":" + row.b + ":" + row.c;
+  }).join("|"));
+  console.log("");
+}
diff --git a/tests/test-csv-parse.out b/tests/test-csv-parse.out
new file mode 100644
index 00000000..fabe9033
--- /dev/null
+++ b/tests/test-csv-parse.out
@@ -0,0 +1,9 @@
+parse UNIX newlines:
+  1:2:3|4:5:6
+
+parse DOS newlines:
+  1:2:3|4:5:6
+
+parse Mac newlines:
+  1:2:3|4:5:6
+

From e6d21d4d1222a33a3bacef81f3a7ad4850d8605f Mon Sep 17 00:00:00 2001
From: Jason Davies <jason@jasondavies.com>
Date: Sat, 12 Mar 2011 21:25:39 +0000
Subject: [PATCH 2/2] Fix newline parsing for quoted values.

---
 d3.csv.js                | 8 +++++++-
 d3.csv.min.js            | 4 ++--
 src/csv/parse.js         | 8 +++++++-
 tests/test-csv-parse.js  | 2 +-
 tests/test-csv-parse.out | 6 +++---
 5 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/d3.csv.js b/d3.csv.js
index 677d20a3..df9ddb7c 100644
--- a/d3.csv.js
+++ b/d3.csv.js
@@ -41,8 +41,14 @@ d3.csv.parseRows = function(text, f) {
           i++;
         }
       }
-      if (text.charCodeAt(i + 1) == 10) eol = true;
       re.lastIndex = i + 2;
+      var c = text.charAt(i + 1);
+      if (c === '\r') {
+        eol = true;
+        if (text.charAt(i + 2) === '\n') re.lastIndex++;
+      } else if (c === '\n') {
+        eol = true;
+      }
       return text.substring(j + 1, i).replace(/""/g, "\"");
     }
 
diff --git a/d3.csv.min.js b/d3.csv.min.js
index da803c88..c49d7bd7 100644
--- a/d3.csv.min.js
+++ b/d3.csv.min.js
@@ -1,2 +1,2 @@
-(function(){function m(a){return a.map(n).join(",")}function n(a){return/[",\n]/.test(a)?'"'+a.replace(/\"/g,'""')+'"':a}d3.csv=function(a,c){d3.text(a,"text/csv",function(d){c(d&&d3.csv.parse(d))})};d3.csv.parse=function(a){var c;return d3.csv.parseRows(a,function(d,j){if(j){for(var f={},g=-1,e=c.length;++g<e;)f[c[g]]=d[g];return f}else{c=d;return null}})};d3.csv.parseRows=function(a,c){function d(){if(e.lastIndex==a.length)return f;if(k){k=false;return j}var h=e.lastIndex;if(a.charCodeAt(h)==34){for(var b=
-h;b++<a.length;)if(a.charCodeAt(b)==34){if(a.charCodeAt(b+1)!=34)break;b++}if(a.charCodeAt(b+1)==10)k=true;e.lastIndex=b+2;return a.substring(h+1,b).replace(/""/g,'"')}if(b=e.exec(a)){if(b[0]=="\n"||b[0]=="\r"||b[0]=="\r\n")k=true;return a.substring(h,b.index)}e.lastIndex=a.length;return a.substring(h)}for(var j={},f={},g=[],e=/\r\n|[,\r\n]/g,o=0,i,k;(i=d())!==f;){for(var l=[];i!==j&&i!==f;){l.push(i);i=d()}c&&!(l=c(l,o++))||g.push(l)}return g};d3.csv.format=function(a){return a.map(m).join("\n")}})();
+(function(){function n(a){return a.map(o).join(",")}function o(a){return/[",\n]/.test(a)?'"'+a.replace(/\"/g,'""')+'"':a}d3.csv=function(a,c){d3.text(a,"text/csv",function(d){c(d&&d3.csv.parse(d))})};d3.csv.parse=function(a){var c;return d3.csv.parseRows(a,function(d,j){if(j){for(var f={},g=-1,e=c.length;++g<e;)f[c[g]]=d[g];return f}else{c=d;return null}})};d3.csv.parseRows=function(a,c){function d(){if(e.lastIndex==a.length)return f;if(k){k=false;return j}var h=e.lastIndex;if(a.charCodeAt(h)==34){for(var b=
+h;b++<a.length;)if(a.charCodeAt(b)==34){if(a.charCodeAt(b+1)!=34)break;b++}var m=a.charCodeAt(b+1);if(m==="\r"||m==="\n")k=true;e.lastIndex=b+2;return a.substring(h+1,b).replace(/""/g,'"')}if(b=e.exec(a)){if(b[0]=="\n"||b[0]=="\r"||b[0]=="\r\n")k=true;return a.substring(h,b.index)}e.lastIndex=a.length;return a.substring(h)}for(var j={},f={},g=[],e=/\r\n|[,\r\n]/g,p=0,i,k;(i=d())!==f;){for(var l=[];i!==j&&i!==f;){l.push(i);i=d()}c&&!(l=c(l,p++))||g.push(l)}return g};d3.csv.format=function(a){return a.map(n).join("\n")}})();
diff --git a/src/csv/parse.js b/src/csv/parse.js
index 315af58f..05fafb38 100644
--- a/src/csv/parse.js
+++ b/src/csv/parse.js
@@ -36,8 +36,14 @@ d3.csv.parseRows = function(text, f) {
           i++;
         }
       }
-      if (text.charCodeAt(i + 1) == 10) eol = true;
       re.lastIndex = i + 2;
+      var c = text.charAt(i + 1);
+      if (c === '\r') {
+        eol = true;
+        if (text.charAt(i + 2) === '\n') re.lastIndex++;
+      } else if (c === '\n') {
+        eol = true;
+      }
       return text.substring(j + 1, i).replace(/""/g, "\"");
     }
 
diff --git a/tests/test-csv-parse.js b/tests/test-csv-parse.js
index dbe12d40..53ac7ae3 100644
--- a/tests/test-csv-parse.js
+++ b/tests/test-csv-parse.js
@@ -10,7 +10,7 @@ var newlines = {
 
 for (os in newlines) {
   console.log("parse " + os + " newlines:");
-  var rows = d3.csv.parse(["a,b,c", "1,2,3", "4,5,6"].join(newlines[os]));
+  var rows = d3.csv.parse(["a,b,c", "1,2,3", "4,5,\"6\"", "7,8,9"].join(newlines[os]));
   console.log("  " + rows.map(function(row) {
     return row.a + ":" + row.b + ":" + row.c;
   }).join("|"));
diff --git a/tests/test-csv-parse.out b/tests/test-csv-parse.out
index fabe9033..632a9307 100644
--- a/tests/test-csv-parse.out
+++ b/tests/test-csv-parse.out
@@ -1,9 +1,9 @@
 parse UNIX newlines:
-  1:2:3|4:5:6
+  1:2:3|4:5:6|7:8:9
 
 parse DOS newlines:
-  1:2:3|4:5:6
+  1:2:3|4:5:6|7:8:9
 
 parse Mac newlines:
-  1:2:3|4:5:6
+  1:2:3|4:5:6|7:8:9