diff --git a/src/Microsoft.Data.Analysis/DataFrame.IO.cs b/src/Microsoft.Data.Analysis/DataFrame.IO.cs index 4f14615b0..74e0f5f2b 100644 --- a/src/Microsoft.Data.Analysis/DataFrame.IO.cs +++ b/src/Microsoft.Data.Analysis/DataFrame.IO.cs @@ -3,7 +3,6 @@ // See the LICENSE file in the project root for more information. using System; -using System.Collections; using System.Collections.Generic; using System.Globalization; using System.IO; @@ -499,11 +498,11 @@ namespace Microsoft.Data.Analysis if (t == typeof(string)) { - bool needsQuotes = ((string)cell).IndexOf(separator) != -1 || ((string)cell).IndexOf('\n') != -1; - if (needsQuotes) + string stringCell = (string)cell; + if (NeedsQuotes(stringCell, separator)) { record.Append('\"'); - record.Append(cell); + record.Append(stringCell.Replace("\"", "\"\"")); // Quotations in CSV data must be escaped with another quotation record.Append('\"'); continue; } @@ -519,6 +518,7 @@ namespace Microsoft.Data.Analysis } } } + private static void WriteHeader(StreamWriter csvFile, IReadOnlyList columnNames, char separator) { bool firstColumn = true; @@ -533,11 +533,10 @@ namespace Microsoft.Data.Analysis firstColumn = false; } - bool needsQuotes = name.IndexOf(separator) != -1 || name.IndexOf('\n') != -1; - if (needsQuotes) + if (NeedsQuotes(name, separator)) { csvFile.Write('\"'); - csvFile.Write(name); + csvFile.Write(name.Replace("\"", "\"\"")); // Quotations in CSV data must be escaped with another quotation csvFile.Write('\"'); } else @@ -545,8 +544,12 @@ namespace Microsoft.Data.Analysis csvFile.Write(name); } } - csvFile.WriteLine(); } + + private static bool NeedsQuotes(string csvCell, char separator) + { + return csvCell.AsSpan().IndexOfAny(separator, '\n', '\"') != -1; + } } } diff --git a/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs b/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs index 371a29749..1b2e34a0f 100644 --- a/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs +++ b/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs @@ -1064,10 +1064,12 @@ CMT,"; { yield return new object[] // Comma Separators in Data { - @"Name,Age,Description -Paul,34,""Paul lives in Vermont, VA."" -Victor,29,""Victor: Funny guy"" -Maria,31,", + """ + Name,Age,Description + Paul,34,"Paul lives in Vermont, VA." + Victor,29,"Victor: Funny guy" + Maria,31, + """, ',', new Type[] { typeof(string), typeof(int), typeof(string) }, new LoadCsvVerifyingHelper( @@ -1085,10 +1087,12 @@ Maria,31,", }; yield return new object[] // Colon Separators in Data { - @"Name:Age:Description -Paul:34:""Paul lives in Vermont, VA."" -Victor:29:""Victor: Funny guy"" -Maria:31:", + """ + Name:Age:Description + Paul:34:"Paul lives in Vermont, VA." + Victor:29:"Victor: Funny guy" + Maria:31: + """, ':', new Type[] { typeof(string), typeof(int), typeof(string) }, new LoadCsvVerifyingHelper( @@ -1106,10 +1110,12 @@ Maria:31:", }; yield return new object[] // Comma Separators in Header { - @"""Na,me"",Age,Description -Paul,34,""Paul lives in Vermont, VA."" -Victor,29,""Victor: Funny guy"" -Maria,31,", + """ + "Na,me",Age,Description + Paul,34,"Paul lives in Vermont, VA." + Victor,29,"Victor: Funny guy" + Maria,31, + """, ',', new Type[] { typeof(string), typeof(int), typeof(string) }, new LoadCsvVerifyingHelper( @@ -1127,11 +1133,13 @@ Maria,31,", }; yield return new object[] // Newlines In Data { - @"Name,Age,Description -Paul,34,""Paul lives in Vermont -VA."" -Victor,29,""Victor: Funny guy"" -Maria,31,", + """ + Name,Age,Description + Paul,34,"Paul lives in Vermont + VA." + Victor,29,"Victor: Funny guy" + Maria,31, + """, ',', new Type[] { typeof(string), typeof(int), typeof(string) }, new LoadCsvVerifyingHelper( @@ -1141,8 +1149,15 @@ Maria,31,", new Type[] { typeof(string), typeof(int), typeof(string) }, new object[][] { - new object[] { "Paul", 34, @"Paul lives in Vermont -VA." }, + new object[] + { + "Paul", + 34, + """ + Paul lives in Vermont + VA. + """ + }, new object[] { "Victor", 29, "Victor: Funny guy" }, new object[] { "Maria", 31, "" } } @@ -1150,18 +1165,73 @@ VA." }, }; yield return new object[] // Newlines In Header { - @"""Na -me"":Age:Description -Paul:34:""Paul lives in Vermont, VA."" -Victor:29:""Victor: Funny guy"" -Maria:31:", + """ + "Na + me":Age:Description + Paul:34:"Paul lives in Vermont, VA." + Victor:29:"Victor: Funny guy" + Maria:31: + """, ':', new Type[] { typeof(string), typeof(int), typeof(string) }, new LoadCsvVerifyingHelper( 3, 3, - new string[] { @"Na -me", "Age", "Description" }, + new string[] + { + """ + Na + me + """, + "Age", + "Description" + }, + new Type[] { typeof(string), typeof(int), typeof(string) }, + new object[][] + { + new object[] { "Paul", 34, "Paul lives in Vermont, VA." }, + new object[] { "Victor", 29, "Victor: Funny guy" }, + new object[] { "Maria", 31, "" } + } + ) + }; + yield return new object[] // Quotations in Data + { + """ + Name,Age,Description + Paul,34,"Paul lives in ""Vermont VA""." + Victor,29,"Victor: Funny guy" + Maria,31, + """, + ',', + new Type[] { typeof(string), typeof(int), typeof(string) }, + new LoadCsvVerifyingHelper( + 3, + 3, + new string[] { "Name", "Age", "Description" }, + new Type[] { typeof(string), typeof(int), typeof(string) }, + new object[][] + { + new object[] { "Paul", 34, """Paul lives in "Vermont VA".""" }, + new object[] { "Victor", 29, "Victor: Funny guy" }, + new object[] { "Maria", 31, "" } + } + ) + }; + yield return new object[] // Quotations in Header + { + """ + Name,Age,"De""script""ion" + Paul,34,"Paul lives in Vermont, VA." + Victor,29,"Victor: Funny guy" + Maria,31, + """, + ',', + new Type[] { typeof(string), typeof(int), typeof(string) }, + new LoadCsvVerifyingHelper( + 3, + 3, + new string[] { "Name", "Age", """De"script"ion""" }, new Type[] { typeof(string), typeof(int), typeof(string) }, new object[][] { diff --git a/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj b/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj index 803281292..07c3ea2c3 100644 --- a/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj +++ b/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj @@ -1,6 +1,7 @@  $(NoWarn);MSML_ParameterLocalVarName;MSML_PrivateFieldName;MSML_ExtendBaseTestClass;MSML_GeneralName + preview