From 309f4107d55c3e9ebb48842328750f379a07f7da Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Mon, 29 Mar 2021 21:09:03 -0700 Subject: [PATCH] Revert "Add 'Z' to the string format in Timestamp.ToString() to indicate UTC time. (#871)" (#873) This reverts commit 53717b43f577d87d5060f1b03df7fbef507e0a75. --- azure-pipelines.yml | 29 +++++++------------ .../UdfTests/UdfSimpleTypesTests.cs | 8 ++--- .../Sql/TimestampTests.cs | 13 ++------- .../Microsoft.Spark/Sql/Types/Timestamp.cs | 2 +- 4 files changed, 17 insertions(+), 35 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 72ea3377..fd209f90 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -16,27 +16,20 @@ variables: backwardCompatibleRelease: '1.0.0' forwardCompatibleRelease: '1.0.0' - # Filter UdfSimpleTypesTests.TestUdfWithReturnAsTimestampType and UdfSimpleTypesTests.TestUdfWithTimestampType - # backward and forward compatibility tests due to bug with Timestamp.ToString(). This is not a breaking change. - # Please see https://github.com/dotnet/spark/pull/871 - backwardCompatibleTestOptions_Windows_2_3: "--filter \ - (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithReturnAsTimestampType)&\ - (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithTimestampType)" - forwardCompatibleTestOptions_Windows_2_3: $(backwardCompatibleTestOptions_Windows_2_3) - backwardCompatibleTestOptions_Linux_2_3: $(backwardCompatibleTestOptions_Windows_2_3) - forwardCompatibleTestOptions_Linux_2_3: $(backwardCompatibleTestOptions_Windows_2_3) + backwardCompatibleTestOptions_Windows_2_3: "" + forwardCompatibleTestOptions_Windows_2_3: "" + backwardCompatibleTestOptions_Linux_2_3: "" + forwardCompatibleTestOptions_Linux_2_3: "" - backwardCompatibleTestOptions_Windows_2_4: $(backwardCompatibleTestOptions_Windows_2_3) - forwardCompatibleTestOptions_Windows_2_4: $(backwardCompatibleTestOptions_Windows_2_3) - backwardCompatibleTestOptions_Linux_2_4: $(backwardCompatibleTestOptions_Windows_2_3) + backwardCompatibleTestOptions_Windows_2_4: "" + forwardCompatibleTestOptions_Windows_2_4: "" + backwardCompatibleTestOptions_Linux_2_4: "" # Filter HyperspaceTests not due to functionality changes, but to incompatible tests running on Linux. # Please see https://github.com/dotnet/spark/pull/737 for the fix. forwardCompatibleTestOptions_Linux_2_4: "--filter \ (FullyQualifiedName!=Microsoft.Spark.Extensions.Hyperspace.E2ETest.HyperspaceTests.TestExplainAPI)&\ (FullyQualifiedName!=Microsoft.Spark.Extensions.Hyperspace.E2ETest.HyperspaceTests.TestIndexCreateAndDelete)&\ - (FullyQualifiedName!=Microsoft.Spark.Extensions.Hyperspace.E2ETest.HyperspaceTests.TestSignatures&\ - (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithReturnAsTimestampType)&\ - (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithTimestampType)" + (FullyQualifiedName!=Microsoft.Spark.Extensions.Hyperspace.E2ETest.HyperspaceTests.TestSignatures)" # Filter DataFrameTests.TestDataFrameGroupedMapUdf and DataFrameTests.TestGroupedMapUdf backwardCompatible # tests due to https://github.com/dotnet/spark/pull/711 @@ -45,10 +38,8 @@ variables: backwardCompatibleTestOptions_Windows_3_0: "--filter \ (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestDataFrameGroupedMapUdf)&\ (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestGroupedMapUdf)&\ - (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithDuplicateTimestamps)&\ - (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithReturnAsTimestampType)&\ - (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithTimestampType)" - forwardCompatibleTestOptions_Windows_3_0: $(backwardCompatibleTestOptions_Windows_2_3) + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithDuplicateTimestamps)" + forwardCompatibleTestOptions_Windows_3_0: "" backwardCompatibleTestOptions_Linux_3_0: $(backwardCompatibleTestOptions_Windows_3_0) forwardCompatibleTestOptions_Linux_3_0: $(forwardCompatibleTestOptions_Linux_2_4) diff --git a/src/csharp/Microsoft.Spark.E2ETest/UdfTests/UdfSimpleTypesTests.cs b/src/csharp/Microsoft.Spark.E2ETest/UdfTests/UdfSimpleTypesTests.cs index c7388143..79f7b8aa 100644 --- a/src/csharp/Microsoft.Spark.E2ETest/UdfTests/UdfSimpleTypesTests.cs +++ b/src/csharp/Microsoft.Spark.E2ETest/UdfTests/UdfSimpleTypesTests.cs @@ -118,8 +118,8 @@ namespace Microsoft.Spark.E2ETest.UdfTests var expected = new string[] { - "2020-01-01 00:00:00.000000Z", - "2020-01-02 15:30:30.123456Z" + "2020-01-01 00:00:00.000000", + "2020-01-02 15:30:30.123456" }; string[] rowsToArray = rows.Select(x => x[0].ToString()).ToArray(); Assert.Equal(expected, rowsToArray); @@ -193,8 +193,8 @@ namespace Microsoft.Spark.E2ETest.UdfTests var expected = new string[] { - "2020-01-04 15:30:30.123456Z", - "2050-01-04 15:30:30.123456Z" + "2020-01-04 15:30:30.123456", + "2050-01-04 15:30:30.123456" }; for (int i = 0; i < rows.Length; ++i) { diff --git a/src/csharp/Microsoft.Spark.UnitTest/Sql/TimestampTests.cs b/src/csharp/Microsoft.Spark.UnitTest/Sql/TimestampTests.cs index 14a76673..dfbef74d 100644 --- a/src/csharp/Microsoft.Spark.UnitTest/Sql/TimestampTests.cs +++ b/src/csharp/Microsoft.Spark.UnitTest/Sql/TimestampTests.cs @@ -29,7 +29,7 @@ namespace Microsoft.Spark.UnitTest Assert.Equal(DateTimeKind.Utc, timestamp.ToDateTime().Kind); // Validate ToString(). - Assert.Equal("2020-01-01 08:30:30.000123Z", timestamp.ToString()); + Assert.Equal("2020-01-01 08:30:30.000123", timestamp.ToString()); // Validate ToDateTime(). Assert.Equal(testDate, timestamp.ToDateTime()); @@ -57,7 +57,7 @@ namespace Microsoft.Spark.UnitTest Assert.Equal(DateTimeKind.Utc, timestamp.ToDateTime().Kind); // Validate ToString(). - Assert.Equal("2020-01-02 15:30:30.123456Z", timestamp.ToString()); + Assert.Equal("2020-01-02 15:30:30.123456", timestamp.ToString()); // Validate ToDateTime(). Assert.Equal( @@ -71,14 +71,5 @@ namespace Microsoft.Spark.UnitTest () => new Timestamp(2020, 1, 2, 15, 30, 30, 1234567)); } } - - [Fact] - public void TestTimestampToString() - { - var dateTimeObj = new DateTime(2021, 01, 01); - Assert.Equal( - new Timestamp(DateTime.Parse(new Timestamp(dateTimeObj).ToString())), - new Timestamp(dateTimeObj)); - } } } diff --git a/src/csharp/Microsoft.Spark/Sql/Types/Timestamp.cs b/src/csharp/Microsoft.Spark/Sql/Types/Timestamp.cs index 992944c3..d07d0517 100644 --- a/src/csharp/Microsoft.Spark/Sql/Types/Timestamp.cs +++ b/src/csharp/Microsoft.Spark/Sql/Types/Timestamp.cs @@ -94,7 +94,7 @@ namespace Microsoft.Spark.Sql.Types /// /// Readable string representation for this type. /// - public override string ToString() => _dateTime.ToString("yyyy-MM-dd HH:mm:ss.ffffffZ"); + public override string ToString() => _dateTime.ToString("yyyy-MM-dd HH:mm:ss.ffffff"); /// /// Checks if the given object is same as the current object.