Revert "Revert "Add 'Z' to the string format in Timestamp.ToString() to indicate UTC time. (#871)" (#873)" (#897)

This reverts commit 309f4107d5.

azure-pipelines.yml

@@ -16,20 +16,27 @@ variables:
   backwardCompatibleRelease: '1.0.0'
   forwardCompatibleRelease: '1.0.0'
 
-  backwardCompatibleTestOptions_Windows_2_3: ""
-  forwardCompatibleTestOptions_Windows_2_3: ""
-  backwardCompatibleTestOptions_Linux_2_3: ""
-  forwardCompatibleTestOptions_Linux_2_3: ""
+  # Filter UdfSimpleTypesTests.TestUdfWithReturnAsTimestampType and UdfSimpleTypesTests.TestUdfWithTimestampType
+  # backward and forward compatibility tests due to bug with Timestamp.ToString(). This is not a breaking change.
+  # Please see https://github.com/dotnet/spark/pull/871
+  backwardCompatibleTestOptions_Windows_2_3: "--filter \
+  (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithReturnAsTimestampType)&\
+  (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithTimestampType)"
+  forwardCompatibleTestOptions_Windows_2_3: $(backwardCompatibleTestOptions_Windows_2_3)
+  backwardCompatibleTestOptions_Linux_2_3: $(backwardCompatibleTestOptions_Windows_2_3)
+  forwardCompatibleTestOptions_Linux_2_3: $(backwardCompatibleTestOptions_Windows_2_3)
 
-  backwardCompatibleTestOptions_Windows_2_4: ""
-  forwardCompatibleTestOptions_Windows_2_4: ""
-  backwardCompatibleTestOptions_Linux_2_4: ""
+  backwardCompatibleTestOptions_Windows_2_4: $(backwardCompatibleTestOptions_Windows_2_3)
+  forwardCompatibleTestOptions_Windows_2_4: $(backwardCompatibleTestOptions_Windows_2_3)
+  backwardCompatibleTestOptions_Linux_2_4: $(backwardCompatibleTestOptions_Windows_2_3)
   # Filter HyperspaceTests not due to functionality changes, but to incompatible tests running on Linux.
   # Please see https://github.com/dotnet/spark/pull/737 for the fix.
   forwardCompatibleTestOptions_Linux_2_4: "--filter \
   (FullyQualifiedName!=Microsoft.Spark.Extensions.Hyperspace.E2ETest.HyperspaceTests.TestExplainAPI)&\
   (FullyQualifiedName!=Microsoft.Spark.Extensions.Hyperspace.E2ETest.HyperspaceTests.TestIndexCreateAndDelete)&\
-  (FullyQualifiedName!=Microsoft.Spark.Extensions.Hyperspace.E2ETest.HyperspaceTests.TestSignatures)"
+  (FullyQualifiedName!=Microsoft.Spark.Extensions.Hyperspace.E2ETest.HyperspaceTests.TestSignatures&\
+  (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithReturnAsTimestampType)&\
+  (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithTimestampType)"
 
   # Filter DataFrameTests.TestDataFrameGroupedMapUdf and DataFrameTests.TestGroupedMapUdf backwardCompatible
   # tests due to https://github.com/dotnet/spark/pull/711
@@ -38,8 +45,10 @@ variables:
   backwardCompatibleTestOptions_Windows_3_0: "--filter \
   (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestDataFrameGroupedMapUdf)&\
   (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestGroupedMapUdf)&\
-  (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithDuplicateTimestamps)"
-  forwardCompatibleTestOptions_Windows_3_0: ""
+  (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithDuplicateTimestamps)&\
+  (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithReturnAsTimestampType)&\
+  (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithTimestampType)"
+  forwardCompatibleTestOptions_Windows_3_0: $(backwardCompatibleTestOptions_Windows_2_3)
   backwardCompatibleTestOptions_Linux_3_0: $(backwardCompatibleTestOptions_Windows_3_0)
   forwardCompatibleTestOptions_Linux_3_0: $(forwardCompatibleTestOptions_Linux_2_4)
 

src/csharp/Microsoft.Spark.E2ETest/UdfTests/UdfSimpleTypesTests.cs

@@ -118,8 +118,8 @@ namespace Microsoft.Spark.E2ETest.UdfTests
 
             var expected = new string[]
             {
-                "2020-01-01 00:00:00.000000",
-                "2020-01-02 15:30:30.123456"
+                "2020-01-01 00:00:00.000000Z",
+                "2020-01-02 15:30:30.123456Z"
             };
             string[] rowsToArray = rows.Select(x => x[0].ToString()).ToArray();
             Assert.Equal(expected, rowsToArray);
@@ -193,8 +193,8 @@ namespace Microsoft.Spark.E2ETest.UdfTests
 
                 var expected = new string[]
                 {
-                    "2020-01-04 15:30:30.123456",
-                    "2050-01-04 15:30:30.123456"
+                    "2020-01-04 15:30:30.123456Z",
+                    "2050-01-04 15:30:30.123456Z"
                 };
                 for (int i = 0; i < rows.Length; ++i)
                 {

src/csharp/Microsoft.Spark.UnitTest/Sql/TimestampTests.cs

@@ -29,7 +29,7 @@ namespace Microsoft.Spark.UnitTest
                 Assert.Equal(DateTimeKind.Utc, timestamp.ToDateTime().Kind);
 
                 // Validate ToString().
-                Assert.Equal("2020-01-01 08:30:30.000123", timestamp.ToString());
+                Assert.Equal("2020-01-01 08:30:30.000123Z", timestamp.ToString());
 
                 // Validate ToDateTime().
                 Assert.Equal(testDate, timestamp.ToDateTime());
@@ -57,7 +57,7 @@ namespace Microsoft.Spark.UnitTest
                 Assert.Equal(DateTimeKind.Utc, timestamp.ToDateTime().Kind);
 
                 // Validate ToString().
-                Assert.Equal("2020-01-02 15:30:30.123456", timestamp.ToString());
+                Assert.Equal("2020-01-02 15:30:30.123456Z", timestamp.ToString());
 
                 // Validate ToDateTime().
                 Assert.Equal(
@@ -71,5 +71,14 @@ namespace Microsoft.Spark.UnitTest
                     () => new Timestamp(2020, 1, 2, 15, 30, 30, 1234567));
             }
         }
+
+        [Fact]
+        public void TestTimestampToString()
+        {
+            var dateTimeObj = new DateTime(2021, 01, 01);
+            Assert.Equal(
+                new Timestamp(DateTime.Parse(new Timestamp(dateTimeObj).ToString())),
+                new Timestamp(dateTimeObj));
+        }
     }
 }

src/csharp/Microsoft.Spark/Sql/Types/Timestamp.cs

@@ -94,7 +94,7 @@ namespace Microsoft.Spark.Sql.Types
         /// <summary>
         /// Readable string representation for this type.
         /// </summary>
-        public override string ToString() => _dateTime.ToString("yyyy-MM-dd HH:mm:ss.ffffff");
+        public override string ToString() => _dateTime.ToString("yyyy-MM-dd HH:mm:ss.ffffffZ");
 
         /// <summary>
         /// Checks if the given object is same as the current object.