Resolve Digest Round Trip Issues (#8325)

* we have a test case that actually exercises the round tripping
* we have successful round trip repro of the problem. time to figure out how to fix it
* ensure that content type 'application/vnd.docker.distribution.manifest.v2' is not treated as a text type, ensuring that the whitespace gets stored exactly as-is

tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/RecordSessionTests.cs

@@ -9,10 +9,14 @@ using System.Linq;
 using System.Text;
 using System.Text.Encodings.Web;
 using System.Text.Json;
+using System.Threading.Tasks;
 using Azure.Core;
 using Azure.Core.Pipeline;
 using Azure.Sdk.Tools.TestProxy.Common;
+using Microsoft.AspNetCore.Http;
+using Microsoft.VisualBasic;
 using Moq;
+using NuGet.ContentModel;
 using Xunit;
 
 namespace Azure.Sdk.Tools.TestProxy.Tests
@@ -95,6 +99,82 @@ namespace Azure.Sdk.Tools.TestProxy.Tests
             Assert.Equal(bodyBytes, deserializedRecord.Response.Body);
         }
 
+        [Fact]
+        public async Task CanRoundTripDockerDigest()
+        {
+            // get everything organized
+            var sampleExpected = "{\n   \"schemaVersion\": 2,\n   \"mediaType\": \"application/vnd.docker.distribution.manifest.v2+json\",\n   \"config\": {\n      \"mediaType\": \"application/vnd.docker.container.image.v1+json\",\n      \"size\"" +
+                ": 1472,\n      \"digest\": \"sha256:042a816809aac8d0f7d7cacac7965782ee2ecac3f21bcf9f24b1de1a7387b769\"\n   },\n   \"layers\": [\n      {\n         \"mediaType\": \"application/vnd.docker.image.rootfs.diff.tar.gzip\",\n         \"size\"" + "" +
+                ": 3370628,\n         \"digest\": \"sha256:8921db27df2831fa6eaa85321205a2470c669b855f3ec95d5a3c2b46de0442c9\"\n      }\n   ]\n}";
+            var testName = "roundtrip.json";
+            DefaultHttpContext ctx = new DefaultHttpContext();
+            Assets assets = new Assets()
+            {
+                AssetsRepo = "Azure/azure-sdk-assets-integration",
+                AssetsRepoPrefixPath = "pull/scenarios",
+                AssetsRepoId = "",
+                TagPrefix = "language/tables",
+                Tag = "python/tables_fc54d0"
+            };
+            var folderStructure = new string[]
+            {
+                GitStoretests.AssetsJson
+            };
+            var testEntry = new RecordEntry()
+            {
+                RequestUri = "https://Sanitized.azurecr.io/v2/alpine/manifests/3.17.1",
+                RequestMethod = RequestMethod.Get,
+                Request = new RequestOrResponse()
+                {
+                    Headers = new SortedDictionary<string, string[]>()
+                    {
+                        { "Accept", new string[]{ "application/json", "application/vnd.docker.distribution.manifest.v2+json" } },
+                        { "Accept-Encoding", new string[]{ "gzip" } },
+                        { "Authorization", new string[]{ "Sanitized" } },
+                        { "User-Agent", new string[]{ "azsdk-go-azcontainerregistry/v0.2.2 (go1.21.6; linux)" } },
+                    },
+                    Body = null,
+                },
+                StatusCode = 200,
+                Response = new RequestOrResponse()
+                {
+                    Headers = new SortedDictionary<string, string[]>()
+                    {
+                        { "Access-Control-Expose-Headers", new string[]{ "Docker-Content-Digest", "WWW-Authenticate", "Link","X-Ms-Correlation-Request-Id" } },
+                        { "Connection", new string[]{ "keep-alive" } },
+                        { "Content-Length", new string[]{ "528" } },
+                        { "Content-Type", new string[]{ "application/vnd.docker.distribution.manifest.v2+json" } },
+                        { "Date", new string[]{ "Fri, 17 May 2024 21:42:34 GMT" } },
+                        { "Docker-Content-Digest", new string[]{ "sha256:93d5a28ff72d288d69b5997b8ba47396d2cbb62a72b5d87cd3351094b5d578a0" } },
+                        { "Docker-Distribution-Api-Version", new string[]{ "registry/2.0" } },
+                        { "ETag", new string[]{ "\"sha256:93d5a28ff72d288d69b5997b8ba47396d2cbb62a72b5d87cd3351094b5d578a0\"" } },
+                        { "Server", new string[]{ "AzureContainerRegistry" } },
+                        { "Strict-Transport-Security", new string[]{ "max-age=31536000; includeSubDomains", "max-age=31536000; includeSubDomains" } },
+                        { "X-Content-Type-Options", new string[]{ "nosniff" } },
+                        { "X-Ms-Client-Request-Id", new string[]{ "" } },
+                        { "X-Ms-Correlation-Request-Id", new string[]{ "caf56438-d3ba-469d-a30c-360a4ff536c1" } },
+                        { "X-Ms-Request-Id", new string[]{ "Sanitized" } },
+                    },
+                    Body = Encoding.UTF8.GetBytes(sampleExpected)
+                },
+            };
+
+            // create the session which will be saved to disk, then save it
+            var testFolder = TestHelpers.DescribeTestFolder(assets, folderStructure);
+            var handler = new RecordingHandler(testFolder);
+            await handler.StartRecordingAsync(testName, ctx.Response);
+            var recordingId = ctx.Response.Headers["x-recording-id"].ToString();
+            var session = handler.RecordingSessions[recordingId];
+            session.Session.Entries.Add(testEntry);
+            handler.StopRecording(recordingId);
+
+            // now load it, did we avoid mangling it?
+            var sessionFromDisk = TestHelpers.LoadRecordSession(Path.Combine(testFolder, testName));
+            var targetEntry = sessionFromDisk.Session.Entries[0];
+            var content = Encoding.UTF8.GetString(targetEntry.Response.Body);
+            Assert.Equal(sampleExpected, content);
+        }
+
         [Fact]
         public void EnsureJsonEscaping()
         {

tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/Test.RecordEntries/response_with_content_digest.json

@@ -0,0 +1,60 @@
+{
+  "Entries": [
+    {
+      "RequestUri": "https://Sanitized.azurecr.io/v2/alpine/manifests/3.17.1",
+      "RequestMethod": "GET",
+      "RequestHeaders": {
+        "Accept": [
+          "application/json",
+          "application/vnd.docker.distribution.manifest.v2+json"
+        ],
+        "Accept-Encoding": "gzip",
+        "Authorization": "Sanitized",
+        "User-Agent": "azsdk-go-azcontainerregistry/v0.2.2 (go1.22.2; Windows_NT)"
+      },
+      "RequestBody": null,
+      "StatusCode": 200,
+      "ResponseHeaders": {
+        "Access-Control-Expose-Headers": [
+          "Docker-Content-Digest",
+          "WWW-Authenticate",
+          "Link",
+          "X-Ms-Correlation-Request-Id"
+        ],
+        "Connection": "keep-alive",
+        "Content-Length": "528",
+        "Content-Type": "application/vnd.docker.distribution.manifest.v2+json",
+        "Date": "Wed, 22 May 2024 20:40:43 GMT",
+        "Docker-Content-Digest": "sha256:93d5a28ff72d288d69b5997b8ba47396d2cbb62a72b5d87cd3351094b5d578a0",
+        "Docker-Distribution-Api-Version": "registry/2.0",
+        "ETag": "\"sha256:93d5a28ff72d288d69b5997b8ba47396d2cbb62a72b5d87cd3351094b5d578a0\"",
+        "Server": "AzureContainerRegistry",
+        "Strict-Transport-Security": [
+          "max-age=31536000; includeSubDomains",
+          "max-age=31536000; includeSubDomains"
+        ],
+        "X-Content-Type-Options": "nosniff",
+        "X-Ms-Client-Request-Id": "",
+        "X-Ms-Correlation-Request-Id": "19affbee-3510-45b1-8248-9dc23982613b",
+        "X-Ms-Request-Id": "Sanitized"
+      },
+      "ResponseBody": {
+        "schemaVersion": 2,
+        "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+        "config": {
+          "mediaType": "application/vnd.docker.container.image.v1+json",
+          "size": 1472,
+          "digest": "sha256:042a816809aac8d0f7d7cacac7965782ee2ecac3f21bcf9f24b1de1a7387b769"
+        },
+        "layers": [
+          {
+            "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+            "size": 3370628,
+            "digest": "sha256:8921db27df2831fa6eaa85321205a2470c669b855f3ec95d5a3c2b46de0442c9"
+          }
+        ]
+      }
+    }
+  ],
+  "Variables": {}
+}

tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/TestHelpers.cs

@@ -148,6 +148,27 @@ namespace Azure.Sdk.Tools.TestProxy.Tests
             File.WriteAllText(path, content);
         }
 
+        public static string GetTmpPath(string[] pathsBeyondFolder = null)
+        {
+            var pathSuffix = string.Empty;
+
+            if (pathsBeyondFolder != null && pathsBeyondFolder.Length > 0) {
+                pathSuffix += Path.Combine(pathsBeyondFolder);
+            }
+            else
+            {
+                pathSuffix = Guid.NewGuid().ToString();
+            }
+
+            var tmpPath = Path.Join(Path.GetTempPath(), pathSuffix);
+
+            if (!Directory.Exists(tmpPath)) {
+                Directory.CreateDirectory(tmpPath);
+            }
+
+            return tmpPath;
+        }
+
         /// <summary>
         /// Used to define any set of file constructs we want. This enables us to roll a target environment to point various GitStore functionalities at.
         ///
@@ -169,9 +190,8 @@ namespace Azure.Sdk.Tools.TestProxy.Tests
             }
             // the guid will be used to create a unique test folder root and, if this is a push test,
             // it'll be used as part of the generated branch name
-            string testGuid = Guid.NewGuid().ToString();
+            var testGuid = Guid.NewGuid().ToString();
-            // generate a test folder root
+            var tmpPath = GetTmpPath(new string[] { testGuid });
-            var tmpPath = Path.Join(Path.GetTempPath(), testGuid);
 
             // Push tests need some special setup for automation
             // 1. The AssetsReproBranch

tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/ContentTypeUtilities.cs

@@ -1,4 +1,4 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
+// Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
 #nullable disable
@@ -22,6 +22,7 @@ namespace Azure.Sdk.Tools.TestProxy.Common
 
             // Default is technically US-ASCII, but will default to UTF-8 which is a superset.
             const string appFormUrlEncoded = "application/x-www-form-urlencoded";
+            const string dockerManifest = "application/vnd.docker.distribution.manifest.v2";
 
             if (contentType == null)
             {
@@ -40,17 +41,23 @@ namespace Azure.Sdk.Tools.TestProxy.Common
                 }
             }
 
-            if (contentType.StartsWith(textContentTypePrefix, StringComparison.OrdinalIgnoreCase) ||
+
-                contentType.EndsWith(jsonSuffix, StringComparison.OrdinalIgnoreCase) ||
+            if (
-                contentType.EndsWith(xmlSuffix, StringComparison.OrdinalIgnoreCase) ||
+                    (
-                contentType.EndsWith(urlEncodedSuffix, StringComparison.OrdinalIgnoreCase) ||
+                        contentType.StartsWith(textContentTypePrefix, StringComparison.OrdinalIgnoreCase) ||
-                contentType.StartsWith(appJsonPrefix, StringComparison.OrdinalIgnoreCase) ||
+                        contentType.EndsWith(jsonSuffix, StringComparison.OrdinalIgnoreCase) ||
-                contentType.StartsWith(appFormUrlEncoded, StringComparison.OrdinalIgnoreCase))
+                        contentType.EndsWith(xmlSuffix, StringComparison.OrdinalIgnoreCase) ||
+                        contentType.EndsWith(urlEncodedSuffix, StringComparison.OrdinalIgnoreCase) ||
+                        contentType.StartsWith(appJsonPrefix, StringComparison.OrdinalIgnoreCase) ||
+                        contentType.StartsWith(appFormUrlEncoded, StringComparison.OrdinalIgnoreCase)
+                    ) && !contentType.Contains(dockerManifest)
+                )
             {
                 encoding = Encoding.UTF8;
                 return true;
             }
 
+
             encoding = null;
             return false;
         }