From 25aa71454056ddb5217a80ddc7fb8d6e451ad414 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Mon, 10 Jun 2024 11:59:03 -0700 Subject: [PATCH] Resolve Digest Round Trip Issues (#8325) * we have a test case that actually exercises the round tripping * we have successful round trip repro of the problem. time to figure out how to fix it * ensure that content type 'application/vnd.docker.distribution.manifest.v2' is not treated as a text type, ensuring that the whitespace gets stored exactly as-is --- .../RecordSessionTests.cs | 80 +++++++++++++++++++ .../response_with_content_digest.json | 60 ++++++++++++++ .../TestHelpers.cs | 26 +++++- .../Common/ContentTypeUtilities.cs | 21 +++-- 4 files changed, 177 insertions(+), 10 deletions(-) create mode 100644 tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/Test.RecordEntries/response_with_content_digest.json diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/RecordSessionTests.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/RecordSessionTests.cs index ded14d6b1..03d3e12bd 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/RecordSessionTests.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/RecordSessionTests.cs @@ -9,10 +9,14 @@ using System.Linq; using System.Text; using System.Text.Encodings.Web; using System.Text.Json; +using System.Threading.Tasks; using Azure.Core; using Azure.Core.Pipeline; using Azure.Sdk.Tools.TestProxy.Common; +using Microsoft.AspNetCore.Http; +using Microsoft.VisualBasic; using Moq; +using NuGet.ContentModel; using Xunit; namespace Azure.Sdk.Tools.TestProxy.Tests @@ -95,6 +99,82 @@ namespace Azure.Sdk.Tools.TestProxy.Tests Assert.Equal(bodyBytes, deserializedRecord.Response.Body); } + [Fact] + public async Task CanRoundTripDockerDigest() + { + // get everything organized + var sampleExpected = "{\n \"schemaVersion\": 2,\n \"mediaType\": \"application/vnd.docker.distribution.manifest.v2+json\",\n \"config\": {\n \"mediaType\": \"application/vnd.docker.container.image.v1+json\",\n \"size\"" + + ": 1472,\n \"digest\": \"sha256:042a816809aac8d0f7d7cacac7965782ee2ecac3f21bcf9f24b1de1a7387b769\"\n },\n \"layers\": [\n {\n \"mediaType\": \"application/vnd.docker.image.rootfs.diff.tar.gzip\",\n \"size\"" + "" + + ": 3370628,\n \"digest\": \"sha256:8921db27df2831fa6eaa85321205a2470c669b855f3ec95d5a3c2b46de0442c9\"\n }\n ]\n}"; + var testName = "roundtrip.json"; + DefaultHttpContext ctx = new DefaultHttpContext(); + Assets assets = new Assets() + { + AssetsRepo = "Azure/azure-sdk-assets-integration", + AssetsRepoPrefixPath = "pull/scenarios", + AssetsRepoId = "", + TagPrefix = "language/tables", + Tag = "python/tables_fc54d0" + }; + var folderStructure = new string[] + { + GitStoretests.AssetsJson + }; + var testEntry = new RecordEntry() + { + RequestUri = "https://Sanitized.azurecr.io/v2/alpine/manifests/3.17.1", + RequestMethod = RequestMethod.Get, + Request = new RequestOrResponse() + { + Headers = new SortedDictionary() + { + { "Accept", new string[]{ "application/json", "application/vnd.docker.distribution.manifest.v2+json" } }, + { "Accept-Encoding", new string[]{ "gzip" } }, + { "Authorization", new string[]{ "Sanitized" } }, + { "User-Agent", new string[]{ "azsdk-go-azcontainerregistry/v0.2.2 (go1.21.6; linux)" } }, + }, + Body = null, + }, + StatusCode = 200, + Response = new RequestOrResponse() + { + Headers = new SortedDictionary() + { + { "Access-Control-Expose-Headers", new string[]{ "Docker-Content-Digest", "WWW-Authenticate", "Link","X-Ms-Correlation-Request-Id" } }, + { "Connection", new string[]{ "keep-alive" } }, + { "Content-Length", new string[]{ "528" } }, + { "Content-Type", new string[]{ "application/vnd.docker.distribution.manifest.v2+json" } }, + { "Date", new string[]{ "Fri, 17 May 2024 21:42:34 GMT" } }, + { "Docker-Content-Digest", new string[]{ "sha256:93d5a28ff72d288d69b5997b8ba47396d2cbb62a72b5d87cd3351094b5d578a0" } }, + { "Docker-Distribution-Api-Version", new string[]{ "registry/2.0" } }, + { "ETag", new string[]{ "\"sha256:93d5a28ff72d288d69b5997b8ba47396d2cbb62a72b5d87cd3351094b5d578a0\"" } }, + { "Server", new string[]{ "AzureContainerRegistry" } }, + { "Strict-Transport-Security", new string[]{ "max-age=31536000; includeSubDomains", "max-age=31536000; includeSubDomains" } }, + { "X-Content-Type-Options", new string[]{ "nosniff" } }, + { "X-Ms-Client-Request-Id", new string[]{ "" } }, + { "X-Ms-Correlation-Request-Id", new string[]{ "caf56438-d3ba-469d-a30c-360a4ff536c1" } }, + { "X-Ms-Request-Id", new string[]{ "Sanitized" } }, + }, + Body = Encoding.UTF8.GetBytes(sampleExpected) + }, + }; + + // create the session which will be saved to disk, then save it + var testFolder = TestHelpers.DescribeTestFolder(assets, folderStructure); + var handler = new RecordingHandler(testFolder); + await handler.StartRecordingAsync(testName, ctx.Response); + var recordingId = ctx.Response.Headers["x-recording-id"].ToString(); + var session = handler.RecordingSessions[recordingId]; + session.Session.Entries.Add(testEntry); + handler.StopRecording(recordingId); + + // now load it, did we avoid mangling it? + var sessionFromDisk = TestHelpers.LoadRecordSession(Path.Combine(testFolder, testName)); + var targetEntry = sessionFromDisk.Session.Entries[0]; + var content = Encoding.UTF8.GetString(targetEntry.Response.Body); + Assert.Equal(sampleExpected, content); + } + [Fact] public void EnsureJsonEscaping() { diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/Test.RecordEntries/response_with_content_digest.json b/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/Test.RecordEntries/response_with_content_digest.json new file mode 100644 index 000000000..f89b903ee --- /dev/null +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/Test.RecordEntries/response_with_content_digest.json @@ -0,0 +1,60 @@ +{ + "Entries": [ + { + "RequestUri": "https://Sanitized.azurecr.io/v2/alpine/manifests/3.17.1", + "RequestMethod": "GET", + "RequestHeaders": { + "Accept": [ + "application/json", + "application/vnd.docker.distribution.manifest.v2+json" + ], + "Accept-Encoding": "gzip", + "Authorization": "Sanitized", + "User-Agent": "azsdk-go-azcontainerregistry/v0.2.2 (go1.22.2; Windows_NT)" + }, + "RequestBody": null, + "StatusCode": 200, + "ResponseHeaders": { + "Access-Control-Expose-Headers": [ + "Docker-Content-Digest", + "WWW-Authenticate", + "Link", + "X-Ms-Correlation-Request-Id" + ], + "Connection": "keep-alive", + "Content-Length": "528", + "Content-Type": "application/vnd.docker.distribution.manifest.v2+json", + "Date": "Wed, 22 May 2024 20:40:43 GMT", + "Docker-Content-Digest": "sha256:93d5a28ff72d288d69b5997b8ba47396d2cbb62a72b5d87cd3351094b5d578a0", + "Docker-Distribution-Api-Version": "registry/2.0", + "ETag": "\"sha256:93d5a28ff72d288d69b5997b8ba47396d2cbb62a72b5d87cd3351094b5d578a0\"", + "Server": "AzureContainerRegistry", + "Strict-Transport-Security": [ + "max-age=31536000; includeSubDomains", + "max-age=31536000; includeSubDomains" + ], + "X-Content-Type-Options": "nosniff", + "X-Ms-Client-Request-Id": "", + "X-Ms-Correlation-Request-Id": "19affbee-3510-45b1-8248-9dc23982613b", + "X-Ms-Request-Id": "Sanitized" + }, + "ResponseBody": { + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "size": 1472, + "digest": "sha256:042a816809aac8d0f7d7cacac7965782ee2ecac3f21bcf9f24b1de1a7387b769" + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 3370628, + "digest": "sha256:8921db27df2831fa6eaa85321205a2470c669b855f3ec95d5a3c2b46de0442c9" + } + ] + } + } + ], + "Variables": {} +} diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/TestHelpers.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/TestHelpers.cs index 6715ca070..3fdf58d63 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/TestHelpers.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/TestHelpers.cs @@ -148,6 +148,27 @@ namespace Azure.Sdk.Tools.TestProxy.Tests File.WriteAllText(path, content); } + public static string GetTmpPath(string[] pathsBeyondFolder = null) + { + var pathSuffix = string.Empty; + + if (pathsBeyondFolder != null && pathsBeyondFolder.Length > 0) { + pathSuffix += Path.Combine(pathsBeyondFolder); + } + else + { + pathSuffix = Guid.NewGuid().ToString(); + } + + var tmpPath = Path.Join(Path.GetTempPath(), pathSuffix); + + if (!Directory.Exists(tmpPath)) { + Directory.CreateDirectory(tmpPath); + } + + return tmpPath; + } + /// /// Used to define any set of file constructs we want. This enables us to roll a target environment to point various GitStore functionalities at. /// @@ -169,9 +190,8 @@ namespace Azure.Sdk.Tools.TestProxy.Tests } // the guid will be used to create a unique test folder root and, if this is a push test, // it'll be used as part of the generated branch name - string testGuid = Guid.NewGuid().ToString(); - // generate a test folder root - var tmpPath = Path.Join(Path.GetTempPath(), testGuid); + var testGuid = Guid.NewGuid().ToString(); + var tmpPath = GetTmpPath(new string[] { testGuid }); // Push tests need some special setup for automation // 1. The AssetsReproBranch diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/ContentTypeUtilities.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/ContentTypeUtilities.cs index 7c925078a..3decf6be0 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/ContentTypeUtilities.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/ContentTypeUtilities.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. #nullable disable @@ -22,6 +22,7 @@ namespace Azure.Sdk.Tools.TestProxy.Common // Default is technically US-ASCII, but will default to UTF-8 which is a superset. const string appFormUrlEncoded = "application/x-www-form-urlencoded"; + const string dockerManifest = "application/vnd.docker.distribution.manifest.v2"; if (contentType == null) { @@ -40,17 +41,23 @@ namespace Azure.Sdk.Tools.TestProxy.Common } } - if (contentType.StartsWith(textContentTypePrefix, StringComparison.OrdinalIgnoreCase) || - contentType.EndsWith(jsonSuffix, StringComparison.OrdinalIgnoreCase) || - contentType.EndsWith(xmlSuffix, StringComparison.OrdinalIgnoreCase) || - contentType.EndsWith(urlEncodedSuffix, StringComparison.OrdinalIgnoreCase) || - contentType.StartsWith(appJsonPrefix, StringComparison.OrdinalIgnoreCase) || - contentType.StartsWith(appFormUrlEncoded, StringComparison.OrdinalIgnoreCase)) + + if ( + ( + contentType.StartsWith(textContentTypePrefix, StringComparison.OrdinalIgnoreCase) || + contentType.EndsWith(jsonSuffix, StringComparison.OrdinalIgnoreCase) || + contentType.EndsWith(xmlSuffix, StringComparison.OrdinalIgnoreCase) || + contentType.EndsWith(urlEncodedSuffix, StringComparison.OrdinalIgnoreCase) || + contentType.StartsWith(appJsonPrefix, StringComparison.OrdinalIgnoreCase) || + contentType.StartsWith(appFormUrlEncoded, StringComparison.OrdinalIgnoreCase) + ) && !contentType.Contains(dockerManifest) + ) { encoding = Encoding.UTF8; return true; } + encoding = null; return false; }