From 885106f55faf54b9dfe5252238eb98a57029dc0e Mon Sep 17 00:00:00 2001 From: Chet Husk Date: Mon, 23 Sep 2024 13:28:41 -0500 Subject: [PATCH 1/2] allow reading of OCI Image Indexes to determine base image for multi-architecture manifests --- .../ManifestListV2.cs | 2 + .../PublicAPI/net9.0/PublicAPI.Unshipped.txt | 18 ++++++- .../Registry/Registry.cs | 53 ++++++++++++++++++- .../Registry/SchemaTypes.cs | 1 + 4 files changed, 72 insertions(+), 2 deletions(-) diff --git a/src/Containers/Microsoft.NET.Build.Containers/ManifestListV2.cs b/src/Containers/Microsoft.NET.Build.Containers/ManifestListV2.cs index 44a294d65473..b75547d8f166 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/ManifestListV2.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/ManifestListV2.cs @@ -10,3 +10,5 @@ public record struct ManifestListV2(int schemaVersion, string mediaType, Platfor public record struct PlatformInformation(string architecture, string os, string? variant, string[] features, [property: JsonPropertyName("os.version")][field: JsonPropertyName("os.version")] string? version); public record struct PlatformSpecificManifest(string mediaType, long size, string digest, PlatformInformation platform); + +public record struct ImageIndexV1(int schemaVersion, string mediaType, PlatformSpecificManifest[] manifests); diff --git a/src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net9.0/PublicAPI.Unshipped.txt b/src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net9.0/PublicAPI.Unshipped.txt index 698c2894586b..6d705a8b371f 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net9.0/PublicAPI.Unshipped.txt +++ b/src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net9.0/PublicAPI.Unshipped.txt @@ -72,6 +72,15 @@ Microsoft.NET.Build.Containers.ManifestListV2.mediaType.get -> string! Microsoft.NET.Build.Containers.ManifestListV2.mediaType.set -> void Microsoft.NET.Build.Containers.ManifestListV2.schemaVersion.get -> int Microsoft.NET.Build.Containers.ManifestListV2.schemaVersion.set -> void +Microsoft.NET.Build.Containers.ImageIndexV1 +Microsoft.NET.Build.Containers.ImageIndexV1.ImageIndexV1() -> void +Microsoft.NET.Build.Containers.ImageIndexV1.ImageIndexV1(int schemaVersion, string! mediaType, Microsoft.NET.Build.Containers.PlatformSpecificManifest[]! manifests) -> void +Microsoft.NET.Build.Containers.ImageIndexV1.manifests.get -> Microsoft.NET.Build.Containers.PlatformSpecificManifest[]! +Microsoft.NET.Build.Containers.ImageIndexV1.manifests.set -> void +Microsoft.NET.Build.Containers.ImageIndexV1.mediaType.get -> string! +Microsoft.NET.Build.Containers.ImageIndexV1.mediaType.set -> void +Microsoft.NET.Build.Containers.ImageIndexV1.schemaVersion.get -> int +Microsoft.NET.Build.Containers.ImageIndexV1.schemaVersion.set -> void Microsoft.NET.Build.Containers.ManifestV2 Microsoft.NET.Build.Containers.ManifestV2.Config.get -> Microsoft.NET.Build.Containers.ManifestConfig Microsoft.NET.Build.Containers.ManifestV2.Config.init -> void @@ -268,4 +277,11 @@ static Microsoft.NET.Build.Containers.ManifestListV2.operator ==(Microsoft.NET.B override Microsoft.NET.Build.Containers.ManifestListV2.GetHashCode() -> int ~override Microsoft.NET.Build.Containers.ManifestListV2.Equals(object obj) -> bool Microsoft.NET.Build.Containers.ManifestListV2.Equals(Microsoft.NET.Build.Containers.ManifestListV2 other) -> bool -Microsoft.NET.Build.Containers.ManifestListV2.Deconstruct(out int schemaVersion, out string! mediaType, out Microsoft.NET.Build.Containers.PlatformSpecificManifest[]! manifests) -> void \ No newline at end of file +Microsoft.NET.Build.Containers.ManifestListV2.Deconstruct(out int schemaVersion, out string! mediaType, out Microsoft.NET.Build.Containers.PlatformSpecificManifest[]! manifests) -> void +~override Microsoft.NET.Build.Containers.ImageIndexV1.ToString() -> string +static Microsoft.NET.Build.Containers.ImageIndexV1.operator !=(Microsoft.NET.Build.Containers.ImageIndexV1 left, Microsoft.NET.Build.Containers.ImageIndexV1 right) -> bool +static Microsoft.NET.Build.Containers.ImageIndexV1.operator ==(Microsoft.NET.Build.Containers.ImageIndexV1 left, Microsoft.NET.Build.Containers.ImageIndexV1 right) -> bool +override Microsoft.NET.Build.Containers.ImageIndexV1.GetHashCode() -> int +~override Microsoft.NET.Build.Containers.ImageIndexV1.Equals(object obj) -> bool +Microsoft.NET.Build.Containers.ImageIndexV1.Equals(Microsoft.NET.Build.Containers.ImageIndexV1 other) -> bool +Microsoft.NET.Build.Containers.ImageIndexV1.Deconstruct(out int schemaVersion, out string! mediaType, out Microsoft.NET.Build.Containers.PlatformSpecificManifest[]! manifests) -> void \ No newline at end of file diff --git a/src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs b/src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs index 512433c7cb73..3b052b7663f0 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs @@ -81,7 +81,7 @@ internal Registry(string registryName, ILogger logger, IRegistryAPI registryAPI, this(new Uri($"https://{registryName}"), logger, registryAPI, settings) { } - internal Registry(string registryName, ILogger logger, RegistryMode mode, RegistrySettings? settings = null) : + internal Registry(string registryName, ILogger logger, RegistryMode mode, RegistrySettings? settings = null) : this(new Uri($"https://{registryName}"), logger, new RegistryApiFactory(mode), settings) { } @@ -191,6 +191,14 @@ await initialManifestResponse.Content.ReadFromJsonAsync(cancella runtimeIdentifier, manifestPicker, cancellationToken).ConfigureAwait(false), + SchemaTypes.OciImageIndexV1 => + await PickBestImageFromImageIndexAsync( + repositoryName, + reference, + await initialManifestResponse.Content.ReadFromJsonAsync(cancellationToken: cancellationToken).ConfigureAwait(false), + runtimeIdentifier, + manifestPicker, + cancellationToken).ConfigureAwait(false), var unknownMediaType => throw new NotImplementedException(Resource.FormatString( nameof(Strings.UnknownMediaType), repositoryName, @@ -250,6 +258,20 @@ private static IReadOnlyDictionary GetManifest return ridDict; } + private static IReadOnlyDictionary GetManifestsByRid(ImageIndexV1 manifestList) + { + var ridDict = new Dictionary(); + foreach (var manifest in manifestList.manifests) + { + if (CreateRidForPlatform(manifest.platform) is { } rid) + { + ridDict.TryAdd(rid, manifest); + } + } + + return ridDict; + } + private static string? CreateRidForPlatform(PlatformInformation platform) { // we only support linux and windows containers explicitly, so anything else we should skip past. @@ -313,6 +335,35 @@ private async Task PickBestImageFromManifestListAsync( } } + private async Task PickBestImageFromImageIndexAsync( + string repositoryName, + string reference, + ImageIndexV1 index, + string runtimeIdentifier, + IManifestPicker manifestPicker, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var ridManifestDict = GetManifestsByRid(index); + if (manifestPicker.PickBestManifestForRid(ridManifestDict, runtimeIdentifier) is PlatformSpecificManifest matchingManifest) + { + using HttpResponseMessage manifestResponse = await _registryAPI.Manifest.GetAsync(repositoryName, matchingManifest.digest, cancellationToken).ConfigureAwait(false); + + cancellationToken.ThrowIfCancellationRequested(); + var manifest = await manifestResponse.Content.ReadFromJsonAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + if (manifest is null) throw new BaseImageNotFoundException(runtimeIdentifier, repositoryName, reference, ridManifestDict.Keys); + manifest.KnownDigest = matchingManifest.digest; + return await ReadSingleImageAsync( + repositoryName, + manifest, + cancellationToken).ConfigureAwait(false); + } + else + { + throw new BaseImageNotFoundException(runtimeIdentifier, repositoryName, reference, ridManifestDict.Keys); + } + } + /// /// Ensure a blob associated with from the registry is available locally. /// diff --git a/src/Containers/Microsoft.NET.Build.Containers/Registry/SchemaTypes.cs b/src/Containers/Microsoft.NET.Build.Containers/Registry/SchemaTypes.cs index d740c7621ecc..8021f652f72d 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Registry/SchemaTypes.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/Registry/SchemaTypes.cs @@ -10,6 +10,7 @@ internal class SchemaTypes internal const string DockerManifestListV2 = "application/vnd.docker.distribution.manifest.list.v2+json"; internal const string DockerManifestV2 = "application/vnd.docker.distribution.manifest.v2+json"; internal const string OciManifestV1 = "application/vnd.oci.image.manifest.v1+json"; // https://containers.gitbook.io/build-containers-the-hard-way/#registry-format-oci-image-manifest + internal const string OciImageIndexV1 = "application/vnd.oci.image.index.v1+json"; internal const string DockerLayerGzip = "application/vnd.docker.image.rootfs.diff.tar.gzip"; internal const string OciLayerGzipV1 = "application/vnd.oci.image.layer.v1.tar+gzip"; } From 7f1b189933408824fa9290a5f24c0d75ff79e8ce Mon Sep 17 00:00:00 2001 From: Chet Husk Date: Thu, 3 Oct 2024 11:46:08 -0500 Subject: [PATCH 2/2] add OCI Image Index to supported manifest types, and deduplicate the logic for selecting the best manifest from a set of manifests --- .../Registry/HttpExtensions.cs | 18 +++-- .../Registry/Registry.cs | 70 +++++++++---------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/src/Containers/Microsoft.NET.Build.Containers/Registry/HttpExtensions.cs b/src/Containers/Microsoft.NET.Build.Containers/Registry/HttpExtensions.cs index 09d2274467db..5439d3755aa7 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Registry/HttpExtensions.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/Registry/HttpExtensions.cs @@ -1,21 +1,27 @@ // Copyright (c) .NET Foundation and contributors. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -// +using System.Net.Http.Headers; using Microsoft.Extensions.Logging; +using NuGet.Packaging; namespace Microsoft.NET.Build.Containers; internal static class HttpExtensions { + private static readonly MediaTypeWithQualityHeaderValue[] _knownManifestFormats = [ + new("application/json"), + new(SchemaTypes.DockerManifestListV2), + new(SchemaTypes.OciImageIndexV1), + new(SchemaTypes.DockerManifestV2), + new(SchemaTypes.OciManifestV1), + new(SchemaTypes.DockerContainerV1), + ]; + internal static HttpRequestMessage AcceptManifestFormats(this HttpRequestMessage request) { request.Headers.Accept.Clear(); - request.Headers.Accept.Add(new("application/json")); - request.Headers.Accept.Add(new(SchemaTypes.DockerManifestListV2)); - request.Headers.Accept.Add(new(SchemaTypes.DockerManifestV2)); - request.Headers.Accept.Add(new(SchemaTypes.OciManifestV1)); - request.Headers.Accept.Add(new(SchemaTypes.DockerContainerV1)); + request.Headers.Accept.AddRange(_knownManifestFormats); return request; } diff --git a/src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs b/src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs index 3b052b7663f0..20093b764c77 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs @@ -244,24 +244,10 @@ private async Task ReadSingleImageAsync(string repositoryName, Man } - private static IReadOnlyDictionary GetManifestsByRid(ManifestListV2 manifestList) + private static IReadOnlyDictionary GetManifestsByRid(PlatformSpecificManifest[] manifestList) { var ridDict = new Dictionary(); - foreach (var manifest in manifestList.manifests) - { - if (CreateRidForPlatform(manifest.platform) is { } rid) - { - ridDict.TryAdd(rid, manifest); - } - } - - return ridDict; - } - - private static IReadOnlyDictionary GetManifestsByRid(ImageIndexV1 manifestList) - { - var ridDict = new Dictionary(); - foreach (var manifest in manifestList.manifests) + foreach (var manifest in manifestList) { if (CreateRidForPlatform(manifest.platform) is { } rid) { @@ -315,24 +301,14 @@ private async Task PickBestImageFromManifestListAsync( CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - var ridManifestDict = GetManifestsByRid(manifestList); - if (manifestPicker.PickBestManifestForRid(ridManifestDict, runtimeIdentifier) is PlatformSpecificManifest matchingManifest) - { - using HttpResponseMessage manifestResponse = await _registryAPI.Manifest.GetAsync(repositoryName, matchingManifest.digest, cancellationToken).ConfigureAwait(false); - - cancellationToken.ThrowIfCancellationRequested(); - var manifest = await manifestResponse.Content.ReadFromJsonAsync(cancellationToken: cancellationToken).ConfigureAwait(false); - if (manifest is null) throw new BaseImageNotFoundException(runtimeIdentifier, repositoryName, reference, ridManifestDict.Keys); - manifest.KnownDigest = matchingManifest.digest; - return await ReadSingleImageAsync( - repositoryName, - manifest, - cancellationToken).ConfigureAwait(false); - } - else - { - throw new BaseImageNotFoundException(runtimeIdentifier, repositoryName, reference, ridManifestDict.Keys); - } + var ridManifestDict = GetManifestsByRid(manifestList.manifests); + return await PickBestImageFromManifestsAsync( + repositoryName, + reference, + ridManifestDict, + runtimeIdentifier, + manifestPicker, + cancellationToken).ConfigureAwait(false); } private async Task PickBestImageFromImageIndexAsync( @@ -344,14 +320,32 @@ private async Task PickBestImageFromImageIndexAsync( CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - var ridManifestDict = GetManifestsByRid(index); - if (manifestPicker.PickBestManifestForRid(ridManifestDict, runtimeIdentifier) is PlatformSpecificManifest matchingManifest) + var ridManifestDict = GetManifestsByRid(index.manifests); + return await PickBestImageFromManifestsAsync( + repositoryName, + reference, + ridManifestDict, + runtimeIdentifier, + manifestPicker, + cancellationToken).ConfigureAwait(false); + } + + private async Task PickBestImageFromManifestsAsync( + string repositoryName, + string reference, + IReadOnlyDictionary knownManifests, + string runtimeIdentifier, + IManifestPicker manifestPicker, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (manifestPicker.PickBestManifestForRid(knownManifests, runtimeIdentifier) is PlatformSpecificManifest matchingManifest) { using HttpResponseMessage manifestResponse = await _registryAPI.Manifest.GetAsync(repositoryName, matchingManifest.digest, cancellationToken).ConfigureAwait(false); cancellationToken.ThrowIfCancellationRequested(); var manifest = await manifestResponse.Content.ReadFromJsonAsync(cancellationToken: cancellationToken).ConfigureAwait(false); - if (manifest is null) throw new BaseImageNotFoundException(runtimeIdentifier, repositoryName, reference, ridManifestDict.Keys); + if (manifest is null) throw new BaseImageNotFoundException(runtimeIdentifier, repositoryName, reference, knownManifests.Keys); manifest.KnownDigest = matchingManifest.digest; return await ReadSingleImageAsync( repositoryName, @@ -360,7 +354,7 @@ private async Task PickBestImageFromImageIndexAsync( } else { - throw new BaseImageNotFoundException(runtimeIdentifier, repositoryName, reference, ridManifestDict.Keys); + throw new BaseImageNotFoundException(runtimeIdentifier, repositoryName, reference, knownManifests.Keys); } }