From 64c6314d7357e689ea5f4eaaa975817b8e9e1994 Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Wed, 5 Jul 2023 17:10:54 +0800 Subject: [PATCH 01/14] update Successors() Signed-off-by: Lixia (Sylvia) Lei --- content/graph.go | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/content/graph.go b/content/graph.go index fa2f9efe..f81bb09d 100644 --- a/content/graph.go +++ b/content/graph.go @@ -18,6 +18,7 @@ package content import ( "context" "encoding/json" + "fmt" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2/internal/docker" @@ -57,7 +58,7 @@ func Successors(ctx context.Context, fetcher Fetcher, node ocispec.Descriptor) ( // OCI manifest schema can be used to marshal docker manifest var manifest ocispec.Manifest if err := json.Unmarshal(content, &manifest); err != nil { - return nil, err + return nil, fmt.Errorf("failed to unmarshal %s: %s: %w", node.Digest.String(), node.MediaType, err) } return append([]ocispec.Descriptor{manifest.Config}, manifest.Layers...), nil case ocispec.MediaTypeImageManifest: @@ -67,7 +68,7 @@ func Successors(ctx context.Context, fetcher Fetcher, node ocispec.Descriptor) ( } var manifest ocispec.Manifest if err := json.Unmarshal(content, &manifest); err != nil { - return nil, err + return nil, fmt.Errorf("failed to unmarshal %s: %s: %w", node.Digest.String(), node.MediaType, err) } var nodes []ocispec.Descriptor if manifest.Subject != nil { @@ -75,18 +76,33 @@ func Successors(ctx context.Context, fetcher Fetcher, node ocispec.Descriptor) ( } nodes = append(nodes, manifest.Config) return append(nodes, manifest.Layers...), nil - case docker.MediaTypeManifestList, ocispec.MediaTypeImageIndex: + case docker.MediaTypeManifestList: content, err := FetchAll(ctx, fetcher, node) if err != nil { return nil, err } - // docker manifest list and oci index are equivalent for successors. + // OCI Index schema can be used to marshal docker manifest list var index ocispec.Index if err := json.Unmarshal(content, &index); err != nil { - return nil, err + return nil, fmt.Errorf("failed to unmarshal %s: %s: %w", node.Digest.String(), node.MediaType, err) } return index.Manifests, nil + case ocispec.MediaTypeImageIndex: + content, err := FetchAll(ctx, fetcher, node) + if err != nil { + return nil, err + } + + var index ocispec.Index + if err := json.Unmarshal(content, &index); err != nil { + return nil, fmt.Errorf("failed to unmarshal %s: %s: %w", node.Digest.String(), node.MediaType, err) + } + var manifests []ocispec.Descriptor + if index.Subject != nil { + manifests = append(manifests, *index.Subject) + } + return append(manifests, index.Manifests...), nil case spec.MediaTypeArtifactManifest: content, err := FetchAll(ctx, fetcher, node) if err != nil { @@ -95,7 +111,7 @@ func Successors(ctx context.Context, fetcher Fetcher, node ocispec.Descriptor) ( var manifest spec.Artifact if err := json.Unmarshal(content, &manifest); err != nil { - return nil, err + return nil, fmt.Errorf("failed to unmarshal %s: %s: %w", node.Digest.String(), node.MediaType, err) } var nodes []ocispec.Descriptor if manifest.Subject != nil { From cefa823468a5361c9747242669cae266825d3673 Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Wed, 5 Jul 2023 19:31:27 +0800 Subject: [PATCH 02/14] add manifest parse method Signed-off-by: Lixia (Sylvia) Lei --- internal/manifestutil/parse.go | 93 ++++++++++++++++++++++++++++++++++ internal/platform/platform.go | 8 +-- 2 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 internal/manifestutil/parse.go diff --git a/internal/manifestutil/parse.go b/internal/manifestutil/parse.go new file mode 100644 index 00000000..a7c69e5a --- /dev/null +++ b/internal/manifestutil/parse.go @@ -0,0 +1,93 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manifestutil + +import ( + "context" + "encoding/json" + "fmt" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/content" + "oras.land/oras-go/v2/internal/docker" + "oras.land/oras-go/v2/internal/spec" +) + +func Parse(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) (subject, config *ocispec.Descriptor, items []ocispec.Descriptor, err error) { + switch desc.MediaType { + case docker.MediaTypeManifest: + content, err := content.FetchAll(ctx, fetcher, desc) + if err != nil { + return nil, nil, nil, err + } + // OCI manifest schema can be used to marshal docker manifest + var manifest ocispec.Manifest + if err := json.Unmarshal(content, &manifest); err != nil { + return nil, nil, nil, fmt.Errorf("failed to unmarshal %s: %s: %w", desc.Digest.String(), desc.MediaType, err) + } + config = &manifest.Config + items = manifest.Layers + case ocispec.MediaTypeImageManifest: + content, err := content.FetchAll(ctx, fetcher, desc) + if err != nil { + return nil, nil, nil, err + } + var manifest ocispec.Manifest + if err := json.Unmarshal(content, &manifest); err != nil { + return nil, nil, nil, fmt.Errorf("failed to unmarshal %s: %s: %w", desc.Digest.String(), desc.MediaType, err) + } + subject = manifest.Subject + config = &manifest.Config + items = manifest.Layers + case docker.MediaTypeManifestList: + content, err := content.FetchAll(ctx, fetcher, desc) + if err != nil { + return nil, nil, nil, err + } + + // OCI Index schema can be used to marshal docker manifest list + var index ocispec.Index + if err := json.Unmarshal(content, &index); err != nil { + return nil, nil, nil, fmt.Errorf("failed to unmarshal %s: %s: %w", desc.Digest.String(), desc.MediaType, err) + } + items = index.Manifests + case ocispec.MediaTypeImageIndex: + content, err := content.FetchAll(ctx, fetcher, desc) + if err != nil { + return nil, nil, nil, err + } + + var index ocispec.Index + if err := json.Unmarshal(content, &index); err != nil { + return nil, nil, nil, fmt.Errorf("failed to unmarshal %s: %s: %w", desc.Digest.String(), desc.MediaType, err) + } + subject = index.Subject + items = index.Manifests + case spec.MediaTypeArtifactManifest: + content, err := content.FetchAll(ctx, fetcher, desc) + if err != nil { + return nil, nil, nil, err + } + + var manifest spec.Artifact + if err := json.Unmarshal(content, &manifest); err != nil { + return nil, nil, nil, fmt.Errorf("failed to unmarshal %s: %s: %w", desc.Digest.String(), desc.MediaType, err) + } + subject = manifest.Subject + items = manifest.Blobs + } + return +} diff --git a/internal/platform/platform.go b/internal/platform/platform.go index 38d8d47f..232d6d4f 100644 --- a/internal/platform/platform.go +++ b/internal/platform/platform.go @@ -25,6 +25,7 @@ import ( "oras.land/oras-go/v2/content" "oras.land/oras-go/v2/errdef" "oras.land/oras-go/v2/internal/docker" + "oras.land/oras-go/v2/internal/manifestutil" ) // Match checks whether the current platform matches the target platform. @@ -77,7 +78,7 @@ func isSubset(a, b []string) bool { func SelectManifest(ctx context.Context, src content.ReadOnlyStorage, root ocispec.Descriptor, p *ocispec.Platform) (ocispec.Descriptor, error) { switch root.MediaType { case docker.MediaTypeManifestList, ocispec.MediaTypeImageIndex: - manifests, err := content.Successors(ctx, src, root) + _, _, manifests, err := manifestutil.Parse(ctx, src, root) if err != nil { return ocispec.Descriptor{}, err } @@ -90,7 +91,7 @@ func SelectManifest(ctx context.Context, src content.ReadOnlyStorage, root ocisp } return ocispec.Descriptor{}, fmt.Errorf("%s: %w: no matching manifest was found in the manifest list", root.Digest, errdef.ErrNotFound) case docker.MediaTypeManifest, ocispec.MediaTypeImageManifest: - descs, err := content.Successors(ctx, src, root) + _, config, _, err := manifestutil.Parse(ctx, src, root) if err != nil { return ocispec.Descriptor{}, err } @@ -99,8 +100,7 @@ func SelectManifest(ctx context.Context, src content.ReadOnlyStorage, root ocisp if root.MediaType == ocispec.MediaTypeImageManifest { configMediaType = ocispec.MediaTypeImageConfig } - - cfgPlatform, err := getPlatformFromConfig(ctx, src, descs[0], configMediaType) + cfgPlatform, err := getPlatformFromConfig(ctx, src, *config, configMediaType) if err != nil { return ocispec.Descriptor{}, err } From 872c9e141403500c0bea39233092aae3370cd98c Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Thu, 6 Jul 2023 15:52:56 +0800 Subject: [PATCH 03/14] move function and add TODOs Signed-off-by: Lixia (Sylvia) Lei --- content/graph.go | 109 ++++++++++++++++++++--------- copy_test.go | 2 + internal/manifestutil/parse.go | 93 ------------------------ internal/platform/platform.go | 7 +- internal/platform/platform_test.go | 1 + 5 files changed, 82 insertions(+), 130 deletions(-) delete mode 100644 internal/manifestutil/parse.go diff --git a/content/graph.go b/content/graph.go index f81bb09d..3cdbf6fe 100644 --- a/content/graph.go +++ b/content/graph.go @@ -25,6 +25,8 @@ import ( "oras.land/oras-go/v2/internal/spec" ) +// TODO: import cycle + // PredecessorFinder finds out the nodes directly pointing to a given node of a // directed acyclic graph. // In other words, returns the "parents" of the current descriptor. @@ -51,73 +53,114 @@ type ReadOnlyGraphStorage interface { func Successors(ctx context.Context, fetcher Fetcher, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { switch node.MediaType { case docker.MediaTypeManifest: - content, err := FetchAll(ctx, fetcher, node) + _, config, layers, err := SuccessorsParts(ctx, fetcher, node) + if err != nil { + return nil, err + } + return append([]ocispec.Descriptor{*config}, layers...), nil + case ocispec.MediaTypeImageManifest: + subject, config, layers, err := SuccessorsParts(ctx, fetcher, node) + if err != nil { + return nil, err + } + var nodes []ocispec.Descriptor + if subject != nil { + nodes = append(nodes, *subject) + } + nodes = append(nodes, *config) + return append(nodes, layers...), nil + case docker.MediaTypeManifestList: + _, _, manifests, err := SuccessorsParts(ctx, fetcher, node) + if err != nil { + return nil, err + } + return manifests, nil + case ocispec.MediaTypeImageIndex: + subject, _, manifests, err := SuccessorsParts(ctx, fetcher, node) + if err != nil { + return nil, err + } + var nodes []ocispec.Descriptor + if subject != nil { + nodes = append(nodes, *subject) + } + return append(nodes, manifests...), nil + case spec.MediaTypeArtifactManifest: + subject, _, blobs, err := SuccessorsParts(ctx, fetcher, node) if err != nil { return nil, err } + var nodes []ocispec.Descriptor + if subject != nil { + nodes = append(nodes, *subject) + } + return append(nodes, blobs...), nil + } + return nil, nil +} + +func SuccessorsParts(ctx context.Context, fetcher Fetcher, desc ocispec.Descriptor) (subject, config *ocispec.Descriptor, items []ocispec.Descriptor, err error) { + switch desc.MediaType { + case docker.MediaTypeManifest: + content, err := FetchAll(ctx, fetcher, desc) + if err != nil { + return nil, nil, nil, err + } // OCI manifest schema can be used to marshal docker manifest var manifest ocispec.Manifest if err := json.Unmarshal(content, &manifest); err != nil { - return nil, fmt.Errorf("failed to unmarshal %s: %s: %w", node.Digest.String(), node.MediaType, err) + return nil, nil, nil, fmt.Errorf("failed to unmarshal %s: %s: %w", desc.Digest.String(), desc.MediaType, err) } - return append([]ocispec.Descriptor{manifest.Config}, manifest.Layers...), nil + config = &manifest.Config + items = manifest.Layers case ocispec.MediaTypeImageManifest: - content, err := FetchAll(ctx, fetcher, node) + content, err := FetchAll(ctx, fetcher, desc) if err != nil { - return nil, err + return nil, nil, nil, err } var manifest ocispec.Manifest if err := json.Unmarshal(content, &manifest); err != nil { - return nil, fmt.Errorf("failed to unmarshal %s: %s: %w", node.Digest.String(), node.MediaType, err) - } - var nodes []ocispec.Descriptor - if manifest.Subject != nil { - nodes = append(nodes, *manifest.Subject) + return nil, nil, nil, fmt.Errorf("failed to unmarshal %s: %s: %w", desc.Digest.String(), desc.MediaType, err) } - nodes = append(nodes, manifest.Config) - return append(nodes, manifest.Layers...), nil + subject = manifest.Subject + config = &manifest.Config + items = manifest.Layers case docker.MediaTypeManifestList: - content, err := FetchAll(ctx, fetcher, node) + content, err := FetchAll(ctx, fetcher, desc) if err != nil { - return nil, err + return nil, nil, nil, err } // OCI Index schema can be used to marshal docker manifest list var index ocispec.Index if err := json.Unmarshal(content, &index); err != nil { - return nil, fmt.Errorf("failed to unmarshal %s: %s: %w", node.Digest.String(), node.MediaType, err) + return nil, nil, nil, fmt.Errorf("failed to unmarshal %s: %s: %w", desc.Digest.String(), desc.MediaType, err) } - return index.Manifests, nil + items = index.Manifests case ocispec.MediaTypeImageIndex: - content, err := FetchAll(ctx, fetcher, node) + content, err := FetchAll(ctx, fetcher, desc) if err != nil { - return nil, err + return nil, nil, nil, err } var index ocispec.Index if err := json.Unmarshal(content, &index); err != nil { - return nil, fmt.Errorf("failed to unmarshal %s: %s: %w", node.Digest.String(), node.MediaType, err) + return nil, nil, nil, fmt.Errorf("failed to unmarshal %s: %s: %w", desc.Digest.String(), desc.MediaType, err) } - var manifests []ocispec.Descriptor - if index.Subject != nil { - manifests = append(manifests, *index.Subject) - } - return append(manifests, index.Manifests...), nil + subject = index.Subject + items = index.Manifests case spec.MediaTypeArtifactManifest: - content, err := FetchAll(ctx, fetcher, node) + content, err := FetchAll(ctx, fetcher, desc) if err != nil { - return nil, err + return nil, nil, nil, err } var manifest spec.Artifact if err := json.Unmarshal(content, &manifest); err != nil { - return nil, fmt.Errorf("failed to unmarshal %s: %s: %w", node.Digest.String(), node.MediaType, err) + return nil, nil, nil, fmt.Errorf("failed to unmarshal %s: %s: %w", desc.Digest.String(), desc.MediaType, err) } - var nodes []ocispec.Descriptor - if manifest.Subject != nil { - nodes = append(nodes, *manifest.Subject) - } - return append(nodes, manifest.Blobs...), nil + subject = manifest.Subject + items = manifest.Blobs } - return nil, nil + return } diff --git a/copy_test.go b/copy_test.go index b1031bc6..093b01db 100644 --- a/copy_test.go +++ b/copy_test.go @@ -39,6 +39,8 @@ import ( "oras.land/oras-go/v2/internal/spec" ) +// TODO: test subject + // storageTracker tracks storage API counts. type storageTracker struct { content.Storage diff --git a/internal/manifestutil/parse.go b/internal/manifestutil/parse.go deleted file mode 100644 index a7c69e5a..00000000 --- a/internal/manifestutil/parse.go +++ /dev/null @@ -1,93 +0,0 @@ -/* -Copyright The ORAS Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package manifestutil - -import ( - "context" - "encoding/json" - "fmt" - - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "oras.land/oras-go/v2/content" - "oras.land/oras-go/v2/internal/docker" - "oras.land/oras-go/v2/internal/spec" -) - -func Parse(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) (subject, config *ocispec.Descriptor, items []ocispec.Descriptor, err error) { - switch desc.MediaType { - case docker.MediaTypeManifest: - content, err := content.FetchAll(ctx, fetcher, desc) - if err != nil { - return nil, nil, nil, err - } - // OCI manifest schema can be used to marshal docker manifest - var manifest ocispec.Manifest - if err := json.Unmarshal(content, &manifest); err != nil { - return nil, nil, nil, fmt.Errorf("failed to unmarshal %s: %s: %w", desc.Digest.String(), desc.MediaType, err) - } - config = &manifest.Config - items = manifest.Layers - case ocispec.MediaTypeImageManifest: - content, err := content.FetchAll(ctx, fetcher, desc) - if err != nil { - return nil, nil, nil, err - } - var manifest ocispec.Manifest - if err := json.Unmarshal(content, &manifest); err != nil { - return nil, nil, nil, fmt.Errorf("failed to unmarshal %s: %s: %w", desc.Digest.String(), desc.MediaType, err) - } - subject = manifest.Subject - config = &manifest.Config - items = manifest.Layers - case docker.MediaTypeManifestList: - content, err := content.FetchAll(ctx, fetcher, desc) - if err != nil { - return nil, nil, nil, err - } - - // OCI Index schema can be used to marshal docker manifest list - var index ocispec.Index - if err := json.Unmarshal(content, &index); err != nil { - return nil, nil, nil, fmt.Errorf("failed to unmarshal %s: %s: %w", desc.Digest.String(), desc.MediaType, err) - } - items = index.Manifests - case ocispec.MediaTypeImageIndex: - content, err := content.FetchAll(ctx, fetcher, desc) - if err != nil { - return nil, nil, nil, err - } - - var index ocispec.Index - if err := json.Unmarshal(content, &index); err != nil { - return nil, nil, nil, fmt.Errorf("failed to unmarshal %s: %s: %w", desc.Digest.String(), desc.MediaType, err) - } - subject = index.Subject - items = index.Manifests - case spec.MediaTypeArtifactManifest: - content, err := content.FetchAll(ctx, fetcher, desc) - if err != nil { - return nil, nil, nil, err - } - - var manifest spec.Artifact - if err := json.Unmarshal(content, &manifest); err != nil { - return nil, nil, nil, fmt.Errorf("failed to unmarshal %s: %s: %w", desc.Digest.String(), desc.MediaType, err) - } - subject = manifest.Subject - items = manifest.Blobs - } - return -} diff --git a/internal/platform/platform.go b/internal/platform/platform.go index 232d6d4f..6f7c3e46 100644 --- a/internal/platform/platform.go +++ b/internal/platform/platform.go @@ -25,7 +25,6 @@ import ( "oras.land/oras-go/v2/content" "oras.land/oras-go/v2/errdef" "oras.land/oras-go/v2/internal/docker" - "oras.land/oras-go/v2/internal/manifestutil" ) // Match checks whether the current platform matches the target platform. @@ -36,7 +35,7 @@ import ( // array of the current platform. // // Note: Variant, OSVersion and OSFeatures are optional fields, will skip -// the comparison if the target platform does not provide specfic value. +// the comparison if the target platform does not provide specific value. func Match(got *ocispec.Platform, want *ocispec.Platform) bool { if got.Architecture != want.Architecture || got.OS != want.OS { return false @@ -78,7 +77,7 @@ func isSubset(a, b []string) bool { func SelectManifest(ctx context.Context, src content.ReadOnlyStorage, root ocispec.Descriptor, p *ocispec.Platform) (ocispec.Descriptor, error) { switch root.MediaType { case docker.MediaTypeManifestList, ocispec.MediaTypeImageIndex: - _, _, manifests, err := manifestutil.Parse(ctx, src, root) + _, _, manifests, err := content.SuccessorsParts(ctx, src, root) if err != nil { return ocispec.Descriptor{}, err } @@ -91,7 +90,7 @@ func SelectManifest(ctx context.Context, src content.ReadOnlyStorage, root ocisp } return ocispec.Descriptor{}, fmt.Errorf("%s: %w: no matching manifest was found in the manifest list", root.Digest, errdef.ErrNotFound) case docker.MediaTypeManifest, ocispec.MediaTypeImageManifest: - _, config, _, err := manifestutil.Parse(ctx, src, root) + _, config, _, err := content.SuccessorsParts(ctx, src, root) if err != nil { return ocispec.Descriptor{}, err } diff --git a/internal/platform/platform_test.go b/internal/platform/platform_test.go index a19e0b37..1868fa86 100644 --- a/internal/platform/platform_test.go +++ b/internal/platform/platform_test.go @@ -111,6 +111,7 @@ func TestMatch(t *testing.T) { } } +// TODO: test manifest with subject func TestSelectManifest(t *testing.T) { storage := cas.NewMemory() arc_1 := "test-arc-1" From e277cb694ca692cd1b12403d23eed63f905f6b70 Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Fri, 7 Jul 2023 11:47:16 +0800 Subject: [PATCH 04/14] test platform Signed-off-by: Lixia (Sylvia) Lei --- content/graph.go | 2 +- internal/platform/platform_test.go | 91 +++++++++++++++++------------- 2 files changed, 52 insertions(+), 41 deletions(-) diff --git a/content/graph.go b/content/graph.go index 3cdbf6fe..df1c8c17 100644 --- a/content/graph.go +++ b/content/graph.go @@ -25,7 +25,7 @@ import ( "oras.land/oras-go/v2/internal/spec" ) -// TODO: import cycle +// TODO: unit tests // PredecessorFinder finds out the nodes directly pointing to a given node of a // directed acyclic graph. diff --git a/internal/platform/platform_test.go b/internal/platform/platform_test.go index 1868fa86..fd4589c7 100644 --- a/internal/platform/platform_test.go +++ b/internal/platform/platform_test.go @@ -111,7 +111,6 @@ func TestMatch(t *testing.T) { } } -// TODO: test manifest with subject func TestSelectManifest(t *testing.T) { storage := cas.NewMemory() arc_1 := "test-arc-1" @@ -145,10 +144,11 @@ func TestSelectManifest(t *testing.T) { }, }) } - generateManifest := func(arc, os, variant string, config ocispec.Descriptor, layers ...ocispec.Descriptor) { + generateManifest := func(arc, os, variant string, subject, config ocispec.Descriptor, layers ...ocispec.Descriptor) { manifest := ocispec.Manifest{ - Config: config, - Layers: layers, + Subject: &subject, + Config: config, + Layers: layers, } manifestJSON, err := json.Marshal(manifest) if err != nil { @@ -156,8 +156,9 @@ func TestSelectManifest(t *testing.T) { } appendManifest(arc, os, variant, ocispec.MediaTypeImageManifest, manifestJSON) } - generateIndex := func(manifests ...ocispec.Descriptor) { + generateIndex := func(subject ocispec.Descriptor, manifests ...ocispec.Descriptor) { index := ocispec.Index{ + Subject: &subject, Manifests: manifests, } indexJSON, err := json.Marshal(index) @@ -167,20 +168,21 @@ func TestSelectManifest(t *testing.T) { appendBlob(ocispec.MediaTypeImageIndex, indexJSON) } + appendBlob("test/subject", []byte("dummy subject")) // Blob 0 appendBlob(ocispec.MediaTypeImageConfig, []byte(`{"mediaType":"application/vnd.oci.image.config.v1+json", "created":"2022-07-29T08:13:55Z", "author":"test author", "architecture":"test-arc-1", "os":"test-os-1", -"variant":"v1"}`)) // Blob 0 - appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 1 - appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 2 - generateManifest(arc_1, os_1, variant_1, descs[0], descs[1:3]...) // Blob 3 - appendBlob(ocispec.MediaTypeImageLayer, []byte("hello1")) // Blob 4 - generateManifest(arc_2, os_2, variant_1, descs[0], descs[4]) // Blob 5 - appendBlob(ocispec.MediaTypeImageLayer, []byte("hello2")) // Blob 6 - generateManifest(arc_1, os_1, variant_2, descs[0], descs[6]) // Blob 7 - generateIndex(descs[3], descs[5], descs[7]) // Blob 8 +"variant":"v1"}`)) // Blob 1 + appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 2 + appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 3 + generateManifest(arc_1, os_1, variant_1, descs[0], descs[1], descs[2:4]...) // Blob 4 + appendBlob(ocispec.MediaTypeImageLayer, []byte("hello1")) // Blob 5 + generateManifest(arc_2, os_2, variant_1, descs[0], descs[1], descs[5]) // Blob 6 + appendBlob(ocispec.MediaTypeImageLayer, []byte("hello2")) // Blob 7 + generateManifest(arc_1, os_1, variant_2, descs[0], descs[1], descs[7]) // Blob 8 + generateIndex(descs[0], descs[4], descs[6], descs[8]) // Blob 9 ctx := context.Background() for i := range blobs { @@ -191,12 +193,12 @@ func TestSelectManifest(t *testing.T) { } // test SelectManifest on image index, only one matching manifest found - root := descs[8] + root := descs[9] targetPlatform := ocispec.Platform{ Architecture: arc_2, OS: os_2, } - wantDesc := descs[5] + wantDesc := descs[6] gotDesc, err := SelectManifest(ctx, storage, root, &targetPlatform) if err != nil { t.Fatalf("SelectManifest() error = %v, wantErr %v", err, false) @@ -212,7 +214,7 @@ func TestSelectManifest(t *testing.T) { Architecture: arc_1, OS: os_1, } - wantDesc = descs[3] + wantDesc = descs[4] gotDesc, err = SelectManifest(ctx, storage, root, &targetPlatform) if err != nil { t.Fatalf("SelectManifest() error = %v, wantErr %v", err, false) @@ -222,12 +224,12 @@ func TestSelectManifest(t *testing.T) { } // test SelectManifest on manifest - root = descs[7] + root = descs[8] targetPlatform = ocispec.Platform{ Architecture: arc_1, OS: os_1, } - wantDesc = descs[7] + wantDesc = descs[8] gotDesc, err = SelectManifest(ctx, storage, root, &targetPlatform) if err != nil { t.Fatalf("SelectManifest() error = %v, wantErr %v", err, false) @@ -238,7 +240,7 @@ func TestSelectManifest(t *testing.T) { // test SelectManifest on manifest, but there is no matching node. // Should return not found error. - root = descs[7] + root = descs[8] targetPlatform = ocispec.Platform{ Architecture: arc_1, OS: os_1, @@ -256,24 +258,26 @@ func TestSelectManifest(t *testing.T) { Architecture: arc_1, OS: os_1, } - root = descs[1] + root = descs[2] _, err = SelectManifest(ctx, storage, root, &targetPlatform) if !errors.Is(err, errdef.ErrUnsupported) { t.Fatalf("SelectManifest() error = %v, wantErr %v", err, errdef.ErrUnsupported) } // generate incorrect test content + storage = cas.NewMemory() blobs = nil descs = nil + appendBlob("test/subject", []byte("dummy subject")) // Blob 0 appendBlob(docker.MediaTypeConfig, []byte(`{"mediaType":"application/vnd.oci.image.config.v1+json", -"created":"2022-07-29T08:13:55Z", -"author":"test author 1", -"architecture":"test-arc-1", -"os":"test-os-1", -"variant":"v1"}`)) // Blob 0 - appendBlob(ocispec.MediaTypeImageLayer, []byte("foo1")) // Blob 1 - generateManifest(arc_1, os_1, variant_1, descs[0], descs[1]) // Blob 2 - generateIndex(descs[2]) // Blob 3 + "created":"2022-07-29T08:13:55Z", + "author":"test author 1", + "architecture":"test-arc-1", + "os":"test-os-1", + "variant":"v1"}`)) // Blob 1 + appendBlob(ocispec.MediaTypeImageLayer, []byte("foo1")) // Blob 2 + generateManifest(arc_1, os_1, variant_1, descs[0], descs[1], descs[2]) // Blob 3 + generateIndex(descs[3]) // Blob 4 ctx = context.Background() for i := range blobs { @@ -286,7 +290,7 @@ func TestSelectManifest(t *testing.T) { // test SelectManifest on manifest, but the manifest is // invalid by having docker mediaType config in the manifest and oci // mediaType in the image config. Should return error. - root = descs[2] + root = descs[3] targetPlatform = ocispec.Platform{ Architecture: arc_1, OS: os_1, @@ -298,12 +302,14 @@ func TestSelectManifest(t *testing.T) { } // generate test content with null config blob + storage = cas.NewMemory() blobs = nil descs = nil - appendBlob(ocispec.MediaTypeImageConfig, []byte("null")) // Blob 0 - appendBlob(ocispec.MediaTypeImageLayer, []byte("foo2")) // Blob 1 - generateManifest(arc_1, os_1, variant_1, descs[0], descs[1]) // Blob 2 - generateIndex(descs[2]) // Blob 3 + appendBlob("test/subject", []byte("dummy subject")) // Blob 0 + appendBlob(ocispec.MediaTypeImageConfig, []byte("null")) // Blob 1 + appendBlob(ocispec.MediaTypeImageLayer, []byte("foo2")) // Blob 2 + generateManifest(arc_1, os_1, variant_1, descs[0], descs[1], descs[2]) // Blob 3 + generateIndex(descs[3]) // Blob 4 ctx = context.Background() for i := range blobs { @@ -315,7 +321,7 @@ func TestSelectManifest(t *testing.T) { // test SelectManifest on manifest with null config blob, // should return not found error. - root = descs[2] + root = descs[3] targetPlatform = ocispec.Platform{ Architecture: arc_1, OS: os_1, @@ -327,12 +333,14 @@ func TestSelectManifest(t *testing.T) { } // generate test content with empty config blob + storage = cas.NewMemory() blobs = nil descs = nil - appendBlob(ocispec.MediaTypeImageConfig, []byte("")) // Blob 0 - appendBlob(ocispec.MediaTypeImageLayer, []byte("foo3")) // Blob 1 - generateManifest(arc_1, os_1, variant_1, descs[0], descs[1]) // Blob 2 - generateIndex(descs[2]) // Blob 3 + appendBlob("test/subject", []byte("dummy subject")) // Blob 0 + appendBlob(ocispec.MediaTypeImageConfig, []byte("")) // Blob 1 + appendBlob(ocispec.MediaTypeImageLayer, []byte("foo3")) // Blob 2 + generateManifest(arc_1, os_1, variant_1, descs[0], descs[1], descs[2]) // Blob 3 + generateIndex(descs[3]) // Blob 4 ctx = context.Background() for i := range blobs { @@ -344,13 +352,16 @@ func TestSelectManifest(t *testing.T) { // test SelectManifest on manifest with empty config blob // should return not found error - root = descs[2] + root = descs[3] + targetPlatform = ocispec.Platform{ Architecture: arc_1, OS: os_1, } + _, err = SelectManifest(ctx, storage, root, &targetPlatform) expected = fmt.Sprintf("%s: %v: platform in manifest does not match target platform", root.Digest, errdef.ErrNotFound) + if err.Error() != expected { t.Fatalf("SelectManifest() error = %v, wantErr %v", err, expected) } From 69a33e2e112fb9af6f1ac216f615433cb5ba9313 Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Fri, 7 Jul 2023 11:51:21 +0800 Subject: [PATCH 05/14] improve platform test Signed-off-by: Lixia (Sylvia) Lei --- internal/platform/platform_test.go | 50 +++++++++++++++--------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/internal/platform/platform_test.go b/internal/platform/platform_test.go index fd4589c7..621ce959 100644 --- a/internal/platform/platform_test.go +++ b/internal/platform/platform_test.go @@ -144,9 +144,9 @@ func TestSelectManifest(t *testing.T) { }, }) } - generateManifest := func(arc, os, variant string, subject, config ocispec.Descriptor, layers ...ocispec.Descriptor) { + generateManifest := func(arc, os, variant string, subject *ocispec.Descriptor, config ocispec.Descriptor, layers ...ocispec.Descriptor) { manifest := ocispec.Manifest{ - Subject: &subject, + Subject: subject, Config: config, Layers: layers, } @@ -156,9 +156,9 @@ func TestSelectManifest(t *testing.T) { } appendManifest(arc, os, variant, ocispec.MediaTypeImageManifest, manifestJSON) } - generateIndex := func(subject ocispec.Descriptor, manifests ...ocispec.Descriptor) { + generateIndex := func(subject *ocispec.Descriptor, manifests ...ocispec.Descriptor) { index := ocispec.Index{ - Subject: &subject, + Subject: subject, Manifests: manifests, } indexJSON, err := json.Marshal(index) @@ -175,14 +175,14 @@ func TestSelectManifest(t *testing.T) { "architecture":"test-arc-1", "os":"test-os-1", "variant":"v1"}`)) // Blob 1 - appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 2 - appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 3 - generateManifest(arc_1, os_1, variant_1, descs[0], descs[1], descs[2:4]...) // Blob 4 - appendBlob(ocispec.MediaTypeImageLayer, []byte("hello1")) // Blob 5 - generateManifest(arc_2, os_2, variant_1, descs[0], descs[1], descs[5]) // Blob 6 - appendBlob(ocispec.MediaTypeImageLayer, []byte("hello2")) // Blob 7 - generateManifest(arc_1, os_1, variant_2, descs[0], descs[1], descs[7]) // Blob 8 - generateIndex(descs[0], descs[4], descs[6], descs[8]) // Blob 9 + appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 2 + appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 3 + generateManifest(arc_1, os_1, variant_1, &descs[0], descs[1], descs[2:4]...) // Blob 4 + appendBlob(ocispec.MediaTypeImageLayer, []byte("hello1")) // Blob 5 + generateManifest(arc_2, os_2, variant_1, nil, descs[1], descs[5]) // Blob 6 + appendBlob(ocispec.MediaTypeImageLayer, []byte("hello2")) // Blob 7 + generateManifest(arc_1, os_1, variant_2, nil, descs[1], descs[7]) // Blob 8 + generateIndex(&descs[0], descs[4], descs[6], descs[8]) // Blob 9 ctx := context.Background() for i := range blobs { @@ -275,9 +275,9 @@ func TestSelectManifest(t *testing.T) { "architecture":"test-arc-1", "os":"test-os-1", "variant":"v1"}`)) // Blob 1 - appendBlob(ocispec.MediaTypeImageLayer, []byte("foo1")) // Blob 2 - generateManifest(arc_1, os_1, variant_1, descs[0], descs[1], descs[2]) // Blob 3 - generateIndex(descs[3]) // Blob 4 + appendBlob(ocispec.MediaTypeImageLayer, []byte("foo1")) // Blob 2 + generateManifest(arc_1, os_1, variant_1, &descs[0], descs[1], descs[2]) // Blob 3 + generateIndex(&descs[0], descs[3]) // Blob 4 ctx = context.Background() for i := range blobs { @@ -305,11 +305,11 @@ func TestSelectManifest(t *testing.T) { storage = cas.NewMemory() blobs = nil descs = nil - appendBlob("test/subject", []byte("dummy subject")) // Blob 0 - appendBlob(ocispec.MediaTypeImageConfig, []byte("null")) // Blob 1 - appendBlob(ocispec.MediaTypeImageLayer, []byte("foo2")) // Blob 2 - generateManifest(arc_1, os_1, variant_1, descs[0], descs[1], descs[2]) // Blob 3 - generateIndex(descs[3]) // Blob 4 + appendBlob("test/subject", []byte("dummy subject")) // Blob 0 + appendBlob(ocispec.MediaTypeImageConfig, []byte("null")) // Blob 1 + appendBlob(ocispec.MediaTypeImageLayer, []byte("foo2")) // Blob 2 + generateManifest(arc_1, os_1, variant_1, &descs[0], descs[1], descs[2]) // Blob 3 + generateIndex(nil, descs[3]) // Blob 4 ctx = context.Background() for i := range blobs { @@ -336,11 +336,11 @@ func TestSelectManifest(t *testing.T) { storage = cas.NewMemory() blobs = nil descs = nil - appendBlob("test/subject", []byte("dummy subject")) // Blob 0 - appendBlob(ocispec.MediaTypeImageConfig, []byte("")) // Blob 1 - appendBlob(ocispec.MediaTypeImageLayer, []byte("foo3")) // Blob 2 - generateManifest(arc_1, os_1, variant_1, descs[0], descs[1], descs[2]) // Blob 3 - generateIndex(descs[3]) // Blob 4 + appendBlob("test/subject", []byte("dummy subject")) // Blob 0 + appendBlob(ocispec.MediaTypeImageConfig, []byte("")) // Blob 1 + appendBlob(ocispec.MediaTypeImageLayer, []byte("foo3")) // Blob 2 + generateManifest(arc_1, os_1, variant_1, nil, descs[1], descs[2]) // Blob 3 + generateIndex(&descs[0], descs[3]) // Blob 4 ctx = context.Background() for i := range blobs { From 1062ed7b4185e6ef7e46bc25228d26a5d1238336 Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Fri, 7 Jul 2023 15:23:11 +0800 Subject: [PATCH 06/14] test successors Signed-off-by: Lixia (Sylvia) Lei --- content/graph.go | 2 - content/graph_test.go | 360 ++++++++++++++++++++++++++++++++++++++++++ copy_test.go | 2 - 3 files changed, 360 insertions(+), 4 deletions(-) create mode 100644 content/graph_test.go diff --git a/content/graph.go b/content/graph.go index df1c8c17..3f10ecdc 100644 --- a/content/graph.go +++ b/content/graph.go @@ -25,8 +25,6 @@ import ( "oras.land/oras-go/v2/internal/spec" ) -// TODO: unit tests - // PredecessorFinder finds out the nodes directly pointing to a given node of a // directed acyclic graph. // In other words, returns the "parents" of the current descriptor. diff --git a/content/graph_test.go b/content/graph_test.go new file mode 100644 index 00000000..0f6025f0 --- /dev/null +++ b/content/graph_test.go @@ -0,0 +1,360 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package content_test + +import ( + "bytes" + "context" + "encoding/json" + "reflect" + "testing" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/content" + "oras.land/oras-go/v2/internal/cas" + "oras.land/oras-go/v2/internal/docker" + "oras.land/oras-go/v2/internal/spec" +) + +func TestSuccessors_dockerManifest(t *testing.T) { + storage := cas.NewMemory() + + // generate test content + var blobs [][]byte + var descs []ocispec.Descriptor + appendBlob := func(mediaType string, blob []byte) { + blobs = append(blobs, blob) + descs = append(descs, ocispec.Descriptor{ + MediaType: mediaType, + Digest: digest.FromBytes(blob), + Size: int64(len(blob)), + }) + } + generateManifest := func(config ocispec.Descriptor, layers ...ocispec.Descriptor) { + manifest := ocispec.Manifest{ + Config: config, + Layers: layers, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + appendBlob(docker.MediaTypeManifest, manifestJSON) + } + + appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0 + appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 1 + appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 2 + appendBlob(ocispec.MediaTypeImageLayer, []byte("hello")) // Blob 3 + generateManifest(descs[0], descs[1:4]...) // Blob 4 + + ctx := context.Background() + for i := range blobs { + err := storage.Push(ctx, descs[i], bytes.NewReader(blobs[i])) + if err != nil { + t.Fatalf("failed to push test content to src: %d: %v", i, err) + } + } + + // test Successors + manifestDesc := descs[4] + got, err := content.Successors(ctx, storage, manifestDesc) + if err != nil { + t.Fatal("Successors() error =", err) + } + if want := descs[0:4]; !equalDescriptorSet(got, want) { + t.Errorf("Successors() = %v, want %v", got, want) + } +} + +func TestSuccessors_imageManifest(t *testing.T) { + storage := cas.NewMemory() + + // generate test content + var blobs [][]byte + var descs []ocispec.Descriptor + appendBlob := func(mediaType string, blob []byte) { + blobs = append(blobs, blob) + descs = append(descs, ocispec.Descriptor{ + MediaType: mediaType, + Digest: digest.FromBytes(blob), + Size: int64(len(blob)), + }) + } + generateManifest := func(subject *ocispec.Descriptor, config ocispec.Descriptor, layers ...ocispec.Descriptor) { + manifest := ocispec.Manifest{ + Subject: subject, + Config: config, + Layers: layers, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + appendBlob(ocispec.MediaTypeImageManifest, manifestJSON) + } + + appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0 + appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 1 + appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 2 + appendBlob(ocispec.MediaTypeImageLayer, []byte("hello")) // Blob 3 + generateManifest(nil, descs[0], descs[1:4]...) // Blob 4 + appendBlob(ocispec.MediaTypeImageConfig, []byte("{}")) // Blob 5 + appendBlob("test/sig", []byte("sig")) // Blob 6 + generateManifest(&descs[4], descs[5], descs[6]) // Blob 7 + + ctx := context.Background() + for i := range blobs { + err := storage.Push(ctx, descs[i], bytes.NewReader(blobs[i])) + if err != nil { + t.Fatalf("failed to push test content to src: %d: %v", i, err) + } + } + + // test Successors: image manifest without a subject + manifestDesc := descs[4] + got, err := content.Successors(ctx, storage, manifestDesc) + if err != nil { + t.Fatal("Successors() error =", err) + } + if want := descs[0:4]; !equalDescriptorSet(got, want) { + t.Errorf("Successors() = %v, want %v", got, want) + } + + // test Successors: image manifest with a subject + manifestDesc = descs[7] + got, err = content.Successors(ctx, storage, manifestDesc) + if err != nil { + t.Fatal("Successors() error =", err) + } + if want := descs[4:7]; !equalDescriptorSet(got, want) { + t.Errorf("Successors() = %v, want %v", got, want) + } +} + +func TestSuccessors_dockerManifestList(t *testing.T) { + storage := cas.NewMemory() + + // generate test content + var blobs [][]byte + var descs []ocispec.Descriptor + appendBlob := func(mediaType string, blob []byte) { + blobs = append(blobs, blob) + descs = append(descs, ocispec.Descriptor{ + MediaType: mediaType, + Digest: digest.FromBytes(blob), + Size: int64(len(blob)), + }) + } + generateManifest := func(config ocispec.Descriptor, layers ...ocispec.Descriptor) { + manifest := ocispec.Manifest{ + Config: config, + Layers: layers, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + appendBlob(docker.MediaTypeManifest, manifestJSON) + } + generateIndex := func(manifests ...ocispec.Descriptor) { + index := ocispec.Index{ + Manifests: manifests, + } + indexJSON, err := json.Marshal(index) + if err != nil { + t.Fatal(err) + } + appendBlob(docker.MediaTypeManifestList, indexJSON) + } + + appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0 + appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 1 + appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 2 + appendBlob(ocispec.MediaTypeImageLayer, []byte("hello")) // Blob 3 + generateManifest(descs[0], descs[1:3]...) // Blob 4 + generateManifest(descs[0], descs[3]) // Blob 5 + generateIndex(descs[4:6]...) // Blob 6 + + ctx := context.Background() + for i := range blobs { + err := storage.Push(ctx, descs[i], bytes.NewReader(blobs[i])) + if err != nil { + t.Fatalf("failed to push test content to src: %d: %v", i, err) + } + } + + // test Successors + manifestDesc := descs[6] + got, err := content.Successors(ctx, storage, manifestDesc) + if err != nil { + t.Fatal("Successors() error =", err) + } + if want := descs[4:6]; !equalDescriptorSet(got, want) { + t.Errorf("Successors() = %v, want %v", got, want) + } +} + +func TestSuccessors_imageIndex(t *testing.T) { + storage := cas.NewMemory() + + // generate test content + var blobs [][]byte + var descs []ocispec.Descriptor + appendBlob := func(mediaType string, blob []byte) { + blobs = append(blobs, blob) + descs = append(descs, ocispec.Descriptor{ + MediaType: mediaType, + Digest: digest.FromBytes(blob), + Size: int64(len(blob)), + }) + } + generateManifest := func(subject *ocispec.Descriptor, config ocispec.Descriptor, layers ...ocispec.Descriptor) { + manifest := ocispec.Manifest{ + Subject: subject, + Config: config, + Layers: layers, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + appendBlob(ocispec.MediaTypeImageManifest, manifestJSON) + } + generateIndex := func(subject *ocispec.Descriptor, manifests ...ocispec.Descriptor) { + index := ocispec.Index{ + Subject: subject, + Manifests: manifests, + } + indexJSON, err := json.Marshal(index) + if err != nil { + t.Fatal(err) + } + appendBlob(ocispec.MediaTypeImageIndex, indexJSON) + } + + appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0 + appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 1 + appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 2 + appendBlob(ocispec.MediaTypeImageLayer, []byte("hello")) // Blob 3 + generateManifest(nil, descs[0], descs[1:3]...) // Blob 4 + generateManifest(nil, descs[0], descs[3]) // Blob 5 + appendBlob(ocispec.MediaTypeImageConfig, []byte("{}")) // Blob 6 + appendBlob("test/sig", []byte("sig")) // Blob 7 + generateManifest(&descs[4], descs[5], descs[6]) // Blob 8 + generateIndex(&descs[8], descs[4:6]...) // Blob 9 + + ctx := context.Background() + for i := range blobs { + err := storage.Push(ctx, descs[i], bytes.NewReader(blobs[i])) + if err != nil { + t.Fatalf("failed to push test content to src: %d: %v", i, err) + } + } + + // test Successors + manifestDesc := descs[9] + got, err := content.Successors(ctx, storage, manifestDesc) + if err != nil { + t.Fatal("Successors() error =", err) + } + if want := append([]ocispec.Descriptor{descs[8]}, descs[4:6]...); !equalDescriptorSet(got, want) { + t.Errorf("Successors() = %v, want %v", got, want) + } +} + +func TestSuccessors_artifactManifest(t *testing.T) { + storage := cas.NewMemory() + + // generate test content + var blobs [][]byte + var descs []ocispec.Descriptor + appendBlob := func(mediaType string, blob []byte) { + blobs = append(blobs, blob) + descs = append(descs, ocispec.Descriptor{ + MediaType: mediaType, + Digest: digest.FromBytes(blob), + Size: int64(len(blob)), + }) + } + generateArtifactManifest := func(subject *ocispec.Descriptor, blobs ...ocispec.Descriptor) { + manifest := spec.Artifact{ + Subject: subject, + Blobs: blobs, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + appendBlob(spec.MediaTypeArtifactManifest, manifestJSON) + } + + appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 0 + appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 1 + appendBlob(ocispec.MediaTypeImageLayer, []byte("hello")) // Blob 2 + generateArtifactManifest(nil, descs[0:3]...) // Blob 3 + appendBlob("test/sig", []byte("sig")) // Blob 4 + generateArtifactManifest(&descs[3], descs[4]) // Blob 5 + + ctx := context.Background() + for i := range blobs { + err := storage.Push(ctx, descs[i], bytes.NewReader(blobs[i])) + if err != nil { + t.Fatalf("failed to push test content to src: %d: %v", i, err) + } + } + + // test Successors: image manifest without a subject + manifestDesc := descs[3] + got, err := content.Successors(ctx, storage, manifestDesc) + if err != nil { + t.Fatal("Successors() error =", err) + } + if want := descs[0:3]; !equalDescriptorSet(got, want) { + t.Errorf("Successors() = %v, want %v", got, want) + } + + // test Successors: image manifest with a subject + manifestDesc = descs[5] + got, err = content.Successors(ctx, storage, manifestDesc) + if err != nil { + t.Fatal("Successors() error =", err) + } + if want := descs[3:5]; !equalDescriptorSet(got, want) { + t.Errorf("Successors() = %v, want %v", got, want) + } +} + +func equalDescriptorSet(actual []ocispec.Descriptor, expected []ocispec.Descriptor) bool { + if len(actual) != len(expected) { + return false + } + contains := func(node ocispec.Descriptor) bool { + for _, candidate := range actual { + if reflect.DeepEqual(candidate, node) { + return true + } + } + return false + } + for _, node := range expected { + if !contains(node) { + return false + } + } + return true +} diff --git a/copy_test.go b/copy_test.go index 093b01db..b1031bc6 100644 --- a/copy_test.go +++ b/copy_test.go @@ -39,8 +39,6 @@ import ( "oras.land/oras-go/v2/internal/spec" ) -// TODO: test subject - // storageTracker tracks storage API counts. type storageTracker struct { content.Storage From b8894292d45d88f820cbf572d328ad199523b673 Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Fri, 7 Jul 2023 16:02:48 +0800 Subject: [PATCH 07/14] add comments Signed-off-by: Lixia (Sylvia) Lei --- content/graph.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/content/graph.go b/content/graph.go index 3f10ecdc..7b93460e 100644 --- a/content/graph.go +++ b/content/graph.go @@ -47,7 +47,7 @@ type ReadOnlyGraphStorage interface { } // Successors returns the nodes directly pointed by the current node. -// In other words, returns the "children" of the current descriptor. +// In other words, it returns the "children" of the current descriptor. func Successors(ctx context.Context, fetcher Fetcher, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { switch node.MediaType { case docker.MediaTypeManifest: @@ -97,6 +97,13 @@ func Successors(ctx context.Context, fetcher Fetcher, node ocispec.Descriptor) ( return nil, nil } +// SuccessorsParts returns the parts of the nodes directly pointed by the +// current node, in following order: +// - Subject (If present for OCI Image Manifest and OCI Image Index) +// - Config (If present for OCI Image Manifest and Docker Manifest) +// - Layers (For OCI Image Manifest and Docker Manifest), or Blobs (For OCI +// Artifact Manifest), or Manifests (For OCI Image Index or Docker Manifest +// List) func SuccessorsParts(ctx context.Context, fetcher Fetcher, desc ocispec.Descriptor) (subject, config *ocispec.Descriptor, items []ocispec.Descriptor, err error) { switch desc.MediaType { case docker.MediaTypeManifest: From e6c7c687166d14148679551123cbe64ecd5bfc04 Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Fri, 7 Jul 2023 16:28:15 +0800 Subject: [PATCH 08/14] revert successors Signed-off-by: Lixia (Sylvia) Lei --- content/graph.go | 59 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/content/graph.go b/content/graph.go index 7b93460e..025175c9 100644 --- a/content/graph.go +++ b/content/graph.go @@ -51,48 +51,73 @@ type ReadOnlyGraphStorage interface { func Successors(ctx context.Context, fetcher Fetcher, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { switch node.MediaType { case docker.MediaTypeManifest: - _, config, layers, err := SuccessorsParts(ctx, fetcher, node) + content, err := FetchAll(ctx, fetcher, node) if err != nil { return nil, err } - return append([]ocispec.Descriptor{*config}, layers...), nil + // OCI manifest schema can be used to marshal docker manifest + var manifest ocispec.Manifest + if err := json.Unmarshal(content, &manifest); err != nil { + return nil, err + } + return append([]ocispec.Descriptor{manifest.Config}, manifest.Layers...), nil case ocispec.MediaTypeImageManifest: - subject, config, layers, err := SuccessorsParts(ctx, fetcher, node) + content, err := FetchAll(ctx, fetcher, node) if err != nil { return nil, err } + var manifest ocispec.Manifest + if err := json.Unmarshal(content, &manifest); err != nil { + return nil, err + } var nodes []ocispec.Descriptor - if subject != nil { - nodes = append(nodes, *subject) + if manifest.Subject != nil { + nodes = append(nodes, *manifest.Subject) } - nodes = append(nodes, *config) - return append(nodes, layers...), nil + nodes = append(nodes, manifest.Config) + return append(nodes, manifest.Layers...), nil case docker.MediaTypeManifestList: - _, _, manifests, err := SuccessorsParts(ctx, fetcher, node) + content, err := FetchAll(ctx, fetcher, node) if err != nil { return nil, err } - return manifests, nil + + // OCI manifest index schema can be used to marshal docker manifest list + var index ocispec.Index + if err := json.Unmarshal(content, &index); err != nil { + return nil, err + } + return index.Manifests, nil case ocispec.MediaTypeImageIndex: - subject, _, manifests, err := SuccessorsParts(ctx, fetcher, node) + content, err := FetchAll(ctx, fetcher, node) if err != nil { return nil, err } + + var index ocispec.Index + if err := json.Unmarshal(content, &index); err != nil { + return nil, err + } var nodes []ocispec.Descriptor - if subject != nil { - nodes = append(nodes, *subject) + if index.Subject != nil { + nodes = append(nodes, *index.Subject) } - return append(nodes, manifests...), nil + return append(nodes, index.Manifests...), nil case spec.MediaTypeArtifactManifest: - subject, _, blobs, err := SuccessorsParts(ctx, fetcher, node) + content, err := FetchAll(ctx, fetcher, node) if err != nil { return nil, err } + + var manifest spec.Artifact + if err := json.Unmarshal(content, &manifest); err != nil { + return nil, err + } var nodes []ocispec.Descriptor - if subject != nil { - nodes = append(nodes, *subject) + if manifest.Subject != nil { + nodes = append(nodes, *manifest.Subject) } - return append(nodes, blobs...), nil + return append(nodes, manifest.Blobs...), nil } return nil, nil } From 56b8c3af3e3b9731268c90fe66d8553b14223dad Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Fri, 7 Jul 2023 16:59:37 +0800 Subject: [PATCH 09/14] add parser Signed-off-by: Lixia (Sylvia) Lei --- internal/manifestutil/parser.go | 64 +++++++++++++++++++++++++++++++++ internal/platform/platform.go | 9 +++-- 2 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 internal/manifestutil/parser.go diff --git a/internal/manifestutil/parser.go b/internal/manifestutil/parser.go new file mode 100644 index 00000000..3c957ceb --- /dev/null +++ b/internal/manifestutil/parser.go @@ -0,0 +1,64 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manifestutil + +import ( + "context" + "encoding/json" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + "oras.land/oras-go/v2/content" + "oras.land/oras-go/v2/internal/docker" +) + +// Config returns the config of desc, if present. +func Config(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { + switch desc.MediaType { + case docker.MediaTypeManifest, ocispec.MediaTypeImageManifest: + content, err := content.FetchAll(ctx, fetcher, desc) + if err != nil { + return nil, err + } + // OCI manifest schema can be used to marshal docker manifest + var manifest ocispec.Manifest + if err := json.Unmarshal(content, &manifest); err != nil { + return nil, err + } + return &manifest.Config, nil + default: + return nil, nil + } +} + +// Manifest returns the manifests of desc, if present. +func Manifests(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + switch desc.MediaType { + case ocispec.MediaTypeImageIndex: + content, err := content.FetchAll(ctx, fetcher, desc) + if err != nil { + return nil, err + } + + var index ocispec.Index + if err := json.Unmarshal(content, &index); err != nil { + return nil, err + } + return index.Manifests, nil + default: + return nil, nil + } +} diff --git a/internal/platform/platform.go b/internal/platform/platform.go index 6f7c3e46..a84475f9 100644 --- a/internal/platform/platform.go +++ b/internal/platform/platform.go @@ -25,6 +25,7 @@ import ( "oras.land/oras-go/v2/content" "oras.land/oras-go/v2/errdef" "oras.land/oras-go/v2/internal/docker" + "oras.land/oras-go/v2/internal/manifestutil" ) // Match checks whether the current platform matches the target platform. @@ -77,7 +78,7 @@ func isSubset(a, b []string) bool { func SelectManifest(ctx context.Context, src content.ReadOnlyStorage, root ocispec.Descriptor, p *ocispec.Platform) (ocispec.Descriptor, error) { switch root.MediaType { case docker.MediaTypeManifestList, ocispec.MediaTypeImageIndex: - _, _, manifests, err := content.SuccessorsParts(ctx, src, root) + manifests, err := manifestutil.Manifests(ctx, src, root) if err != nil { return ocispec.Descriptor{}, err } @@ -90,10 +91,14 @@ func SelectManifest(ctx context.Context, src content.ReadOnlyStorage, root ocisp } return ocispec.Descriptor{}, fmt.Errorf("%s: %w: no matching manifest was found in the manifest list", root.Digest, errdef.ErrNotFound) case docker.MediaTypeManifest, ocispec.MediaTypeImageManifest: - _, config, _, err := content.SuccessorsParts(ctx, src, root) + config, err := manifestutil.Config(ctx, src, root) if err != nil { return ocispec.Descriptor{}, err } + if config == nil { + // should not happen + return ocispec.Descriptor{}, fmt.Errorf("%s: %s: %w", root.Digest, root.MediaType, errdef.ErrUnsupported) + } configMediaType := docker.MediaTypeConfig if root.MediaType == ocispec.MediaTypeImageManifest { From 35926d502125b48f8cacaf79773860c9b048f12f Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Fri, 7 Jul 2023 17:44:55 +0800 Subject: [PATCH 10/14] test extended copy Signed-off-by: Lixia (Sylvia) Lei --- extendedcopy_test.go | 120 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/extendedcopy_test.go b/extendedcopy_test.go index 6c9d7f3a..0d478005 100644 --- a/extendedcopy_test.go +++ b/extendedcopy_test.go @@ -41,6 +41,8 @@ import ( "oras.land/oras-go/v2/registry/remote" ) +// TODO: test index with subject + func TestExtendedCopy_FullCopy(t *testing.T) { src := memory.New() dst := memory.New() @@ -344,6 +346,124 @@ func TestExtendedCopyGraph_PartialCopy(t *testing.T) { } } +func TestExtendedCopyGraph_artifactIndex(t *testing.T) { + // generate test content + var blobs [][]byte + var descs []ocispec.Descriptor + appendBlob := func(mediaType string, blob []byte) { + blobs = append(blobs, blob) + descs = append(descs, ocispec.Descriptor{ + MediaType: mediaType, + Digest: digest.FromBytes(blob), + Size: int64(len(blob)), + }) + } + generateManifest := func(subject *ocispec.Descriptor, config ocispec.Descriptor, layers ...ocispec.Descriptor) { + manifest := ocispec.Manifest{ + Subject: subject, + Config: config, + Layers: layers, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + appendBlob(ocispec.MediaTypeImageManifest, manifestJSON) + } + generateIndex := func(subject *ocispec.Descriptor, manifests ...ocispec.Descriptor) { + index := ocispec.Index{ + Subject: subject, + Manifests: manifests, + } + indexJSON, err := json.Marshal(index) + if err != nil { + t.Fatal(err) + } + appendBlob(ocispec.MediaTypeImageIndex, indexJSON) + } + + appendBlob(ocispec.MediaTypeImageConfig, []byte("config_1")) // Blob 0 + appendBlob(ocispec.MediaTypeImageLayer, []byte("layer_1")) // Blob 1 + generateManifest(nil, descs[0], descs[1]) // Blob 2 + appendBlob(ocispec.MediaTypeImageConfig, []byte("config_2")) // Blob 3 + appendBlob(ocispec.MediaTypeImageLayer, []byte("layer_2")) // Blob 4 + generateManifest(nil, descs[3], descs[4]) // Blob 5 + appendBlob(ocispec.MediaTypeImageLayer, []byte("{}")) // Blob 6 + appendBlob(ocispec.MediaTypeImageLayer, []byte("sbom_1")) // Blob 7 + generateManifest(&descs[2], descs[6], descs[7]) // Blob 8 + appendBlob(ocispec.MediaTypeImageLayer, []byte("sbom_2")) // Blob 9 + generateManifest(&descs[5], descs[6], descs[9]) // Blob 10 + generateIndex(nil, []ocispec.Descriptor{descs[2], descs[5]}...) // Blob 11 (root) + generateIndex(&descs[11], []ocispec.Descriptor{descs[8], descs[10]}...) // Blob 12 (root) + + ctx := context.Background() + verifyCopy := func(dst content.Fetcher, copiedIndice []int, uncopiedIndice []int) { + for _, i := range copiedIndice { + got, err := content.FetchAll(ctx, dst, descs[i]) + if err != nil { + t.Errorf("content[%d] error = %v, wantErr %v", i, err, false) + continue + } + if want := blobs[i]; !bytes.Equal(got, want) { + t.Errorf("content[%d] = %v, want %v", i, got, want) + } + } + for _, i := range uncopiedIndice { + if _, err := content.FetchAll(ctx, dst, descs[i]); !errors.Is(err, errdef.ErrNotFound) { + t.Errorf("content[%d] error = %v, wantErr %v", i, err, errdef.ErrNotFound) + } + } + } + + src := memory.New() + for i := range blobs { + err := src.Push(ctx, descs[i], bytes.NewReader(blobs[i])) + if err != nil { + t.Fatalf("failed to push test content to src: %d: %v", i, err) + } + } + + // test extended copy by descs[0] + dst := memory.New() + if err := oras.ExtendedCopyGraph(ctx, src, dst, descs[0], oras.ExtendedCopyGraphOptions{}); err != nil { + t.Fatalf("ExtendedCopyGraph() error = %v, wantErr %v", err, false) + } + // all blobs should be copied + copiedIndice := []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12} + uncopiedIndice := []int{} + verifyCopy(dst, copiedIndice, uncopiedIndice) + + // test extended copy by descs[2] + dst = memory.New() + if err := oras.ExtendedCopyGraph(ctx, src, dst, descs[2], oras.ExtendedCopyGraphOptions{}); err != nil { + t.Fatalf("ExtendedCopyGraph() error = %v, wantErr %v", err, false) + } + // all blobs should be copied + copiedIndice = []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12} + uncopiedIndice = []int{} + verifyCopy(dst, copiedIndice, uncopiedIndice) + + // test extended copy by descs[8] + dst = memory.New() + if err := oras.ExtendedCopyGraph(ctx, src, dst, descs[8], oras.ExtendedCopyGraphOptions{}); err != nil { + t.Fatalf("ExtendedCopyGraph() error = %v, wantErr %v", err, false) + } + // all blobs should be copied + copiedIndice = []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12} + uncopiedIndice = []int{} + verifyCopy(dst, copiedIndice, uncopiedIndice) + + // test extended copy by descs[11] + dst = memory.New() + if err := oras.ExtendedCopyGraph(ctx, src, dst, descs[11], oras.ExtendedCopyGraphOptions{}); err != nil { + t.Fatalf("ExtendedCopyGraph() error = %v, wantErr %v", err, false) + } + // all blobs should be copied + copiedIndice = []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12} + uncopiedIndice = []int{} + verifyCopy(dst, copiedIndice, uncopiedIndice) +} + func TestExtendedCopyGraph_WithDepthOption(t *testing.T) { // generate test content var blobs [][]byte From 442e15949b141fabbb27733643e836708f7172e0 Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Fri, 7 Jul 2023 17:45:33 +0800 Subject: [PATCH 11/14] test extended copy Signed-off-by: Lixia (Sylvia) Lei --- extendedcopy_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/extendedcopy_test.go b/extendedcopy_test.go index 0d478005..08a0a8c3 100644 --- a/extendedcopy_test.go +++ b/extendedcopy_test.go @@ -41,8 +41,6 @@ import ( "oras.land/oras-go/v2/registry/remote" ) -// TODO: test index with subject - func TestExtendedCopy_FullCopy(t *testing.T) { src := memory.New() dst := memory.New() From 0f9f01bd3b6e8ebe28dd86ede7cd11424b8f861f Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Fri, 7 Jul 2023 19:19:28 +0800 Subject: [PATCH 12/14] TODOs Signed-off-by: Lixia (Sylvia) Lei --- internal/manifestutil/parser.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/manifestutil/parser.go b/internal/manifestutil/parser.go index 3c957ceb..e242d5d6 100644 --- a/internal/manifestutil/parser.go +++ b/internal/manifestutil/parser.go @@ -25,6 +25,8 @@ import ( "oras.land/oras-go/v2/internal/docker" ) +// TODO: unit tests + // Config returns the config of desc, if present. func Config(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { switch desc.MediaType { From f5578ca117e0816bf22c4af5925340202acc9923 Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Mon, 10 Jul 2023 15:57:10 +0800 Subject: [PATCH 13/14] fix and improve tests Signed-off-by: Lixia (Sylvia) Lei --- content/graph.go | 76 +--------- content/graph_test.go | 71 +++++++--- internal/manifestutil/parser.go | 5 +- internal/manifestutil/parser_test.go | 202 +++++++++++++++++++++++++++ 4 files changed, 256 insertions(+), 98 deletions(-) create mode 100644 internal/manifestutil/parser_test.go diff --git a/content/graph.go b/content/graph.go index 025175c9..9ae83728 100644 --- a/content/graph.go +++ b/content/graph.go @@ -18,7 +18,6 @@ package content import ( "context" "encoding/json" - "fmt" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2/internal/docker" @@ -47,7 +46,7 @@ type ReadOnlyGraphStorage interface { } // Successors returns the nodes directly pointed by the current node. -// In other words, it returns the "children" of the current descriptor. +// In other words, returns the "children" of the current descriptor. func Successors(ctx context.Context, fetcher Fetcher, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { switch node.MediaType { case docker.MediaTypeManifest: @@ -121,76 +120,3 @@ func Successors(ctx context.Context, fetcher Fetcher, node ocispec.Descriptor) ( } return nil, nil } - -// SuccessorsParts returns the parts of the nodes directly pointed by the -// current node, in following order: -// - Subject (If present for OCI Image Manifest and OCI Image Index) -// - Config (If present for OCI Image Manifest and Docker Manifest) -// - Layers (For OCI Image Manifest and Docker Manifest), or Blobs (For OCI -// Artifact Manifest), or Manifests (For OCI Image Index or Docker Manifest -// List) -func SuccessorsParts(ctx context.Context, fetcher Fetcher, desc ocispec.Descriptor) (subject, config *ocispec.Descriptor, items []ocispec.Descriptor, err error) { - switch desc.MediaType { - case docker.MediaTypeManifest: - content, err := FetchAll(ctx, fetcher, desc) - if err != nil { - return nil, nil, nil, err - } - // OCI manifest schema can be used to marshal docker manifest - var manifest ocispec.Manifest - if err := json.Unmarshal(content, &manifest); err != nil { - return nil, nil, nil, fmt.Errorf("failed to unmarshal %s: %s: %w", desc.Digest.String(), desc.MediaType, err) - } - config = &manifest.Config - items = manifest.Layers - case ocispec.MediaTypeImageManifest: - content, err := FetchAll(ctx, fetcher, desc) - if err != nil { - return nil, nil, nil, err - } - var manifest ocispec.Manifest - if err := json.Unmarshal(content, &manifest); err != nil { - return nil, nil, nil, fmt.Errorf("failed to unmarshal %s: %s: %w", desc.Digest.String(), desc.MediaType, err) - } - subject = manifest.Subject - config = &manifest.Config - items = manifest.Layers - case docker.MediaTypeManifestList: - content, err := FetchAll(ctx, fetcher, desc) - if err != nil { - return nil, nil, nil, err - } - - // OCI Index schema can be used to marshal docker manifest list - var index ocispec.Index - if err := json.Unmarshal(content, &index); err != nil { - return nil, nil, nil, fmt.Errorf("failed to unmarshal %s: %s: %w", desc.Digest.String(), desc.MediaType, err) - } - items = index.Manifests - case ocispec.MediaTypeImageIndex: - content, err := FetchAll(ctx, fetcher, desc) - if err != nil { - return nil, nil, nil, err - } - - var index ocispec.Index - if err := json.Unmarshal(content, &index); err != nil { - return nil, nil, nil, fmt.Errorf("failed to unmarshal %s: %s: %w", desc.Digest.String(), desc.MediaType, err) - } - subject = index.Subject - items = index.Manifests - case spec.MediaTypeArtifactManifest: - content, err := FetchAll(ctx, fetcher, desc) - if err != nil { - return nil, nil, nil, err - } - - var manifest spec.Artifact - if err := json.Unmarshal(content, &manifest); err != nil { - return nil, nil, nil, fmt.Errorf("failed to unmarshal %s: %s: %w", desc.Digest.String(), desc.MediaType, err) - } - subject = manifest.Subject - items = manifest.Blobs - } - return -} diff --git a/content/graph_test.go b/content/graph_test.go index 0f6025f0..d90dc81a 100644 --- a/content/graph_test.go +++ b/content/graph_test.go @@ -76,7 +76,7 @@ func TestSuccessors_dockerManifest(t *testing.T) { if err != nil { t.Fatal("Successors() error =", err) } - if want := descs[0:4]; !equalDescriptorSet(got, want) { + if want := descs[0:4]; !reflect.DeepEqual(got, want) { t.Errorf("Successors() = %v, want %v", got, want) } } @@ -131,7 +131,7 @@ func TestSuccessors_imageManifest(t *testing.T) { if err != nil { t.Fatal("Successors() error =", err) } - if want := descs[0:4]; !equalDescriptorSet(got, want) { + if want := descs[0:4]; !reflect.DeepEqual(got, want) { t.Errorf("Successors() = %v, want %v", got, want) } @@ -141,7 +141,7 @@ func TestSuccessors_imageManifest(t *testing.T) { if err != nil { t.Fatal("Successors() error =", err) } - if want := descs[4:7]; !equalDescriptorSet(got, want) { + if want := descs[4:7]; !reflect.DeepEqual(got, want) { t.Errorf("Successors() = %v, want %v", got, want) } } @@ -204,7 +204,7 @@ func TestSuccessors_dockerManifestList(t *testing.T) { if err != nil { t.Fatal("Successors() error =", err) } - if want := descs[4:6]; !equalDescriptorSet(got, want) { + if want := descs[4:6]; !reflect.DeepEqual(got, want) { t.Errorf("Successors() = %v, want %v", got, want) } } @@ -272,7 +272,7 @@ func TestSuccessors_imageIndex(t *testing.T) { if err != nil { t.Fatal("Successors() error =", err) } - if want := append([]ocispec.Descriptor{descs[8]}, descs[4:6]...); !equalDescriptorSet(got, want) { + if want := append([]ocispec.Descriptor{descs[8]}, descs[4:6]...); !reflect.DeepEqual(got, want) { t.Errorf("Successors() = %v, want %v", got, want) } } @@ -324,7 +324,7 @@ func TestSuccessors_artifactManifest(t *testing.T) { if err != nil { t.Fatal("Successors() error =", err) } - if want := descs[0:3]; !equalDescriptorSet(got, want) { + if want := descs[0:3]; !reflect.DeepEqual(got, want) { t.Errorf("Successors() = %v, want %v", got, want) } @@ -334,27 +334,58 @@ func TestSuccessors_artifactManifest(t *testing.T) { if err != nil { t.Fatal("Successors() error =", err) } - if want := descs[3:5]; !equalDescriptorSet(got, want) { + if want := descs[3:5]; !reflect.DeepEqual(got, want) { t.Errorf("Successors() = %v, want %v", got, want) } } -func equalDescriptorSet(actual []ocispec.Descriptor, expected []ocispec.Descriptor) bool { - if len(actual) != len(expected) { - return false +func TestSuccessors_otherMediaType(t *testing.T) { + storage := cas.NewMemory() + + // generate test content + var blobs [][]byte + var descs []ocispec.Descriptor + appendBlob := func(mediaType string, blob []byte) { + blobs = append(blobs, blob) + descs = append(descs, ocispec.Descriptor{ + MediaType: mediaType, + Digest: digest.FromBytes(blob), + Size: int64(len(blob)), + }) } - contains := func(node ocispec.Descriptor) bool { - for _, candidate := range actual { - if reflect.DeepEqual(candidate, node) { - return true - } + generateManifest := func(mediaType string, config ocispec.Descriptor, layers ...ocispec.Descriptor) { + manifest := ocispec.Manifest{ + Config: config, + Layers: layers, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) } - return false + appendBlob(mediaType, manifestJSON) } - for _, node := range expected { - if !contains(node) { - return false + + appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0 + appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 1 + appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 2 + appendBlob(ocispec.MediaTypeImageLayer, []byte("hello")) // Blob 3 + generateManifest("whatever", descs[0], descs[1:4]...) // Blob 4 + + ctx := context.Background() + for i := range blobs { + err := storage.Push(ctx, descs[i], bytes.NewReader(blobs[i])) + if err != nil { + t.Fatalf("failed to push test content to src: %d: %v", i, err) } } - return true + + // test Successors: other media type + manifestDesc := descs[4] + got, err := content.Successors(ctx, storage, manifestDesc) + if err != nil { + t.Fatal("Successors() error =", err) + } + if got != nil { + t.Errorf("Successors() = %v, want nil", got) + } } diff --git a/internal/manifestutil/parser.go b/internal/manifestutil/parser.go index e242d5d6..81dc347f 100644 --- a/internal/manifestutil/parser.go +++ b/internal/manifestutil/parser.go @@ -25,8 +25,6 @@ import ( "oras.land/oras-go/v2/internal/docker" ) -// TODO: unit tests - // Config returns the config of desc, if present. func Config(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { switch desc.MediaType { @@ -49,12 +47,13 @@ func Config(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descripto // Manifest returns the manifests of desc, if present. func Manifests(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { switch desc.MediaType { - case ocispec.MediaTypeImageIndex: + case docker.MediaTypeManifestList, ocispec.MediaTypeImageIndex: content, err := content.FetchAll(ctx, fetcher, desc) if err != nil { return nil, err } + // OCI manifest index schema can be used to marshal docker manifest list var index ocispec.Index if err := json.Unmarshal(content, &index); err != nil { return nil, err diff --git a/internal/manifestutil/parser_test.go b/internal/manifestutil/parser_test.go new file mode 100644 index 00000000..44c5e43e --- /dev/null +++ b/internal/manifestutil/parser_test.go @@ -0,0 +1,202 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manifestutil + +import ( + "bytes" + "context" + "encoding/json" + "reflect" + "testing" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/internal/cas" + "oras.land/oras-go/v2/internal/docker" +) + +func TestConfig(t *testing.T) { + storage := cas.NewMemory() + + // generate test content + var blobs [][]byte + var descs []ocispec.Descriptor + appendBlob := func(mediaType string, blob []byte) { + blobs = append(blobs, blob) + descs = append(descs, ocispec.Descriptor{ + MediaType: mediaType, + Digest: digest.FromBytes(blob), + Size: int64(len(blob)), + }) + } + generateManifest := func(mediaType string, config ocispec.Descriptor, layers ...ocispec.Descriptor) { + manifest := ocispec.Manifest{ + Config: config, + Layers: layers, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + appendBlob(mediaType, manifestJSON) + } + + appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0 + appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 1 + generateManifest(ocispec.MediaTypeImageManifest, descs[0], descs[1]) // Blob 2 + generateManifest(docker.MediaTypeManifest, descs[0], descs[1]) // Blob 3 + generateManifest("whatever", descs[0], descs[1]) // Blob 4 + + ctx := context.Background() + for i := range blobs { + err := storage.Push(ctx, descs[i], bytes.NewReader(blobs[i])) + if err != nil { + t.Fatalf("failed to push test content to src: %d: %v", i, err) + } + } + + tests := []struct { + name string + desc ocispec.Descriptor + want *ocispec.Descriptor + wantErr bool + }{ + { + name: "OCI Image Manifest", + desc: descs[2], + want: &descs[0], + }, + { + name: "Docker Manifest", + desc: descs[3], + want: &descs[0], + wantErr: false, + }, + { + name: "Other media type", + desc: descs[4], + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := Config(ctx, storage, tt.desc) + if (err != nil) != tt.wantErr { + t.Errorf("Config() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Config() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestManifests(t *testing.T) { + storage := cas.NewMemory() + + // generate test content + var blobs [][]byte + var descs []ocispec.Descriptor + appendBlob := func(mediaType string, blob []byte) { + blobs = append(blobs, blob) + descs = append(descs, ocispec.Descriptor{ + MediaType: mediaType, + Digest: digest.FromBytes(blob), + Size: int64(len(blob)), + }) + } + generateManifest := func(subject *ocispec.Descriptor, config ocispec.Descriptor, layers ...ocispec.Descriptor) { + manifest := ocispec.Manifest{ + Subject: subject, + Config: config, + Layers: layers, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + appendBlob(ocispec.MediaTypeImageManifest, manifestJSON) + } + generateIndex := func(mediaType string, subject *ocispec.Descriptor, manifests ...ocispec.Descriptor) { + index := ocispec.Index{ + Subject: subject, + Manifests: manifests, + } + indexJSON, err := json.Marshal(index) + if err != nil { + t.Fatal(err) + } + appendBlob(mediaType, indexJSON) + } + + appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0 + appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 1 + appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 2 + appendBlob(ocispec.MediaTypeImageLayer, []byte("hello")) // Blob 3 + generateManifest(nil, descs[0], descs[1:3]...) // Blob 4 + generateManifest(nil, descs[0], descs[3]) // Blob 5 + appendBlob(ocispec.MediaTypeImageConfig, []byte("{}")) // Blob 6 + appendBlob("test/sig", []byte("sig")) // Blob 7 + generateManifest(&descs[4], descs[5], descs[6]) // Blob 8 + generateIndex(ocispec.MediaTypeImageIndex, &descs[8], descs[4:6]...) // Blob 9 + generateIndex(docker.MediaTypeManifestList, nil, descs[4:6]...) // Blob 10 + generateIndex("whatever", &descs[8], descs[4:6]...) // Blob 11 + + ctx := context.Background() + for i := range blobs { + err := storage.Push(ctx, descs[i], bytes.NewReader(blobs[i])) + if err != nil { + t.Fatalf("failed to push test content to src: %d: %v", i, err) + } + } + + tests := []struct { + name string + desc ocispec.Descriptor + want []ocispec.Descriptor + wantErr bool + }{ + { + name: "OCI Image Index", + desc: descs[9], + want: descs[4:6], + }, + { + name: "Docker Manifest List", + desc: descs[10], + want: descs[4:6], + wantErr: false, + }, + { + name: "Other media type", + desc: descs[11], + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := Manifests(ctx, storage, tt.desc) + if (err != nil) != tt.wantErr { + t.Errorf("Manifests() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Manifests() = %v, want %v", got, tt.want) + } + }) + } +} From 4a53a8a79cc62e8ece7815b60f847d5e118b4a5a Mon Sep 17 00:00:00 2001 From: "Lixia (Sylvia) Lei" Date: Tue, 11 Jul 2023 14:21:04 +0800 Subject: [PATCH 14/14] address comments Signed-off-by: Lixia (Sylvia) Lei --- internal/manifestutil/parser.go | 2 -- internal/platform/platform.go | 5 +---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/internal/manifestutil/parser.go b/internal/manifestutil/parser.go index 81dc347f..c904dc69 100644 --- a/internal/manifestutil/parser.go +++ b/internal/manifestutil/parser.go @@ -20,7 +20,6 @@ import ( "encoding/json" ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "oras.land/oras-go/v2/content" "oras.land/oras-go/v2/internal/docker" ) @@ -52,7 +51,6 @@ func Manifests(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descri if err != nil { return nil, err } - // OCI manifest index schema can be used to marshal docker manifest list var index ocispec.Index if err := json.Unmarshal(content, &index); err != nil { diff --git a/internal/platform/platform.go b/internal/platform/platform.go index a84475f9..e903fe3d 100644 --- a/internal/platform/platform.go +++ b/internal/platform/platform.go @@ -91,14 +91,11 @@ func SelectManifest(ctx context.Context, src content.ReadOnlyStorage, root ocisp } return ocispec.Descriptor{}, fmt.Errorf("%s: %w: no matching manifest was found in the manifest list", root.Digest, errdef.ErrNotFound) case docker.MediaTypeManifest, ocispec.MediaTypeImageManifest: + // config will be non-nil for docker manifest and OCI image manifest config, err := manifestutil.Config(ctx, src, root) if err != nil { return ocispec.Descriptor{}, err } - if config == nil { - // should not happen - return ocispec.Descriptor{}, fmt.Errorf("%s: %s: %w", root.Digest, root.MediaType, errdef.ErrUnsupported) - } configMediaType := docker.MediaTypeConfig if root.MediaType == ocispec.MediaTypeImageManifest {