Skip to content
This repository has been archived by the owner on May 26, 2023. It is now read-only.

Commit

Permalink
uploaders package (#56)
Browse files Browse the repository at this point in the history 
* adds uploaders package

* adds uploader factory

* fix typo in go doc

* adds changes from transport cmd dev branch

* adds changes from dev branch to ociclient

* adds changes in utils package from dev branch

* Update pkg/transport/process/uploaders/local_oci_blob.go

Co-authored-by: Enrico Kaack <[email protected]>

Co-authored-by: Enrico Kaack <[email protected]>
  • Loading branch information
@jschicktanz @enrico-kaack-comp
jschicktanz and enrico-kaack-comp authored Dec 21, 2021
1 parent 03bb24e commit d0e17e1
Show file tree
Hide file tree
Showing 16 changed files with 965 additions and 106 deletions.
27 changes: 27 additions & 0 deletions ociclient/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,33 @@ func (c *client) PushOCIArtifact(ctx context.Context, ref string, artifact *oci.
}
}

func (c *client) PushBlob(ctx context.Context, ref string, desc ocispecv1.Descriptor, options ...PushOption) error {
refspec, err := oci.ParseRef(ref)
if err != nil {
return fmt.Errorf("unable to parse ref: %w", err)
}
ref = refspec.String()

opts := &PushOptions{}
opts.Store = c.cache
opts.ApplyOptions(options)

resolver, err := c.getResolverForRef(ctx, ref, transport.PushScope)
if err != nil {
return err
}
pusher, err := resolver.Pusher(ctx, ref)
if err != nil {
return err
}

if err := c.pushContent(ctx, opts.Store, pusher, desc); err != nil {
return err
}

return nil
}

func (c *client) pushManifest(ctx context.Context, manifest *ocispecv1.Manifest, pusher remotes.Pusher, cache cache.Cache, opts *PushOptions) (ocispecv1.Descriptor, error) {
// add dummy config if it is not set
if manifest.Config.Size == 0 {
Expand Down
43 changes: 32 additions & 11 deletions ociclient/client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ var _ = Describe("client", func() {
defer ctx.Done()

ref := testenv.Addr + "/test/artifact:v0.0.1"
manifest, _, err := testutils.UploadTestManifest(ctx, client, ref)
manifest, mdesc, err := testutils.UploadTestManifest(ctx, client, ref)
Expect(err).ToNot(HaveOccurred())

res, err := client.GetManifest(ctx, ref)
Expand All @@ -41,7 +41,19 @@ var _ = Describe("client", func() {
Expect(res.Layers).To(Equal(manifest.Layers))

// TODO: oci image index test only working because cache is filled in this function with config/layer blobs. should be fixed
testutils.CompareManifestToTestManifest(ctx, client, ref, res)
expectedManifest := oci.Manifest{
Descriptor: mdesc,
Data: manifest,
}
testutils.CompareRemoteManifest(
client,
ref,
expectedManifest,
[]byte("config-data"),
[][]byte{
[]byte("layer-data"),
},
)
}, 20)

It("should push and pull an oci image index", func() {
Expand All @@ -57,7 +69,7 @@ var _ = Describe("client", func() {

Expect(actualArtifact.IsManifest()).To(BeFalse())
Expect(actualArtifact.IsIndex()).To(BeTrue())
testutils.CompareImageIndices(actualArtifact.GetIndex(), index)
Expect(actualArtifact.GetIndex()).To(Equal(index))
}, 20)

It("should push and pull an empty oci image index", func() {
Expand All @@ -83,7 +95,7 @@ var _ = Describe("client", func() {

Expect(actualArtifact.IsManifest()).To(BeFalse())
Expect(actualArtifact.IsIndex()).To(BeTrue())
testutils.CompareImageIndices(actualArtifact.GetIndex(), &index)
Expect(actualArtifact.GetIndex()).To(Equal(&index))
}, 20)

It("should push and pull an oci image index with only 1 manifest and no platform information", func() {
Expand All @@ -92,13 +104,14 @@ var _ = Describe("client", func() {

ref := testenv.Addr + "/image-index/3/img:v0.0.1"
manifest1Ref := testenv.Addr + "/image-index/1/img-platform-1:v0.0.1"
manifest, _, err := testutils.UploadTestManifest(ctx, client, manifest1Ref)
manifest, mdesc, err := testutils.UploadTestManifest(ctx, client, manifest1Ref)
Expect(err).ToNot(HaveOccurred())

index := oci.Index{
Manifests: []*oci.Manifest{
{
Data: manifest,
Descriptor: mdesc,
Data: manifest,
},
},
Annotations: map[string]string{
Expand All @@ -117,7 +130,7 @@ var _ = Describe("client", func() {

Expect(actualArtifact.IsManifest()).To(BeFalse())
Expect(actualArtifact.IsIndex()).To(BeTrue())
testutils.CompareImageIndices(actualArtifact.GetIndex(), &index)
Expect(actualArtifact.GetIndex()).To(Equal(&index))
}, 20)

It("should copy an oci artifact", func() {
Expand Down Expand Up @@ -161,10 +174,18 @@ var _ = Describe("client", func() {

Expect(actualArtifact.IsManifest()).To(BeFalse())
Expect(actualArtifact.IsIndex()).To(BeTrue())
testutils.CompareImageIndices(actualArtifact.GetIndex(), index)

for _, manifest := range actualArtifact.GetIndex().Manifests {
testutils.CompareManifestToTestManifest(ctx, client, newRef, manifest.Data)
Expect(actualArtifact.GetIndex()).To(Equal(index))

for i := range actualArtifact.GetIndex().Manifests {
testutils.CompareRemoteManifest(
client,
ref,
*index.Manifests[i],
[]byte("config-data"),
[][]byte{
[]byte("layer-data"),
},
)
}
}, 20)

Expand Down
3 changes: 3 additions & 0 deletions ociclient/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@ type Client interface {

// PushOCIArtifact uploads the given OCIArtifact to the given ref.
PushOCIArtifact(ctx context.Context, ref string, artifact *oci.Artifact, opts ...PushOption) error

// PushBlob uploads the blob for the given ocispec Descriptor to the given ref
PushBlob(ctx context.Context, ref string, desc ocispecv1.Descriptor, opts ...PushOption) error
}

// ExtendedClient defines an oci client with extended functionality that may not work with all registries.
Expand Down
128 changes: 67 additions & 61 deletions pkg/testutils/oci.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,15 @@ import (

. "github.com/onsi/gomega"
"github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/specs-go"
ocispecv1 "github.com/opencontainers/image-spec/specs-go/v1"

"github.com/gardener/component-cli/ociclient"
"github.com/gardener/component-cli/ociclient/cache"
"github.com/gardener/component-cli/ociclient/oci"
)

// UploadTestManifest uploads an oci image manifest to a registry
func UploadTestManifest(ctx context.Context, client ociclient.Client, ref string) (*ocispecv1.Manifest, ocispecv1.Descriptor, error) {
configData := []byte("config-data")
layerData := []byte("layer-data")
Expand Down Expand Up @@ -65,16 +68,7 @@ func UploadTestManifest(ctx context.Context, client ociclient.Client, ref string
return manifest, desc, nil
}

func CompareManifestToTestManifest(ctx context.Context, client ociclient.Client, ref string, manifest *ocispecv1.Manifest) {
var configBlob bytes.Buffer
Expect(client.Fetch(ctx, ref, manifest.Config, &configBlob)).To(Succeed())
Expect(configBlob.String()).To(Equal("config-data"))

var layerBlob bytes.Buffer
Expect(client.Fetch(ctx, ref, manifest.Layers[0], &layerBlob)).To(Succeed())
Expect(layerBlob.String()).To(Equal("layer-data"))
}

// UploadTestIndex uploads an oci image index to a registry
func UploadTestIndex(ctx context.Context, client ociclient.Client, indexRef string) (*oci.Index, error) {
splitted := strings.Split(indexRef, ":")
indexRepo := strings.Join(splitted[0:len(splitted)-1], ":")
Expand All @@ -83,35 +77,33 @@ func UploadTestIndex(ctx context.Context, client ociclient.Client, indexRef stri
manifest1Ref := fmt.Sprintf("%s-platform-1:%s", indexRepo, tag)
manifest2Ref := fmt.Sprintf("%s-platform-2:%s", indexRepo, tag)

manifest1, _, err := UploadTestManifest(ctx, client, manifest1Ref)
manifest1, mdesc1, err := UploadTestManifest(ctx, client, manifest1Ref)
if err != nil {
return nil, err
}
mdesc1.Platform = &ocispecv1.Platform{
Architecture: "amd64",
OS: "linux",
}

manifest2, _, err := UploadTestManifest(ctx, client, manifest2Ref)
manifest2, mdesc2, err := UploadTestManifest(ctx, client, manifest2Ref)
if err != nil {
return nil, err
}
mdesc2.Platform = &ocispecv1.Platform{
Architecture: "amd64",
OS: "windows",
}

index := oci.Index{
Manifests: []*oci.Manifest{
{
Descriptor: ocispecv1.Descriptor{
Platform: &ocispecv1.Platform{
Architecture: "amd64",
OS: "linux",
},
},
Data: manifest1,
Descriptor: mdesc1,
Data: manifest1,
},
{
Descriptor: ocispecv1.Descriptor{
Platform: &ocispecv1.Platform{
Architecture: "amd64",
OS: "windows",
},
},
Data: manifest2,
Descriptor: mdesc2,
Data: manifest2,
},
},
Annotations: map[string]string{
Expand All @@ -131,53 +123,67 @@ func UploadTestIndex(ctx context.Context, client ociclient.Client, indexRef stri
return &index, nil
}

func CompareImageIndices(actualIndex *oci.Index, expectedIndex *oci.Index) {
Expect(actualIndex.Annotations).To(Equal(expectedIndex.Annotations))
Expect(len(actualIndex.Manifests)).To(Equal(len(expectedIndex.Manifests)))

for i := 0; i < len(actualIndex.Manifests); i++ {
actualManifest := actualIndex.Manifests[i]
expectedManifest := expectedIndex.Manifests[i]

expectedManifestBytes, err := json.Marshal(expectedManifest.Data)
Expect(err).ToNot(HaveOccurred())

Expect(actualManifest.Descriptor.MediaType).To(Equal(ocispecv1.MediaTypeImageManifest))
Expect(actualManifest.Descriptor.Digest).To(Equal(digest.FromBytes(expectedManifestBytes)))
Expect(actualManifest.Descriptor.Size).To(Equal(int64(len(expectedManifestBytes))))
Expect(actualManifest.Descriptor.Platform).To(Equal(expectedManifest.Descriptor.Platform))
Expect(actualManifest.Data).To(Equal(expectedManifest.Data))
}
}

func CreateManifest(configData []byte, layerData []byte) (*ocispecv1.Manifest, ocispecv1.Descriptor, error) {
// CreateManifest creates an oci manifest. if ocicache is set, all blobs are added to cache
func CreateManifest(configData []byte, layersData [][]byte, ocicache cache.Cache) (*ocispecv1.Manifest, ocispecv1.Descriptor) {
configDesc := ocispecv1.Descriptor{
MediaType: "text/plain",
Digest: digest.FromBytes(configData),
Size: int64(len(configData)),
}
if ocicache != nil {
Expect(ocicache.Add(configDesc, io.NopCloser(bytes.NewReader(configData)))).To(Succeed())
}

layerDesc := ocispecv1.Descriptor{
MediaType: "text/plain",
Digest: digest.FromBytes(layerData),
Size: int64(len(layerData)),
layerDescs := []ocispecv1.Descriptor{}
for _, layerData := range layersData {
layerDesc := ocispecv1.Descriptor{
MediaType: "text/plain",
Digest: digest.FromBytes(layerData),
Size: int64(len(layerData)),
}
layerDescs = append(layerDescs, layerDesc)
if ocicache != nil {
Expect(ocicache.Add(layerDesc, io.NopCloser(bytes.NewReader(layerData)))).To(Succeed())
}
}

m := ocispecv1.Manifest{
Config: configDesc,
Layers: []ocispecv1.Descriptor{
layerDesc,
manifest := ocispecv1.Manifest{
Versioned: specs.Versioned{
SchemaVersion: 2,
},
Config: configDesc,
Layers: layerDescs,
}

mBytes, err := json.Marshal(m)
if err != nil {
return nil, ocispecv1.Descriptor{}, err
}
manifestBytes, err := json.Marshal(manifest)
Expect(err).ToNot(HaveOccurred())

d := ocispecv1.Descriptor{
Digest: digest.FromBytes(mBytes),
manifestDesc := ocispecv1.Descriptor{
MediaType: ocispecv1.MediaTypeImageManifest,
Digest: digest.FromBytes(manifestBytes),
Size: int64(len(manifestBytes)),
}
if ocicache != nil {
Expect(ocicache.Add(manifestDesc, io.NopCloser(bytes.NewReader(manifestBytes)))).To(Succeed())
}

return &m, d, nil
return &manifest, manifestDesc
}

func CompareRemoteManifest(client ociclient.Client, ref string, expectedManifest oci.Manifest, expectedCfgBytes []byte, expectedLayers [][]byte) {
buf := bytes.NewBuffer([]byte{})
Expect(client.Fetch(context.TODO(), ref, expectedManifest.Descriptor, buf)).To(Succeed())
manifestFromRemote := ocispecv1.Manifest{}
Expect(json.Unmarshal(buf.Bytes(), &manifestFromRemote)).To(Succeed())
Expect(manifestFromRemote).To(Equal(*expectedManifest.Data))

buf = bytes.NewBuffer([]byte{})
Expect(client.Fetch(context.TODO(), ref, manifestFromRemote.Config, buf)).To(Succeed())
Expect(buf.Bytes()).To(Equal(expectedCfgBytes))

for i, layerDesc := range manifestFromRemote.Layers {
buf = bytes.NewBuffer([]byte{})
Expect(client.Fetch(context.TODO(), ref, layerDesc, buf)).To(Succeed())
Expect(buf.Bytes()).To(Equal(expectedLayers[i]))
}
}
14 changes: 13 additions & 1 deletion pkg/transport/process/downloaders/downloaders_suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,19 @@ func createImageRes(ctx context.Context) cdv2.Resource {
Expect(err).ToNot(HaveOccurred())

// TODO: currently needed to fill the cache. remove from test, also from ociclient unit test
testutils.CompareManifestToTestManifest(context.TODO(), ociClient, imageRef, manifest)
m := oci.Manifest{
Descriptor: desc,
Data: manifest,
}
testutils.CompareRemoteManifest(
ociClient,
imageRef,
m,
[]byte("config-data"),
[][]byte{
[]byte("layer-data"),
},
)

expectedImageManifest = oci.Manifest{
Descriptor: desc,
Expand Down
12 changes: 10 additions & 2 deletions pkg/transport/process/downloaders/oci_artifact_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,15 @@ var _ = Describe("ociArtifact", func() {
actualOciArtifact, err := utils.DeserializeOCIArtifact(resBlobReader, ociCache)
Expect(err).ToNot(HaveOccurred())
Expect(*actualOciArtifact.GetManifest()).To(Equal(expectedImageManifest))
testutils.CompareManifestToTestManifest(context.TODO(), ociClient, imageRef, expectedImageManifest.Data)
testutils.CompareRemoteManifest(
ociClient,
imageRef,
expectedImageManifest,
[]byte("config-data"),
[][]byte{
[]byte("layer-data"),
},
)
})

It("should download and stream oci image index", func() {
Expand All @@ -69,7 +77,7 @@ var _ = Describe("ociArtifact", func() {

actualOciArtifact, err := utils.DeserializeOCIArtifact(resBlobReader, ociCache)
Expect(err).ToNot(HaveOccurred())
testutils.CompareImageIndices(actualOciArtifact.GetIndex(), &expectedImageIndex)
Expect(actualOciArtifact.GetIndex()).To(Equal(&expectedImageIndex))
})

It("should return error if called with resource of invalid type", func() {
Expand Down
Loading

0 comments on commit d0e17e1

Please sign in to comment.