Skip to content

Commit

Permalink
fix sftp chunk upload
Browse files Browse the repository at this point in the history 
Signed-off-by: Maksym Trofimenko <[email protected]>
  • Loading branch information
Maksym Trofimenko committed Feb 10, 2025
1 parent b6c99ac commit 07480ba
Show file tree
Hide file tree
Showing 9 changed files with 125 additions and 228 deletions.
7 changes: 7 additions & 0 deletions src/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -435,6 +435,8 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo=
github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk=
github.com/pkg/sftp v1.13.7 h1:uv+I3nNJvlKZIQGSr8JVQLNHFU9YhhNpvC14Y6KgmSM=
github.com/pkg/sftp v1.13.7/go.mod h1:KMKI0t3T6hfA+lTR/ssZdunHo+uwq7ghoN09/FSu3DY=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
Expand Down Expand Up @@ -607,6 +609,7 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
Expand Down Expand Up @@ -645,6 +648,7 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
Expand Down Expand Up @@ -693,6 +697,7 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand All @@ -702,6 +707,7 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
Expand All @@ -714,6 +720,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
Expand Down
35 changes: 3 additions & 32 deletions src/pkg/reg/adapter/storage/adapter.go
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package storage

import (
"bytes"
"context"
"errors"
"fmt"
Expand Down Expand Up @@ -34,7 +33,6 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er
ctx := context.Background()
var repoNames = make([]string, 1000)

fmt.Println("FetchArtifacts")
// @todo do iteration using last
_, err := a.registry.Repositories(ctx, repoNames, "")

Expand Down Expand Up @@ -135,8 +133,6 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er
func (a *adapter) ManifestExist(repository, ref string) (exist bool, desc *distribution.Descriptor, err error) {
ctx := context.Background()

fmt.Println("ManifestExist", repository, ref)

repo, err := a.getRepo(ctx, repository, ref)
if err != nil {
return false, nil, fmt.Errorf("get repo error: %v", err)
Expand All @@ -149,6 +145,7 @@ func (a *adapter) ManifestExist(repository, ref string) (exist bool, desc *distr
if !strings.HasPrefix(ref, "sha256:") {
// looks like a tag
descriptor, err := tagService.Get(ctx, ref)

if err != nil {
var errTagUnknown distribution.ErrTagUnknown
if errors.As(err, &errTagUnknown) {
Expand Down Expand Up @@ -176,8 +173,6 @@ func (a *adapter) ManifestExist(repository, ref string) (exist bool, desc *distr
func (a *adapter) PullManifest(repository, ref string, _ ...string) (distribution.Manifest, string, error) {
ctx := context.Background()

fmt.Println("PullManifest", repository, ref)

repo, err := a.getRepo(ctx, repository, ref)
if err != nil {
return nil, "", fmt.Errorf("get repo error: %v", err)
Expand Down Expand Up @@ -216,8 +211,6 @@ func (a *adapter) PullManifest(repository, ref string, _ ...string) (distributio
// PushManifest manifests are blobs actually
func (a *adapter) PushManifest(repository, ref, mediaType string, payload []byte) (string, error) {

fmt.Println("PushManifest", repository, ref, mediaType, payload)

ctx := context.Background()

repo, err := a.getRepo(ctx, repository, ref)
Expand Down Expand Up @@ -263,8 +256,6 @@ func (a *adapter) PushManifest(repository, ref, mediaType string, payload []byte
func (a *adapter) DeleteManifest(repository, ref string) error {
ctx := context.Background()

fmt.Println("DeleteManifest", repository, ref)

named, err := reference.WithName(repository)
if err != nil {
return err
Expand Down Expand Up @@ -304,8 +295,6 @@ func (a *adapter) DeleteManifest(repository, ref string) error {
func (a *adapter) BlobExist(repository, d string) (exist bool, err error) {
ctx := context.Background()

fmt.Println("BlobExist", repository, d)

repo, err := a.getRepo(ctx, repository, d)
if err != nil {
return false, fmt.Errorf("get repo error: %v", err)
Expand All @@ -325,8 +314,6 @@ func (a *adapter) BlobExist(repository, d string) (exist bool, err error) {
func (a *adapter) PullBlob(repository, d string) (int64, io.ReadCloser, error) {
ctx := context.Background()

fmt.Println("PullBlob ", repository, d)

repo, err := a.getRepo(ctx, repository, d)
if err != nil {
return 0, nil, fmt.Errorf("get repo error: %v", err)
Expand All @@ -343,19 +330,11 @@ func (a *adapter) PullBlob(repository, d string) (int64, io.ReadCloser, error) {
if err != nil {
return 0, nil, fmt.Errorf("unable to open blob: %v", err)
}
defer readSeeker.Close()

data, err := io.ReadAll(readSeeker)
if err != nil {
return 0, nil, fmt.Errorf("unable to read blob: %v", err)
}

return descriptor.Size, io.NopCloser(bytes.NewReader(data)), nil
return descriptor.Size, io.NopCloser(readSeeker), nil
}

func (a *adapter) PullBlobChunk(repository, d string, totalSize, start, end int64) (size int64, blob io.ReadCloser, err error) {

fmt.Println("PullBlobChunk ", repository, d, "total", totalSize, "start", start, "end", end)
ctx := context.Background()

repo, err := a.getRepo(ctx, repository, d)
Expand All @@ -375,23 +354,16 @@ func (a *adapter) PullBlobChunk(repository, d string, totalSize, start, end int6
return 0, nil, fmt.Errorf("unable to open blob: %v", err)
}

defer readSeeker.Close()

_, err = readSeeker.Seek(end-start, int(start))
if err != nil {
return 0, nil, fmt.Errorf("unable to seek blob: %v", err)
}

chunk, err := io.ReadAll(readSeeker)
if err != nil {
return 0, nil, fmt.Errorf("unable to read blob chunk: %v", err)
}
return descriptor.Size, io.NopCloser(bytes.NewReader(chunk)), nil
return descriptor.Size, io.NopCloser(readSeeker), nil
}

func (a *adapter) PushBlobChunk(repository, d string, size int64, chunk io.Reader, start, end int64, location string) (nextUploadLocation string, endRange int64, err error) {

fmt.Println("PushBlobChunk ", repository, d, "total", size, "start", start, "end", end)
ctx := context.Background()

repo, err := a.getRepo(ctx, repository, d)
Expand All @@ -414,7 +386,6 @@ func (a *adapter) PushBlobChunk(repository, d string, size int64, chunk io.Reade
}

defer writer.Close()

_, err = writer.ReadFrom(chunk)
if err != nil {
return "", 0, fmt.Errorf("unable to read from chunk: %v", err)
Expand Down
Loading

0 comments on commit 07480ba

Please sign in to comment.