From 42305a4628b64b9a6714814b98258291618ba79f Mon Sep 17 00:00:00 2001 From: Johannes Boyne Date: Thu, 29 Dec 2022 07:58:13 -0500 Subject: [PATCH] Initiate work on PutObjectTagging ref PR#71 and issue #72 --- awscli_test.go | 20 ++++++------ backend.go | 3 +- backend/s3mem/backend.go | 3 +- backend/s3mem/bucket.go | 2 ++ gofakes3.go | 69 +++++++++++++++++++++++++++++++++++++--- gofakes3_test.go | 55 ++++++++++++++++++++------------ init_test.go | 8 ++--- messages.go | 10 ++++++ routing.go | 23 ++++++++++++-- routing_test.go | 2 +- 10 files changed, 152 insertions(+), 43 deletions(-) diff --git a/awscli_test.go b/awscli_test.go index 6af3d56..d9ff787 100644 --- a/awscli_test.go +++ b/awscli_test.go @@ -44,20 +44,20 @@ func TestCLILsFiles(t *testing.T) { t.Fatal() } - cli.backendPutString(defaultBucket, "test-one", nil, "hello") + cli.backendPutString(defaultBucket, "test-one", nil, nil, "hello") cli.assertLsFiles(defaultBucket, "", nil, []string{"test-one"}) - cli.backendPutString(defaultBucket, "test-two", nil, "hello") + cli.backendPutString(defaultBucket, "test-two", nil, nil, "hello") cli.assertLsFiles(defaultBucket, "", nil, []string{"test-one", "test-two"}) // only "test-one" and "test-two" should pass the prefix match - cli.backendPutString(defaultBucket, "no-match", nil, "hello") + cli.backendPutString(defaultBucket, "no-match", nil, nil, "hello") cli.assertLsFiles(defaultBucket, "test-", nil, []string{"test-one", "test-two"}) - cli.backendPutString(defaultBucket, "test/yep", nil, "hello") + cli.backendPutString(defaultBucket, "test/yep", nil, nil, "hello") cli.assertLsFiles(defaultBucket, "", []string{"test/"}, []string{"no-match", "test-one", "test-two"}) @@ -74,8 +74,8 @@ func TestCLIRmOne(t *testing.T) { cli := newTestCLI(t) defer cli.Close() - cli.backendPutString(defaultBucket, "foo", nil, "hello") - cli.backendPutString(defaultBucket, "bar", nil, "hello") + cli.backendPutString(defaultBucket, "foo", nil, nil, "hello") + cli.backendPutString(defaultBucket, "bar", nil, nil, "hello") cli.assertLsFiles(defaultBucket, "", nil, []string{"foo", "bar"}) cli.rm(cli.fileArg(defaultBucket, "foo")) @@ -86,9 +86,9 @@ func TestCLIRmMulti(t *testing.T) { cli := newTestCLI(t) defer cli.Close() - cli.backendPutString(defaultBucket, "foo", nil, "hello") - cli.backendPutString(defaultBucket, "bar", nil, "hello") - cli.backendPutString(defaultBucket, "baz", nil, "hello") + cli.backendPutString(defaultBucket, "foo", nil, nil, "hello") + cli.backendPutString(defaultBucket, "bar", nil, nil, "hello") + cli.backendPutString(defaultBucket, "baz", nil, nil, "hello") cli.assertLsFiles(defaultBucket, "", nil, []string{"foo", "bar", "baz"}) cli.rmMulti(defaultBucket, "foo", "bar", "baz") @@ -117,7 +117,7 @@ func TestCLIDownload(t *testing.T) { cli := newTestCLI(t) defer cli.Close() - cli.backendPutBytes(defaultBucket, "foo", nil, tc.in) + cli.backendPutBytes(defaultBucket, "foo", nil, nil, tc.in) out := cli.download(defaultBucket, "foo") if !bytes.Equal(out, tc.in) { t.Fatal() diff --git a/backend.go b/backend.go index 9fa7fb4..d25c086 100644 --- a/backend.go +++ b/backend.go @@ -17,6 +17,7 @@ const ( type Object struct { Name string Metadata map[string]string + Tags map[string]string Size int64 Contents io.ReadCloser Hash []byte @@ -227,7 +228,7 @@ type Backend interface { // // The size can be used if the backend needs to read the whole reader; use // gofakes3.ReadAll() for this job rather than ioutil.ReadAll(). - PutObject(bucketName, key string, meta map[string]string, input io.Reader, size int64) (PutObjectResult, error) + PutObject(bucketName, key string, meta map[string]string, tags map[string]string, input io.Reader, size int64) (PutObjectResult, error) DeleteMulti(bucketName string, objects ...string) (MultiDeleteResult, error) } diff --git a/backend/s3mem/backend.go b/backend/s3mem/backend.go index 716eaa5..4591805 100644 --- a/backend/s3mem/backend.go +++ b/backend/s3mem/backend.go @@ -217,7 +217,7 @@ func (db *Backend) GetObject(bucketName, objectName string, rangeRequest *gofake return result, nil } -func (db *Backend) PutObject(bucketName, objectName string, meta map[string]string, input io.Reader, size int64) (result gofakes3.PutObjectResult, err error) { +func (db *Backend) PutObject(bucketName, objectName string, meta map[string]string, tags map[string]string, input io.Reader, size int64) (result gofakes3.PutObjectResult, err error) { // No need to lock the backend while we read the data into memory; it holds // the write lock open unnecessarily, and could be blocked for an unreasonably // long time by a connection timing out: @@ -247,6 +247,7 @@ func (db *Backend) PutObject(bucketName, objectName string, meta map[string]stri hash: hash[:], etag: `"` + hex.EncodeToString(hash[:]) + `"`, metadata: meta, + tags: tags, lastModified: db.timeSource.Now(), } diff --git a/backend/s3mem/bucket.go b/backend/s3mem/bucket.go index a713d11..1ffea7e 100644 --- a/backend/s3mem/bucket.go +++ b/backend/s3mem/bucket.go @@ -120,6 +120,7 @@ type bucketData struct { hash []byte etag string metadata map[string]string + tags map[string]string } func (bi *bucketData) toObject(rangeRequest *gofakes3.ObjectRangeRequest, withBody bool) (obj *gofakes3.Object, err error) { @@ -152,6 +153,7 @@ func (bi *bucketData) toObject(rangeRequest *gofakes3.ObjectRangeRequest, withBo Name: bi.name, Hash: bi.hash, Metadata: bi.metadata, + Tags: bi.tags, Size: sz, Range: rnge, IsDeleteMarker: bi.deleteMarker, diff --git a/gofakes3.go b/gofakes3.go index f4ec9fa..7486407 100644 --- a/gofakes3.go +++ b/gofakes3.go @@ -549,7 +549,7 @@ func (g *GoFakeS3) createObjectBrowserUpload(bucket string, w http.ResponseWrite return err } - result, err := g.storage.PutObject(bucket, key, meta, rdr, fileHeader.Size) + result, err := g.storage.PutObject(bucket, key, meta, nil, rdr, fileHeader.Size) if err != nil { return err } @@ -623,7 +623,7 @@ func (g *GoFakeS3) createObject(bucket, object string, w http.ResponseWriter, r return err } - result, err := g.storage.PutObject(bucket, object, meta, rdr, size) + result, err := g.storage.PutObject(bucket, object, meta, nil, rdr, size) if err != nil { return err } @@ -681,7 +681,7 @@ func (g *GoFakeS3) copyObject(bucket, object string, meta map[string]string, w h } } - result, err := g.storage.PutObject(bucket, object, meta, srcObj.Contents, srcObj.Size) + result, err := g.storage.PutObject(bucket, object, meta, nil, srcObj.Contents, srcObj.Size) if err != nil { return err } @@ -881,6 +881,67 @@ func (g *GoFakeS3) putMultipartUploadPart(bucket, object string, uploadID Upload return nil } +// From the docs: +// +// Updating is usually done via different e.g., PutObject routes; but +// PutObjectTagging is one example where existing objects are updated in situ +// +// Sets the supplied tag-set to an object that already exists in a bucket. +// +// A tag is a key-value pair. You can associate tags with an object by sending a +// PUT request against the tagging subresource that is associated with the +// object. +func (g *GoFakeS3) updateObjectWithTags(bucket, object string, version string, w http.ResponseWriter, r *http.Request) error { + // write keys / metadata to object + if err := g.ensureBucketExists(bucket); err != nil { + return err + } + + var in Tagging + if err := g.xmlDecodeBody(r.Body, &in); err != nil { + return err + } + + rnge, err := parseRangeHeader(r.Header.Get("Range")) + if err != nil { + return err + } + + obj, err := g.storage.GetObject(bucket, object, rnge) + if err != nil { + return err + } + if obj.Tags == nil { + obj.Tags = map[string]string{} + } + for _, v := range in.TagSet.Tag { + obj.Tags[v.Key] = v.Value + } + + result, err := g.storage.PutObject(bucket, object, obj.Metadata, obj.Tags, obj.Contents, obj.Size) + if err != nil { + return err + } + if result.VersionID != "" { + w.Header().Set("x-amz-version-id", string(result.VersionID)) + } + + return nil +} + +func (g *GoFakeS3) getObjectTags(bucket, object string, version string, w http.ResponseWriter, r *http.Request) error { + obj, err := g.storage.HeadObject(bucket, object) + if err != nil { + return err + } + out := Tagging{} + for k, v := range obj.Tags { + out.TagSet.Tag = append(out.TagSet.Tag, Tag{Key: k, Value: v}) + } + + return g.xmlEncoder(w).Encode(out) +} + func (g *GoFakeS3) abortMultipartUpload(bucket, object string, uploadID UploadID, w http.ResponseWriter, r *http.Request) error { g.log.Print(LogInfo, "abort multipart upload", bucket, object, uploadID) if _, err := g.uploader.Complete(bucket, object, uploadID); err != nil { @@ -908,7 +969,7 @@ func (g *GoFakeS3) completeMultipartUpload(bucket, object string, uploadID Uploa return err } - result, err := g.storage.PutObject(bucket, object, upload.Meta, bytes.NewReader(fileBody), int64(len(fileBody))) + result, err := g.storage.PutObject(bucket, object, upload.Meta, nil, bytes.NewReader(fileBody), int64(len(fileBody))) if err != nil { return err } diff --git a/gofakes3_test.go b/gofakes3_test.go index 36205e6..e69481c 100644 --- a/gofakes3_test.go +++ b/gofakes3_test.go @@ -282,10 +282,11 @@ func TestCreateObjectMetadataAndObjectTagging(t *testing.T) { defer ts.Close() svc := ts.s3Client() + expectedBody := "hello" _, err := svc.PutObject(&s3.PutObjectInput{ Bucket: aws.String(defaultBucket), Key: aws.String("object"), - Body: bytes.NewReader([]byte("hello")), + Body: strings.NewReader(expectedBody), Metadata: map[string]*string{ "Test": aws.String("test"), }, @@ -316,7 +317,7 @@ func TestCreateObjectMetadataAndObjectTagging(t *testing.T) { Key: aws.String("object"), Tagging: &s3.Tagging{ TagSet: []*s3.Tag{ - {Key: aws.String("Tag-Test"), Value: aws.String("test")}, + {Key: aws.String("TagTest"), Value: aws.String("test")}, }, }, }) @@ -341,9 +342,23 @@ func TestCreateObjectMetadataAndObjectTagging(t *testing.T) { }) ts.OK(err) - if *result.TagSet[0].Key != "Tag-Test" && *result.TagSet[0].Value != "test" { + if *result.TagSet[0].Key != "TagTest" && *result.TagSet[0].Value != "test" { t.Fatalf("tag set wrong: %+v", head.Metadata) } + + // receive object after tagging + get, err := svc.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(defaultBucket), + Key: aws.String("object"), + }) + ts.OK(err) + + c, err := io.ReadAll(get.Body) + ts.OK(err) + + if string(c) != expectedBody { + t.Fatalf("body has been changed: %s", string(c)) + } } func TestCopyObject(t *testing.T) { @@ -356,7 +371,7 @@ func TestCopyObject(t *testing.T) { "X-Amz-Meta-One": "src", "X-Amz-Meta-Two": "src", } - ts.backendPutString(defaultBucket, "src-key", srcMeta, "content") + ts.backendPutString(defaultBucket, "src-key", srcMeta, nil, "content") out, err := svc.CopyObject(&s3.CopyObjectInput{ Bucket: aws.String(defaultBucket), @@ -407,7 +422,7 @@ func TestCopyObjectWithSpecialChars(t *testing.T) { } srcKey := "src+key,with special;chars!?=" content := "contents" - ts.backendPutString(defaultBucket, srcKey, srcMeta, content) + ts.backendPutString(defaultBucket, srcKey, srcMeta, nil, content) copySource := "/" + defaultBucket + "/" + url.QueryEscape(srcKey) _, err := svc.CopyObject(&s3.CopyObjectInput{ Bucket: aws.String(defaultBucket), @@ -440,7 +455,7 @@ func TestCopyObjectWithSpecialCharsEscapedInvalied(t *testing.T) { } srcKey := "src+key" //encoded srcKey = src%2Bkey content := "contents" - ts.backendPutString(defaultBucket, srcKey, srcMeta, content) + ts.backendPutString(defaultBucket, srcKey, srcMeta, nil, content) copySource := "/" + defaultBucket + "/src%2key" //invalid encoding _, err := svc.CopyObject(&s3.CopyObjectInput{ Bucket: aws.String(defaultBucket), @@ -470,7 +485,7 @@ func TestDeleteBucket(t *testing.T) { svc := ts.s3Client() ts.backendCreateBucket("test") - ts.backendPutString("test", "test", nil, "test") + ts.backendPutString("test", "test", nil, nil, "test") _, err := svc.DeleteBucket(&s3.DeleteBucketInput{ Bucket: aws.String("test"), }) @@ -503,9 +518,9 @@ func TestDeleteMulti(t *testing.T) { defer ts.Close() svc := ts.s3Client() - ts.backendPutString(defaultBucket, "foo", nil, "one") - ts.backendPutString(defaultBucket, "bar", nil, "two") - ts.backendPutString(defaultBucket, "baz", nil, "three") + ts.backendPutString(defaultBucket, "foo", nil, nil, "one") + ts.backendPutString(defaultBucket, "bar", nil, nil, "two") + ts.backendPutString(defaultBucket, "baz", nil, nil, "three") rs, err := svc.DeleteObjects(&s3.DeleteObjectsInput{ Bucket: aws.String(defaultBucket), @@ -525,9 +540,9 @@ func TestDeleteMulti(t *testing.T) { defer ts.Close() svc := ts.s3Client() - ts.backendPutString(defaultBucket, "foo", nil, "one") - ts.backendPutString(defaultBucket, "bar", nil, "two") - ts.backendPutString(defaultBucket, "baz", nil, "three") + ts.backendPutString(defaultBucket, "foo", nil, nil, "one") + ts.backendPutString(defaultBucket, "bar", nil, nil, "two") + ts.backendPutString(defaultBucket, "baz", nil, nil, "three") rs, err := svc.DeleteObjects(&s3.DeleteObjectsInput{ Bucket: aws.String(defaultBucket), @@ -549,7 +564,7 @@ func TestGetBucketLocation(t *testing.T) { defer ts.Close() svc := ts.s3Client() - ts.backendPutString(defaultBucket, "foo", nil, "one") + ts.backendPutString(defaultBucket, "foo", nil, nil, "one") out, err := svc.GetBucketLocation(&s3.GetBucketLocationInput{ Bucket: aws.String(defaultBucket), @@ -616,7 +631,7 @@ func TestGetObjectRange(t *testing.T) { ts := newTestServer(t) defer ts.Close() - ts.backendPutBytes(defaultBucket, "foo", nil, in) + ts.backendPutBytes(defaultBucket, "foo", nil, nil, in) assertRange(ts, "foo", tc.hdr, tc.expected, tc.fail) }) } @@ -647,7 +662,7 @@ func TestGetObjectRangeInvalid(t *testing.T) { ts := newTestServer(t) defer ts.Close() - ts.backendPutBytes(defaultBucket, "foo", nil, in) + ts.backendPutBytes(defaultBucket, "foo", nil, nil, in) assertRangeInvalid(ts, "foo", tc.hdr) }) } @@ -692,7 +707,7 @@ func TestGetObjectIfNoneMatch(t *testing.T) { ts := newTestServer(t) defer ts.Close() - ts.backendPutString(defaultBucket, objectKey, nil, "hello") + ts.backendPutString(defaultBucket, objectKey, nil, nil, "hello") assertModified(ts, tc.ifNoneMatch, tc.shouldModify) }) @@ -1012,7 +1027,7 @@ func TestObjectVersions(t *testing.T) { const neverVerBucket = "neverver" ts.backendCreateBucket(neverVerBucket) - ts.backendPutString(neverVerBucket, "object", nil, "body 1") + ts.backendPutString(neverVerBucket, "object", nil, nil, "body 1") list(ts, neverVerBucket, "null") // S300005 }) } @@ -1022,7 +1037,7 @@ func TestListBucketPages(t *testing.T) { keys := make([]string, n) for i := int64(0); i < n; i++ { key := fmt.Sprintf("%s%d", prefix, i) - ts.backendPutString(defaultBucket, key, nil, fmt.Sprintf("body-%d", i)) + ts.backendPutString(defaultBucket, key, nil, nil, fmt.Sprintf("body-%d", i)) keys[i] = key } return keys @@ -1127,7 +1142,7 @@ func TestListBucketPagesFallback(t *testing.T) { keys := make([]string, n) for i := int64(0); i < n; i++ { key := fmt.Sprintf("%s%d", prefix, i) - ts.backendPutString(defaultBucket, key, nil, fmt.Sprintf("body-%d", i)) + ts.backendPutString(defaultBucket, key, nil, nil, fmt.Sprintf("body-%d", i)) keys[i] = key } return keys diff --git a/init_test.go b/init_test.go index 0fb3a5b..0bbe291 100644 --- a/init_test.go +++ b/init_test.go @@ -248,14 +248,14 @@ func (ts *testServer) backendObjectExists(bucket, key string) bool { return obj != nil } -func (ts *testServer) backendPutString(bucket, key string, meta map[string]string, in string) { +func (ts *testServer) backendPutString(bucket, key string, meta map[string]string, tags map[string]string, in string) { ts.Helper() - ts.OKAll(ts.backend.PutObject(bucket, key, meta, strings.NewReader(in), int64(len(in)))) + ts.OKAll(ts.backend.PutObject(bucket, key, meta, tags, strings.NewReader(in), int64(len(in)))) } -func (ts *testServer) backendPutBytes(bucket, key string, meta map[string]string, in []byte) { +func (ts *testServer) backendPutBytes(bucket, key string, meta map[string]string, tags map[string]string, in []byte) { ts.Helper() - ts.OKAll(ts.backend.PutObject(bucket, key, meta, bytes.NewReader(in), int64(len(in)))) + ts.OKAll(ts.backend.PutObject(bucket, key, meta, tags, bytes.NewReader(in), int64(len(in)))) } func (ts *testServer) backendGetString(bucket, key string, rnge *gofakes3.ObjectRangeRequest) string { diff --git a/messages.go b/messages.go index 17138fe..533aa89 100644 --- a/messages.go +++ b/messages.go @@ -55,6 +55,16 @@ type CompletedPart struct { type CompleteMultipartUploadRequest struct { Parts []CompletedPart `xml:"Part"` } +type Tag struct { + Key string `xml:"Key"` + Value string `xml:"Value"` +} +type Tagging struct { + XMLName xml.Name `xml:"Tagging"` + TagSet struct { + Tag []Tag `xml:"Tag"` + } `xml:"TagSet"` +} func (c CompleteMultipartUploadRequest) partsAreSorted() bool { return sort.IntsAreSorted(c.partIDs()) diff --git a/routing.go b/routing.go index e139a15..e293e48 100644 --- a/routing.go +++ b/routing.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "fmt" "net/http" + "net/url" "strings" ) @@ -12,12 +13,12 @@ import ( // // URLs are assumed to break down into two common path segments, in the // following format: -// // +// +// // // // The operation for most of the core functionality is built around HTTP // verbs, but outside the core functionality, the clean separation starts // to degrade, especially around multipart uploads. -// func (g *GoFakeS3) routeBase(w http.ResponseWriter, r *http.Request) { var ( path = strings.Trim(r.URL.Path, "/") @@ -48,6 +49,9 @@ func (g *GoFakeS3) routeBase(w http.ResponseWriter, r *http.Request) { } else if _, ok := query["versioning"]; ok { err = g.routeVersioning(bucket, w, r) + } else if _, ok := query["tagging"]; ok { + err = g.routeTagging(bucket, object, query, w, r) + } else if _, ok := query["versions"]; ok { err = g.routeVersions(bucket, w, r) @@ -155,6 +159,19 @@ func (g *GoFakeS3) routeVersions(bucket string, w http.ResponseWriter, r *http.R } } +// routeTagging operates on routes that contain '?tagging' in the query string. +func (g *GoFakeS3) routeTagging(bucket, object string, query url.Values, w http.ResponseWriter, r *http.Request) error { + versionID := versionFromQuery(query["versionId"]) + switch r.Method { + case "PUT": + return g.updateObjectWithTags(bucket, object, versionID, w, r) + case "GET": + return g.getObjectTags(bucket, object, versionID, w, r) + default: + return ErrMethodNotAllowed + } +} + // routeVersion operates on routes that contain '?versionId=' in the // query string. func (g *GoFakeS3) routeVersion(bucket, object string, versionID VersionID, w http.ResponseWriter, r *http.Request) error { @@ -165,6 +182,8 @@ func (g *GoFakeS3) routeVersion(bucket, object string, versionID VersionID, w ht return g.headObject(bucket, object, versionID, w, r) case "DELETE": return g.deleteObjectVersion(bucket, object, versionID, w, r) + case "PUT": + return g.updateObjectWithTags(bucket, object, string(versionID), w, r) default: return ErrMethodNotAllowed } diff --git a/routing_test.go b/routing_test.go index cc427da..88a07ab 100644 --- a/routing_test.go +++ b/routing_test.go @@ -8,7 +8,7 @@ func TestRoutingSlashes(t *testing.T) { ts := newTestServer(t, withoutInitialBuckets()) defer ts.Close() ts.backendCreateBucket("test") - ts.backendPutString("test", "obj", nil, "yep") + ts.backendPutString("test", "obj", nil, nil, "yep") client := httpClient()