From d84d31dbed0f104de3e83c9596d70d58b3c58835 Mon Sep 17 00:00:00 2001 From: "jiwon.yum" Date: Sun, 24 Nov 2024 16:57:42 +0900 Subject: [PATCH 1/3] Fix Snapshot Overflow Modified to create a snapshot of files larger than 16MB and upload them to MongoDB using GridFS. --- server/backend/database/mongo/client.go | 68 ++++++++++++++++--- .../backend/database/testcases/testcases.go | 31 +++++++++ test/complex/mongo_client_test.go | 4 ++ 3 files changed, 93 insertions(+), 10 deletions(-) diff --git a/server/backend/database/mongo/client.go b/server/backend/database/mongo/client.go index de70d1308..25340f4e2 100644 --- a/server/backend/database/mongo/client.go +++ b/server/backend/database/mongo/client.go @@ -22,12 +22,14 @@ import ( "context" "errors" "fmt" + "log" "strings" gotime "time" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/gridfs" "go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/readpref" @@ -1068,21 +1070,67 @@ func (c *Client) CreateSnapshotInfo( docRefKey types.DocRefKey, doc *document.InternalDocument, ) error { + // 스냅샷 생성 snapshot, err := converter.SnapshotToBytes(doc.RootObject(), doc.AllPresences()) if err != nil { return err } - if _, err := c.collection(ColSnapshots).InsertOne(ctx, bson.M{ - "project_id": docRefKey.ProjectID, - "doc_id": docRefKey.DocID, - "server_seq": doc.Checkpoint().ServerSeq, - "lamport": doc.Lamport(), - "version_vector": doc.VersionVector(), - "snapshot": snapshot, - "created_at": gotime.Now(), - }); err != nil { - return fmt.Errorf("insert snapshot: %w", err) + // 16MB 이상이면 GridFS에 저장 + const maxSnapshotSize = 16 * 1024 * 1024 // 16MB + if len(snapshot) > maxSnapshotSize { + log.Println("16MB over!!!") + + db := c.client.Database(c.config.YorkieDatabase) + + // GridFS 버킷 생성 + bucket, err := gridfs.NewBucket(db) // MongoDB의 c.db는 데이터베이스 객체 + if err != nil { + return fmt.Errorf("failed to create GridFS bucket: %w", err) + } + + // GridFS에 파일 업로드 + uploadStream, err := bucket.OpenUploadStream(fmt.Sprintf("%s_snapshot", docRefKey.DocID)) + if err != nil { + return fmt.Errorf("failed to open GridFS upload stream: %w", err) + } + defer uploadStream.Close() + + // 스냅샷 데이터를 GridFS에 저장 + _, err = uploadStream.Write(snapshot) + if err != nil { + return fmt.Errorf("failed to write to GridFS: %w", err) + } + + // 파일의 ID (GridFS에서 파일을 식별하는 ObjectId) + fileID := uploadStream.FileID + + // GridFS에 저장된 파일 ID를 사용하여 문서 삽입 + if _, err := c.collection(ColSnapshots).InsertOne(ctx, bson.M{ + "project_id": docRefKey.ProjectID, + "doc_id": docRefKey.DocID, + "server_seq": doc.Checkpoint().ServerSeq, + "lamport": doc.Lamport(), + "version_vector": doc.VersionVector(), + "snapshot_file_id": fileID, // GridFS 파일 ID + "created_at": gotime.Now(), + }); err != nil { + return fmt.Errorf("insert snapshot info: %w", err) + } + + } else { + // 스냅샷이 16MB 이하일 경우 일반적인 컬렉션에 삽입 + if _, err := c.collection(ColSnapshots).InsertOne(ctx, bson.M{ + "project_id": docRefKey.ProjectID, + "doc_id": docRefKey.DocID, + "server_seq": doc.Checkpoint().ServerSeq, + "lamport": doc.Lamport(), + "version_vector": doc.VersionVector(), + "snapshot": snapshot, + "created_at": gotime.Now(), + }); err != nil { + return fmt.Errorf("insert snapshot: %w", err) + } } return nil diff --git a/server/backend/database/testcases/testcases.go b/server/backend/database/testcases/testcases.go index 8f363db2f..7f6f07164 100644 --- a/server/backend/database/testcases/testcases.go +++ b/server/backend/database/testcases/testcases.go @@ -1619,3 +1619,34 @@ func AssertKeys(t *testing.T, expectedKeys []key.Key, infos []*database.DocInfo) } assert.EqualValues(t, expectedKeys, keys) } + +func CreateLargeSnapshotTest(t *testing.T, db database.Database, projectID types.ID) { + t.Run("store and validate large snapshot test", func(t *testing.T) { + ctx := context.Background() + docKey := key.Key(fmt.Sprintf("tests$%s", t.Name())) + + clientInfo, _ := db.ActivateClient(ctx, projectID, t.Name()) + bytesID, _ := clientInfo.ID.Bytes() + actorID, _ := time.ActorIDFromBytes(bytesID) + docInfo, _ := db.FindDocInfoByKeyAndOwner(ctx, clientInfo.RefKey(), docKey, true) + + doc := document.New(docKey) + doc.SetActor(actorID) + + largeData := make([]byte, 16*1024*1024+1) // 16MB + 1 byte + for i := range largeData { + largeData[i] = byte('A' + (i % 26)) // A-Z 반복 + } + + assert.NoError(t, doc.Update(func(root *json.Object, p *presence.Presence) error { + root.SetBytes("largeField", largeData) + return nil + })) + + docRefKey := docInfo.RefKey() + + // 스냅샷 생성 및 오류 확인 + err := db.CreateSnapshotInfo(ctx, docRefKey, doc.InternalDocument()) + assert.NoError(t, err) + }) +} diff --git a/test/complex/mongo_client_test.go b/test/complex/mongo_client_test.go index 2fd6691fb..27d203406 100644 --- a/test/complex/mongo_client_test.go +++ b/test/complex/mongo_client_test.go @@ -171,4 +171,8 @@ func TestClientWithShardedDB(t *testing.T) { assert.Equal(t, docInfo1.Key, result.Key) assert.Equal(t, docInfo1.ID, result.ID) }) + + t.Run("CreateLargeSnapshotTest test", func(t *testing.T) { + testcases.CreateLargeSnapshotTest(t, cli, dummyProjectID) + }) } From e99ab4c0c9563509366376df64b293f326efc65e Mon Sep 17 00:00:00 2001 From: "jiwon.yum" Date: Sun, 24 Nov 2024 17:15:05 +0900 Subject: [PATCH 2/3] Fixed Typo --- server/backend/database/mongo/client.go | 13 +++---------- server/backend/database/testcases/testcases.go | 7 +++---- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/server/backend/database/mongo/client.go b/server/backend/database/mongo/client.go index 25340f4e2..cc9e875e6 100644 --- a/server/backend/database/mongo/client.go +++ b/server/backend/database/mongo/client.go @@ -1070,56 +1070,49 @@ func (c *Client) CreateSnapshotInfo( docRefKey types.DocRefKey, doc *document.InternalDocument, ) error { - // 스냅샷 생성 snapshot, err := converter.SnapshotToBytes(doc.RootObject(), doc.AllPresences()) if err != nil { return err } - // 16MB 이상이면 GridFS에 저장 const maxSnapshotSize = 16 * 1024 * 1024 // 16MB if len(snapshot) > maxSnapshotSize { log.Println("16MB over!!!") db := c.client.Database(c.config.YorkieDatabase) - // GridFS 버킷 생성 - bucket, err := gridfs.NewBucket(db) // MongoDB의 c.db는 데이터베이스 객체 + // create GridFS bucket + bucket, err := gridfs.NewBucket(db) if err != nil { return fmt.Errorf("failed to create GridFS bucket: %w", err) } - // GridFS에 파일 업로드 uploadStream, err := bucket.OpenUploadStream(fmt.Sprintf("%s_snapshot", docRefKey.DocID)) if err != nil { return fmt.Errorf("failed to open GridFS upload stream: %w", err) } defer uploadStream.Close() - // 스냅샷 데이터를 GridFS에 저장 _, err = uploadStream.Write(snapshot) if err != nil { return fmt.Errorf("failed to write to GridFS: %w", err) } - // 파일의 ID (GridFS에서 파일을 식별하는 ObjectId) fileID := uploadStream.FileID - // GridFS에 저장된 파일 ID를 사용하여 문서 삽입 if _, err := c.collection(ColSnapshots).InsertOne(ctx, bson.M{ "project_id": docRefKey.ProjectID, "doc_id": docRefKey.DocID, "server_seq": doc.Checkpoint().ServerSeq, "lamport": doc.Lamport(), "version_vector": doc.VersionVector(), - "snapshot_file_id": fileID, // GridFS 파일 ID + "snapshot_file_id": fileID, // GridFS file ID "created_at": gotime.Now(), }); err != nil { return fmt.Errorf("insert snapshot info: %w", err) } } else { - // 스냅샷이 16MB 이하일 경우 일반적인 컬렉션에 삽입 if _, err := c.collection(ColSnapshots).InsertOne(ctx, bson.M{ "project_id": docRefKey.ProjectID, "doc_id": docRefKey.DocID, diff --git a/server/backend/database/testcases/testcases.go b/server/backend/database/testcases/testcases.go index 7f6f07164..43d958766 100644 --- a/server/backend/database/testcases/testcases.go +++ b/server/backend/database/testcases/testcases.go @@ -1620,7 +1620,7 @@ func AssertKeys(t *testing.T, expectedKeys []key.Key, infos []*database.DocInfo) assert.EqualValues(t, expectedKeys, keys) } -func CreateLargeSnapshotTest(t *testing.T, db database.Database, projectID types.ID) { +func RunCreateLargeSnapshotTest(t *testing.T, db database.Database, projectID types.ID) { t.Run("store and validate large snapshot test", func(t *testing.T) { ctx := context.Background() docKey := key.Key(fmt.Sprintf("tests$%s", t.Name())) @@ -1633,9 +1633,9 @@ func CreateLargeSnapshotTest(t *testing.T, db database.Database, projectID types doc := document.New(docKey) doc.SetActor(actorID) - largeData := make([]byte, 16*1024*1024+1) // 16MB + 1 byte + largeData := make([]byte, 16*1024*1024+1) for i := range largeData { - largeData[i] = byte('A' + (i % 26)) // A-Z 반복 + largeData[i] = byte('A' + (i % 26)) } assert.NoError(t, doc.Update(func(root *json.Object, p *presence.Presence) error { @@ -1645,7 +1645,6 @@ func CreateLargeSnapshotTest(t *testing.T, db database.Database, projectID types docRefKey := docInfo.RefKey() - // 스냅샷 생성 및 오류 확인 err := db.CreateSnapshotInfo(ctx, docRefKey, doc.InternalDocument()) assert.NoError(t, err) }) From cc026bbea3bad235c0662356d30a46ac08527124 Mon Sep 17 00:00:00 2001 From: "jiwon.yum" Date: Tue, 26 Nov 2024 02:11:03 +0900 Subject: [PATCH 3/3] Reflect a Code Review BSON max snapshot size variable extract as static Remove unnecessary log --- server/backend/database/mongo/client.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/server/backend/database/mongo/client.go b/server/backend/database/mongo/client.go index cc9e875e6..3de8c6fac 100644 --- a/server/backend/database/mongo/client.go +++ b/server/backend/database/mongo/client.go @@ -22,7 +22,6 @@ import ( "context" "errors" "fmt" - "log" "strings" gotime "time" @@ -45,7 +44,8 @@ import ( const ( // StatusKey is the key of the status field. - StatusKey = "status" + StatusKey = "status" + BSONMaxSnapshotSize = 16 * 1024 * 1024 // 16MB ) // Client is a client that connects to Mongo DB and reads or saves Yorkie data. @@ -1075,10 +1075,7 @@ func (c *Client) CreateSnapshotInfo( return err } - const maxSnapshotSize = 16 * 1024 * 1024 // 16MB - if len(snapshot) > maxSnapshotSize { - log.Println("16MB over!!!") - + if len(snapshot) > BSONMaxSnapshotSize { db := c.client.Database(c.config.YorkieDatabase) // create GridFS bucket