conprof: fix genjidb(badger) gc doesn't work problem (#119) (#121)

component/conprof/store/store_test.go

@@ -6,10 +6,14 @@ import (
 	"io/ioutil"
 	"math/rand"
 	"os"
+	"runtime"
 	"testing"
 	"time"
 
+	"github.com/dgraph-io/badger/v3"
+	"github.com/genjidb/genji"
 	"github.com/genjidb/genji/document"
+	"github.com/genjidb/genji/engine/badgerengine"
 	"github.com/genjidb/genji/types"
 	"github.com/pingcap/ng-monitoring/component/conprof/meta"
 	"github.com/pingcap/ng-monitoring/config"
@@ -239,6 +243,78 @@ func TestGenjiDBAddColumn(t *testing.T) {
 	require.NoError(t, err)
 }
 
+func TestGenjiDBGCBug(t *testing.T) {
+	// related issue: https://github.com/genjidb/genji/issues/454
+	tmpDir, err := ioutil.TempDir(os.TempDir(), "ngm-test-genjidb-gc.*")
+	require.NoError(t, err)
+	defer func() {
+		err := os.RemoveAll(tmpDir)
+		require.NoError(t, err)
+	}()
+	opts := badger.DefaultOptions(tmpDir).
+		WithMemTableSize(1024 * 1024).
+		WithNumLevelZeroTables(1).
+		WithNumLevelZeroTablesStall(2).WithValueThreshold(128 * 1024)
+
+	engine, err := badgerengine.NewEngine(opts)
+	require.NoError(t, err)
+	db, err := genji.New(context.Background(), engine)
+	require.NoError(t, err)
+	defer db.Close()
+	badgerDB := engine.DB
+
+	sql := "CREATE TABLE IF NOT EXISTS t (ts INTEGER PRIMARY KEY, data BLOB)"
+	err = db.Exec(sql)
+	require.NoError(t, err)
+	// triage the bug in https://github.com/genjidb/genji/issues/454.
+	err = db.Exec(sql)
+	require.NoError(t, err)
+
+	ts := time.Now().Unix()
+	for i := 0; i < 1000; i++ {
+		data := mockProfile()
+		sql = "INSERT INTO t (ts, data) VALUES (?, ?)"
+		err = db.Exec(sql, int(ts)+i, data)
+		require.NoError(t, err)
+	}
+
+	sql = "DELETE FROM t WHERE ts <= ?"
+	err = db.Exec(sql, int(ts)+1000)
+	require.NoError(t, err)
+
+	// Insert 1000 kv to make sure the deleted kv will been compacted.
+	for i := 1000; i < 2000; i++ {
+		data := mockProfile()
+		sql = "INSERT INTO t (ts, data) VALUES (?, ?)"
+		err = db.Exec(sql, int(ts)+i, data)
+		require.NoError(t, err)
+	}
+
+	err = badgerDB.Flatten(runtime.NumCPU())
+	require.NoError(t, err)
+
+	keyCount := 0
+	err = badgerDB.View(func(txn *badger.Txn) error {
+		opts := badger.DefaultIteratorOptions
+		opts.AllVersions = true
+		it := txn.NewIterator(opts)
+		defer it.Close()
+		for it.Rewind(); it.Valid(); it.Next() {
+			valueSize := it.Item().ValueSize()
+			if valueSize <= 1024 {
+				continue
+			}
+			keyCount++
+		}
+		return nil
+	})
+	// Without the pr-fix, the keyCount will be 2000, since those deleted keys don't been gc.
+	require.Equal(t, keyCount, 1000)
+
+	err = db.Close()
+	require.NoError(t, err)
+}
+
 func mockProfile() []byte {
 	size := 50*1024 + rand.Intn(50*1024)
 	data := make([]byte, size)

database/timeseries/syscall_not_linux_unix.go

@@ -1,3 +1,4 @@
+//go:build !linux
 // +build !linux
 
 package timeseries

go.mod

@@ -41,3 +41,5 @@ require (
 )
 
 replace google.golang.org/grpc => google.golang.org/grpc v1.26.0
+
+replace github.com/genjidb/genji/engine/badgerengine => github.com/crazycs520/genji/engine/badgerengine v0.12.1-0.20220328082424-727a2d089bde

go.sum

@@ -215,6 +215,8 @@ github.com/cosiner/argv v0.1.0/go.mod h1:EusR6TucWKX+zFgtdUsKT2Cvg45K5rtpCcWz4hK
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/crazycs520/genji/engine/badgerengine v0.12.1-0.20220328082424-727a2d089bde h1:URhoDN38NcI0UfTnCQZQ6h1254HPAlf/rDdwCDz03dI=
+github.com/crazycs520/genji/engine/badgerengine v0.12.1-0.20220328082424-727a2d089bde/go.mod h1:VQwifaTDGx8niLOyHeVdZNJ4cCXsxV3Ub4zgJc+Bfck=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
@@ -278,8 +280,6 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/genjidb/genji v0.13.0 h1:WKjuoy4TRemCxVgcZ1/+POGKhjkDj3ZSTB/zlHmS7l0=
 github.com/genjidb/genji v0.13.0/go.mod h1:2QAT8ZovhZiEDQCaeOMI4HW5UiLm6PmXUxlUiKEuSjU=
-github.com/genjidb/genji/engine/badgerengine v0.13.0 h1:eLJKW7mWV2aIDTI8F+vck2Xetuq76TtayLz5Ed7NRko=
-github.com/genjidb/genji/engine/badgerengine v0.13.0/go.mod h1:9a5HvewPm08l82xzujrbSF2XQijTG8N2zVLp9tWbfe4=
 github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=