From 359049391091e74e134ada571c05625155b2cda6 Mon Sep 17 00:00:00 2001 From: Aaron Son Date: Thu, 6 Mar 2025 15:45:32 -0800 Subject: [PATCH 1/2] TEMP COMMIT: Benchmark quick and dirty bytes.Buffer conversion. --- go/cmd/dolt/commands/query_diff.go | 8 ++++---- go/go.mod | 4 +--- go/go.sum | 10 ++++------ go/libraries/doltcore/sqle/sqlutil/sql_row.go | 12 +++++++++--- .../doltcore/table/typed/json/writer.go | 19 +++++++++++++++---- go/performance/microsysbench/sysbench_test.go | 10 ++++++++-- 6 files changed, 41 insertions(+), 22 deletions(-) diff --git a/go/cmd/dolt/commands/query_diff.go b/go/cmd/dolt/commands/query_diff.go index 146a4a32246..69c894669fc 100644 --- a/go/cmd/dolt/commands/query_diff.go +++ b/go/cmd/dolt/commands/query_diff.go @@ -70,8 +70,8 @@ func (q QueryDiff) ArgParser() *argparser.ArgParser { func (q QueryDiff) compareRows(pkOrds []int, row1, row2 sql.Row) (int, bool) { var cmp int for _, pkOrd := range pkOrds { - pk1, _ := gmstypes.ConvertToString(row1[pkOrd], gmstypes.Text, nil) - pk2, _ := gmstypes.ConvertToString(row2[pkOrd], gmstypes.Text, nil) + pk1, _ := gmstypes.ConvertToString(row1[pkOrd], gmstypes.Text) + pk2, _ := gmstypes.ConvertToString(row2[pkOrd], gmstypes.Text) if pk1 < pk2 { cmp = -1 } else if pk1 > pk2 { @@ -82,8 +82,8 @@ func (q QueryDiff) compareRows(pkOrds []int, row1, row2 sql.Row) (int, bool) { } var diff bool for i := 0; i < len(row1); i++ { - a, _ := gmstypes.ConvertToString(row1[i], gmstypes.Text, nil) - b, _ := gmstypes.ConvertToString(row2[i], gmstypes.Text, nil) + a, _ := gmstypes.ConvertToString(row1[i], gmstypes.Text) + b, _ := gmstypes.ConvertToString(row2[i], gmstypes.Text) if a != b { diff = true break diff --git a/go/go.mod b/go/go.mod index 3393538b19e..69d6edb7824 100644 --- a/go/go.mod +++ b/go/go.mod @@ -56,9 +56,8 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.19.1-0.20250305230031-14a57e076a0a + github.com/dolthub/go-mysql-server v0.19.1-0.20250307180035-896fdb378db1 github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 - github.com/dolthub/swiss v0.1.0 github.com/esote/minmaxheap v1.0.0 github.com/goccy/go-json v0.10.2 github.com/google/btree v1.1.2 @@ -111,7 +110,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dolthub/go-icu-regex v0.0.0-20250303123116-549b8d7cad00 // indirect github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71 // indirect - github.com/dolthub/maphash v0.0.0-20221220182448-74e1e1ea1577 // indirect github.com/go-fonts/liberation v0.2.0 // indirect github.com/go-kit/kit v0.10.0 // indirect github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81 // indirect diff --git a/go/go.sum b/go/go.sum index 702c9996da9..f8e83b434c4 100644 --- a/go/go.sum +++ b/go/go.sum @@ -181,18 +181,18 @@ github.com/dolthub/go-icu-regex v0.0.0-20250303123116-549b8d7cad00 h1:rh2ij2yTYK github.com/dolthub/go-icu-regex v0.0.0-20250303123116-549b8d7cad00/go.mod h1:ylU4XjUpsMcvl/BKeRRMXSH7e7WBrPXdSLvnRJYrxEA= github.com/dolthub/go-mysql-server v0.19.1-0.20250305230031-14a57e076a0a h1:lemFIUt0NCKIeX7vnU2yKF8UIgc0DT8zIoEUn7oy+60= github.com/dolthub/go-mysql-server v0.19.1-0.20250305230031-14a57e076a0a/go.mod h1:yr+Vv47/YLOKMgiEY+QxHTlbIVpTuiVtkEZ5l+xruY4= +github.com/dolthub/go-mysql-server v0.19.1-0.20250306232711-0f5daa0f74ad h1:BwhzrefjW44+c1IY8VbwYkRw46VBUiVijGjmCoKTViQ= +github.com/dolthub/go-mysql-server v0.19.1-0.20250306232711-0f5daa0f74ad/go.mod h1:yr+Vv47/YLOKMgiEY+QxHTlbIVpTuiVtkEZ5l+xruY4= +github.com/dolthub/go-mysql-server v0.19.1-0.20250307180035-896fdb378db1 h1:4tXRljpWgZ+b9JnhOtYz8Ja1S4bzApiO5ChWHCLQK7U= +github.com/dolthub/go-mysql-server v0.19.1-0.20250307180035-896fdb378db1/go.mod h1:yr+Vv47/YLOKMgiEY+QxHTlbIVpTuiVtkEZ5l+xruY4= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q= github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 h1:lT7hE5k+0nkBdj/1UOSFwjWpNxf+LCApbRHgnCA17XE= github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto= github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71 h1:bMGS25NWAGTEtT5tOBsCuCrlYnLRKpbJVJkDbrTRhwQ= github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71/go.mod h1:2/2zjLQ/JOOSbbSboojeg+cAwcRV0fDLzIiWch/lhqI= -github.com/dolthub/maphash v0.0.0-20221220182448-74e1e1ea1577 h1:SegEguMxToBn045KRHLIUlF2/jR7Y2qD6fF+3tdOfvI= -github.com/dolthub/maphash v0.0.0-20221220182448-74e1e1ea1577/go.mod h1:gkg4Ch4CdCDu5h6PMriVLawB7koZ+5ijb9puGMV50a4= github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 h1:7/v8q9XGFa6q5Ap4Z/OhNkAMBaK5YeuEzwJt+NZdhiE= github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81/go.mod h1:siLfyv2c92W1eN/R4QqG/+RjjX5W2+gCTRjZxBjI3TY= -github.com/dolthub/swiss v0.1.0 h1:EaGQct3AqeP/MjASHLiH6i4TAmgbG/c4rA6a1bzCOPc= -github.com/dolthub/swiss v0.1.0/go.mod h1:BeucyB08Vb1G9tumVN3Vp/pyY4AMUnr9p7Rz7wJ7kAQ= github.com/dolthub/vitess v0.0.0-20250304211657-920ca9ec2b9a h1:HIH9g4z+yXr4DIFyT6L5qOIEGJ1zVtlj6baPyHAG4Yw= github.com/dolthub/vitess v0.0.0-20250304211657-920ca9ec2b9a/go.mod h1:1gQZs/byeHLMSul3Lvl3MzioMtOW1je79QYGyi2fd70= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -670,8 +670,6 @@ github.com/tealeg/xlsx v1.0.5 h1:+f8oFmvY8Gw1iUXzPk+kz+4GpbDZPK1FhPiQRd+ypgE= github.com/tealeg/xlsx v1.0.5/go.mod h1:btRS8dz54TDnvKNosuAqxrM1QgN1udgk9O34bDCnORM= github.com/tetratelabs/wazero v1.8.2 h1:yIgLR/b2bN31bjxwXHD8a3d+BogigR952csSDdLYEv4= github.com/tetratelabs/wazero v1.8.2/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs= -github.com/thepudds/swisstable v0.0.0-20221011152303-9c77dc657777 h1:5u+6YWU2faS+Sr/x8j9yalMpSDUkatNOZWXV3wMUCGQ= -github.com/thepudds/swisstable v0.0.0-20221011152303-9c77dc657777/go.mod h1:4af3KxEsswy6aTzsTcwa8QZUSh4V+80oHdp1QX9uJHA= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= diff --git a/go/libraries/doltcore/sqle/sqlutil/sql_row.go b/go/libraries/doltcore/sqle/sqlutil/sql_row.go index 7ef3eeebe1c..6c636aedff9 100644 --- a/go/libraries/doltcore/sqle/sqlutil/sql_row.go +++ b/go/libraries/doltcore/sqle/sqlutil/sql_row.go @@ -15,6 +15,7 @@ package sqlutil import ( + "bytes" "context" "errors" "fmt" @@ -228,6 +229,7 @@ var sqlColToStrContext = sql.NewEmptyContext() // SqlColToStr is a utility function for converting a sql column of type interface{} to a string. // NULL values are treated as empty strings. Handle nil separately if you require other behavior. func SqlColToStr(sqlType sql.Type, col interface{}) (string, error) { + var buf bytes.Buffer if col != nil { switch typedCol := col.(type) { case bool: @@ -237,17 +239,21 @@ func SqlColToStr(sqlType sql.Type, col interface{}) (string, error) { return "false", nil } case sql.SpatialColumnType: - res, err := sqlType.SQL(sqlColToStrContext, nil, col) - hexRes := fmt.Sprintf("0x%X", res.Raw()) + bufres, err := sqlType.SQL(sqlColToStrContext, &buf, col) if err != nil { return "", err } + res := bufres.ToValue(buf.Bytes()) + hexRes := fmt.Sprintf("0x%X", res.Raw()) + if err != nil { + } return hexRes, nil default: - res, err := sqlType.SQL(sqlColToStrContext, nil, col) + bufres, err := sqlType.SQL(sqlColToStrContext, &buf, col) if err != nil { return "", err } + res := bufres.ToValue(buf.Bytes()) return res.ToString(), nil } } diff --git a/go/libraries/doltcore/table/typed/json/writer.go b/go/libraries/doltcore/table/typed/json/writer.go index 64ce808e690..3c329bed5d7 100644 --- a/go/libraries/doltcore/table/typed/json/writer.go +++ b/go/libraries/doltcore/table/typed/json/writer.go @@ -16,6 +16,7 @@ package json import ( "bufio" + "bytes" "context" "encoding/json" "errors" @@ -133,6 +134,7 @@ func (j *RowWriter) WriteSqlRow(ctx context.Context, row sql.Row) error { func (j *RowWriter) jsonDataForSchema(row sql.Row) ([]byte, error) { allCols := j.sch.GetAllCols() colValMap := make(map[string]interface{}, allCols.Size()) + var buf bytes.Buffer if err := allCols.Iter(func(tag uint64, col schema.Column) (stop bool, err error) { val := row[allCols.TagToIdx[tag]] if val == nil { @@ -149,17 +151,21 @@ func (j *RowWriter) jsonDataForSchema(row sql.Row) ([]byte, error) { typeinfo.TupleTypeIdentifier, typeinfo.UuidTypeIdentifier, typeinfo.VarBinaryTypeIdentifier: - sqlVal, err := col.TypeInfo.ToSqlType().SQL(sqlContext, nil, val) + bufVal, err := col.TypeInfo.ToSqlType().SQL(sqlContext, &buf, val) if err != nil { return true, err } + sqlVal := bufVal.ToValue(buf.Bytes()) val = sqlVal.ToString() + buf.Reset() case typeinfo.JSONTypeIdentifier: - sqlVal, err := col.TypeInfo.ToSqlType().SQL(sqlContext, nil, val) + bufVal, err := col.TypeInfo.ToSqlType().SQL(sqlContext, &buf, val) if err != nil { return true, err } + sqlVal := bufVal.ToValue(buf.Bytes()) str := sqlVal.ToString() + buf.Reset() // This is kind of silly: we are unmarshalling JSON just to marshall it back again // But it makes marshalling much simpler @@ -193,6 +199,7 @@ func (j *RowWriter) jsonDataForSchema(row sql.Row) ([]byte, error) { // jsonDataForSqlSchema returns a JSON representation of the given row, using the sql schema for serialization hints func (j *RowWriter) jsonDataForSqlSchema(row sql.Row) ([]byte, error) { colValMap := make(map[string]interface{}, len(j.sqlSch)) + var buf bytes.Buffer for i, col := range j.sqlSch { val := row[i] if val == nil { @@ -206,17 +213,21 @@ func (j *RowWriter) jsonDataForSqlSchema(row sql.Row) ([]byte, error) { sql.StringType, sql.SetType, types.TupleType: - sqlVal, err := col.Type.SQL(sqlContext, nil, val) + bufVal, err := col.Type.SQL(sqlContext, &buf, val) if err != nil { return nil, err } + sqlVal := bufVal.ToValue(buf.Bytes()) val = sqlVal.ToString() + buf.Reset() case types.JsonType: - sqlVal, err := col.Type.SQL(sqlContext, nil, val) + bufVal, err := col.Type.SQL(sqlContext, &buf, val) if err != nil { return nil, err } + sqlVal := bufVal.ToValue(buf.Bytes()) str := sqlVal.ToString() + buf.Reset() // This is kind of silly: we are unmarshalling JSON just to marshall it back again // But it makes marshalling much simpler diff --git a/go/performance/microsysbench/sysbench_test.go b/go/performance/microsysbench/sysbench_test.go index ffe312213c7..dcb4d819c57 100644 --- a/go/performance/microsysbench/sysbench_test.go +++ b/go/performance/microsysbench/sysbench_test.go @@ -15,6 +15,7 @@ package microsysbench import ( + "bytes" "context" "fmt" "io" @@ -26,6 +27,7 @@ import ( "github.com/dolthub/go-mysql-server/server" "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/vitess/go/sqltypes" "github.com/stretchr/testify/require" "github.com/dolthub/dolt/go/cmd/dolt/commands" @@ -125,14 +127,18 @@ func benchmarkSysbenchQuery(b *testing.B, getQuery func(int) string) { schema, iter, _, err := eng.Query(ctx, getQuery(i)) require.NoError(b, err) i := 0 - buf := sql.NewByteBuffer(16000) + var buf bytes.Buffer for { i++ row, err := iter.Next(ctx) if err != nil { break } - outputRow, err := server.RowToSQL(ctx, schema, row, nil, buf) + bufRow, err := server.RowToSQL(ctx, schema, row, nil, &buf) + outputRow := make([]sqltypes.Value, len(bufRow)) + for i := range bufRow { + outputRow[i] = bufRow[i].ToValue(buf.Bytes()) + } _ = outputRow if i%128 == 0 { buf.Reset() From 60feb6879b65104a947e8e2a8f8bdea9f70120ca Mon Sep 17 00:00:00 2001 From: Aaron Son Date: Fri, 7 Mar 2025 11:48:26 -0800 Subject: [PATCH 2/2] Bump GMS, further allocation tweaks. --- go/go.mod | 2 +- go/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go/go.mod b/go/go.mod index 69d6edb7824..084918d57fa 100644 --- a/go/go.mod +++ b/go/go.mod @@ -56,7 +56,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.19.1-0.20250307180035-896fdb378db1 + github.com/dolthub/go-mysql-server v0.19.1-0.20250307194723-790e7986bdf2 github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 github.com/esote/minmaxheap v1.0.0 github.com/goccy/go-json v0.10.2 diff --git a/go/go.sum b/go/go.sum index f8e83b434c4..ccfcf1f5829 100644 --- a/go/go.sum +++ b/go/go.sum @@ -185,6 +185,8 @@ github.com/dolthub/go-mysql-server v0.19.1-0.20250306232711-0f5daa0f74ad h1:Bwhz github.com/dolthub/go-mysql-server v0.19.1-0.20250306232711-0f5daa0f74ad/go.mod h1:yr+Vv47/YLOKMgiEY+QxHTlbIVpTuiVtkEZ5l+xruY4= github.com/dolthub/go-mysql-server v0.19.1-0.20250307180035-896fdb378db1 h1:4tXRljpWgZ+b9JnhOtYz8Ja1S4bzApiO5ChWHCLQK7U= github.com/dolthub/go-mysql-server v0.19.1-0.20250307180035-896fdb378db1/go.mod h1:yr+Vv47/YLOKMgiEY+QxHTlbIVpTuiVtkEZ5l+xruY4= +github.com/dolthub/go-mysql-server v0.19.1-0.20250307194723-790e7986bdf2 h1:cuH+LJDk8Fvf2x2lPVsDr48r8uCGsy3qe7eWEUkvEM0= +github.com/dolthub/go-mysql-server v0.19.1-0.20250307194723-790e7986bdf2/go.mod h1:yr+Vv47/YLOKMgiEY+QxHTlbIVpTuiVtkEZ5l+xruY4= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q= github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 h1:lT7hE5k+0nkBdj/1UOSFwjWpNxf+LCApbRHgnCA17XE=