add vtbenchstat

.gitignore

@@ -24,6 +24,8 @@ go.work
 report.xml
 errors/
 vitess-tester
+vtbenchstat
 
 # Do not ignore anything inside the src/vitess-tester directory
-!/src/vitess-tester/
+!/src/vitess-tester/
+!/src/cmd/vtbenchstat

Makefile

@@ -1,11 +1,16 @@
-.PHONY: all build test tidy clean
+.PHONY: all build test tidy clean vitess-tester vtbenchstat
 
 GO := go
 
 default: build
 
-build:
-	$(GO) build -o vitess-tester ./
+build: vitess-tester vtbenchstat
+
+vitess-tester:
+	$(GO) build -o $@ ./
+
+vtbenchstat:
+	$(GO) build -o $@ ./src/cmd/vtbenchstat
 
 test: build
 	$(GO) test -cover ./...
@@ -16,4 +21,4 @@ tidy:
 
 clean:
 	$(GO) clean -i ./...
-	rm -rf vitess-tester
+	rm -f vitess-tester vtbenchstat

go.mod

@@ -10,6 +10,8 @@ require (
 
 require (
 	github.com/jstemmer/go-junit-report/v2 v2.1.0
+	github.com/olekukonko/tablewriter v0.0.5
+	golang.org/x/term v0.22.0
 	vitess.io/vitess v0.10.3-0.20240709144253-eb29999a3f47
 )
 
@@ -19,12 +21,12 @@ require (
 	github.com/DataDog/datadog-agent/pkg/obfuscate v0.54.0 // indirect
 	github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.54.0 // indirect
 	github.com/DataDog/datadog-go/v5 v5.5.0 // indirect
-	github.com/DataDog/go-libddwaf/v2 v2.4.2 // indirect
 	github.com/DataDog/go-libddwaf/v3 v3.3.0 // indirect
 	github.com/DataDog/go-sqllexer v0.0.12 // indirect
 	github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect
 	github.com/DataDog/sketches-go v1.4.6 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/alecthomas/chroma v0.10.0 // indirect
 	github.com/aquarapid/vaultlib v0.5.1 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -33,6 +35,7 @@ require (
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/dlclark/regexp2 v1.11.4 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/ebitengine/purego v0.7.1 // indirect
 	github.com/fatih/color v1.17.0 // indirect
@@ -63,6 +66,7 @@ require (
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
@@ -81,6 +85,7 @@ require (
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sagikazarmark/locafero v0.6.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/secure-systems-lab/go-securesystemslib v0.8.0 // indirect

src/cmd/inserts/create_tpch_inserts.go

@@ -0,0 +1,129 @@
+/*
+Copyright 2024 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"math/rand"
+	"strings"
+	"time"
+)
+
+// main generates inserts for the TPC-H tables
+// The size factor is specified by the user, and the inserts are generated such that they fit
+// within the WHERE clauses of the test queries in t/tpch.test
+func main() {
+	// Set the random seed
+	rand.Seed(time.Now().UnixNano())
+
+	// Specify the size factor
+	var sizeFactor int
+	fmt.Print("Enter size factor: ")
+	fmt.Scan(&sizeFactor)
+
+	// Helper function to join values
+	joinValues := func(values []string) string {
+		return strings.Join(values, ",\n")
+	}
+
+	// Generate inserts for the region table
+	fmt.Println("")
+	var regionValues []string
+	regionValues = append(regionValues, "(1, 'ASIA', 'Eastern Asia')")
+	regionValues = append(regionValues, "(2, 'MIDDLE EAST', 'Rich cultural heritage')")
+	regionValues = append(regionValues, "(3, 'EUROPE', 'Diverse cultures')")
+	for i := 4; i <= sizeFactor; i++ {
+		regionValues = append(regionValues, fmt.Sprintf("(%d, 'Region %d', 'Comment %d')", i, i, i))
+	}
+	fmt.Printf("INSERT INTO region (R_REGIONKEY, R_NAME, R_COMMENT) VALUES\n%s;\n\n", joinValues(regionValues))
+
+	// Generate inserts for the nation table, include 'JAPAN', 'INDIA', 'EGYPT', and 'MOZAMBIQUE' for the test queries
+	fmt.Println()
+	var nationValues []string
+	nationValues = append(nationValues, "(1, 'JAPAN', 1, 'Nation with advanced technology')")
+	nationValues = append(nationValues, "(2, 'INDIA', 1, 'Nation with rich history')")
+	nationValues = append(nationValues, "(3, 'MOZAMBIQUE', 2, 'Southern African nation')")
+	nationValues = append(nationValues, "(4, 'EGYPT', 2, 'Ancient civilization')")
+	for i := 5; i <= sizeFactor*12; i++ {
+		regionKey := (i-1)%7 + 1
+		nationValues = append(nationValues, fmt.Sprintf("(%d, 'Nation %d', %d, 'Nation Comment %d')", i, i, regionKey, i))
+	}
+	fmt.Printf("INSERT INTO nation (N_NATIONKEY, N_NAME, N_REGIONKEY, N_COMMENT) VALUES\n%s;\n\n", joinValues(nationValues))
+
+	// Generate inserts for the supplier table
+	fmt.Println()
+	var supplierValues []string
+	supplierValues = append(supplierValues, "(1, 'Supplier A', '123 Square', 1, '86-123-4567', 5000.00, 'High quality steel')")
+	for i := 2; i <= sizeFactor*7; i++ {
+		nationKey := (i-1)%12 + 1
+		supplierValues = append(supplierValues, fmt.Sprintf("(%d, 'Supplier %d', 'Address %d', %d, 'Phone %d', %d, 'Supplier Comment %d')", i, i, i, nationKey, i, 5000+i*100, i))
+	}
+	fmt.Printf("INSERT INTO supplier (S_SUPPKEY, S_NAME, S_ADDRESS, S_NATIONKEY, S_PHONE, S_ACCTBAL, S_COMMENT) VALUES\n%s;\n\n", joinValues(supplierValues))
+
+	// Generate inserts for the part table
+	fmt.Println()
+	var partValues []string
+	partValues = append(partValues, "(1, 'Part dimension', 'MFGR A', 'Brand#52', 'SMALL PLATED COPPER', 3, 'SM BOX', 45.00, 'Part with special dimensions')")
+	partValues = append(partValues, "(2, 'Large Brush', 'MFGR B', 'Brand#34', 'LARGE BRUSHED COPPER', 12, 'LG BOX', 30.00, 'Brush for industrial use')")
+	for i := 3; i <= sizeFactor*5; i++ {
+		partValues = append(partValues, fmt.Sprintf("(%d, 'Part %d', 'MFGR %d', 'Brand %d', 'Type %d', %d, 'Container %d', %.2f, 'Part Comment %d')", i, i, i, i, i, rand.Intn(100), i, float64(10+i*10), i))
+	}
+	fmt.Printf("INSERT INTO part (P_PARTKEY, P_NAME, P_MFGR, P_BRAND, P_TYPE, P_SIZE, P_CONTAINER, P_RETAILPRICE, P_COMMENT) VALUES\n%s;\n\n", joinValues(partValues))
+
+	// Generate inserts for the partsupp table
+	fmt.Println()
+	var partsuppValues []string
+	for i := 1; i <= sizeFactor*10; i++ {
+		partKey := (i-1)%5 + 1
+		suppKey := (i-1)%7 + 1
+		partsuppValues = append(partsuppValues, fmt.Sprintf("(%d, %d, %d, %.2f, 'Partsupp Comment %d')", partKey, suppKey, rand.Intn(1000), float64(rand.Intn(2000))/100, i))
+	}
+	fmt.Printf("INSERT INTO partsupp (PS_PARTKEY, PS_SUPPKEY, PS_AVAILQTY, PS_SUPPLYCOST, PS_COMMENT) VALUES\n%s;\n\n", joinValues(partsuppValues))
+
+	// Generate inserts for the customer table, include 'AUTOMOBILE' segment for test queries
+	fmt.Println()
+	var customerValues []string
+	customerValues = append(customerValues, "(1, 'Customer A', '1234 Drive Lane', 1, '123-456-7890', 1000.00, 'AUTOMOBILE', 'Frequent automobile orders')")
+	for i := 2; i <= sizeFactor*5; i++ {
+		nationKey := (i-1)%12 + 1
+		customerValues = append(customerValues, fmt.Sprintf("(%d, 'Customer %d', 'Address %d', %d, 'Phone %d', %.2f, 'Segment %d', 'Customer Comment %d')", i, i, i, nationKey, i, float64(rand.Intn(20000))/100, i%5, i))
+	}
+	fmt.Printf("INSERT INTO customer (C_CUSTKEY, C_NAME, C_ADDRESS, C_NATIONKEY, C_PHONE, C_ACCTBAL, C_MKTSEGMENT, C_COMMENT) VALUES\n%s;\n\n", joinValues(customerValues))
+
+	// Generate inserts for the orders table
+	fmt.Println()
+	var orderValues []string
+	orderValues = append(orderValues, "(1, 1, 'O', 15000.00, '1995-03-12', '1-URGENT', 'Clerk#0001', 1, 'Automobile related order')")
+	for i := 2; i <= sizeFactor*5; i++ {
+		custKey := (i-1)%5 + 1
+		orderValues = append(orderValues, fmt.Sprintf("(%d, %d, 'O', %.2f, '1995-02-%02d', 'Priority %d', 'Clerk#%04d', %d, 'Order Comment %d')", i, custKey, float64(rand.Intn(50000)), rand.Intn(28)+1, rand.Intn(5)+1, i, rand.Intn(3)+1, i))
+	}
+	fmt.Printf("INSERT INTO orders (O_ORDERKEY, O_CUSTKEY, O_ORDERSTATUS, O_TOTALPRICE, O_ORDERDATE, O_ORDERPRIORITY, O_CLERK, O_SHIPPRIORITY, O_COMMENT) VALUES\n%s;\n\n", joinValues(orderValues))
+
+	// Generate inserts for the lineitem table
+	fmt.Println()
+	var lineitemValues []string
+	lineitemValues = append(lineitemValues, "(1, 1, 1, 1, 20, 5000.00, 0.05, 0.10, 'R', 'O', '1995-03-15', '1995-03-14', '1995-03-16', 'DELIVER IN PERSON', 'AIR', 'Handle with care')")
+	lineitemValues = append(lineitemValues, "(2, 1, 2, 1, 30, 10000.00, 0.06, 0.05, 'N', 'F', '1995-03-17', '1995-03-16', '1995-03-18', 'NONE', 'RAIL', 'Bulk delivery')")
+	for i := 3; i <= sizeFactor*10; i++ {
+		orderKey := (i-1)%5 + 1
+		partKey := (i-1)%5 + 1
+		suppKey := (i-1)%7 + 1
+		lineitemValues = append(lineitemValues, fmt.Sprintf("(%d, %d, %d, %d, %d, %.2f, %.2f, %.2f, 'N', 'O', '1995-03-%02d', '1995-03-%02d', '1995-03-%02d', 'DELIVER IN PERSON', 'TRUCK', 'Lineitem Comment %d')", orderKey, partKey, suppKey, i, rand.Intn(100), float64(rand.Intn(20000)), float64(rand.Intn(20))/100, float64(rand.Intn(10))/100, rand.Intn(28)+1, rand.Intn(28)+2, rand.Intn(28)+3, i))
+	}
+	fmt.Printf("INSERT INTO lineitem (L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER, L_QUANTITY, L_EXTENDEDPRICE, L_DISCOUNT, L_TAX, L_RETURNFLAG, L_LINESTATUS, L_SHIPDATE, L_COMMITDATE, L_RECEIPTDATE, L_SHIPINSTRUCT, L_SHIPMODE, L_COMMENT) VALUES\n%s;\n\n", joinValues(lineitemValues))
+}

src/cmd/vtbenchstat/main.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2024 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+)
+
+func main() {
+	flag.Parse()
+	args := flag.Args()
+
+	if len(args) < 1 || len(args) > 2 {
+		fmt.Println("Usage: vtbenchstat <trace_file1> [trace_file2]")
+		os.Exit(1)
+	}
+
+	traces := make([]TraceFile, len(args))
+	for i, arg := range args {
+		traces[i] = readTraceFile(arg)
+	}
+
+	if len(traces) == 1 {
+		printSummary(traces[0])
+	} else {
+		compareTraces(traces[0], traces[1])
+	}
+}
+

src/cmd/vtbenchstat/vtbenchstat.go

@@ -0,0 +1,277 @@
+/*
+Copyright 2024 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/alecthomas/chroma/quick"
+	"github.com/olekukonko/tablewriter"
+	"golang.org/x/term"
+	"math"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+type (
+	// TracedQuery represents the structure of each element in the JSON file
+	TracedQuery struct {
+		Trace      Trace  `json:"Trace"`
+		Query      string `json:"Query"`
+		LineNumber string `json:"LineNumber"`
+	}
+
+	// Trace represents the recursive structure of the Trace field
+	Trace struct {
+		OperatorType       string  `json:"OperatorType"`
+		Variant            string  `json:"Variant"`
+		NoOfCalls          int     `json:"NoOfCalls"`
+		AvgNumberOfRows    float64 `json:"AvgNumberOfRows"`
+		MedianNumberOfRows float64 `json:"MedianNumberOfRows"`
+		Inputs             []Trace `json:"Inputs,omitempty"`
+	}
+
+	QuerySummary struct {
+		Q TracedQuery
+		RouteCalls,
+		RowsSent,
+		RowsInMemory int
+	}
+
+	TraceFile struct {
+		Name    string
+		Queries []TracedQuery
+	}
+)
+
+func visit(trace Trace, f func(Trace)) {
+	f(trace)
+	for _, input := range trace.Inputs {
+		visit(input, f)
+	}
+}
+
+func summarizeTraces(file TraceFile) map[string]QuerySummary {
+	summary := make(map[string]QuerySummary)
+	for _, traceElement := range file.Queries {
+		summary[traceElement.Query] = summarizeTrace(traceElement)
+	}
+	return summary
+}
+
+func (trace *Trace) TotalRows() int {
+	return int(trace.AvgNumberOfRows * float64(trace.NoOfCalls))
+}
+
+func summarizeTrace(t TracedQuery) QuerySummary {
+	summary := QuerySummary{
+		Q: t,
+	}
+
+	visit(t.Trace, func(trace Trace) {
+		switch trace.OperatorType {
+		case "Route":
+			summary.RouteCalls += trace.NoOfCalls
+			summary.RowsSent += trace.TotalRows()
+		case "Sort":
+			if trace.Variant == "Memory" {
+				summary.RowsInMemory += int(trace.AvgNumberOfRows)
+			}
+		case "Join":
+			if trace.Variant == "HashJoin" {
+				// HashJoin has to keep the LHS in memory
+				summary.RowsInMemory += trace.Inputs[0].TotalRows()
+			}
+		}
+	})
+
+	return summary
+}
+
+func readTraceFile(fileName string) TraceFile {
+	// Open the JSON file
+	file, err := os.Open(fileName)
+	if err != nil {
+		panic(err.Error())
+	}
+	defer file.Close()
+
+	// Create a decoder
+	decoder := json.NewDecoder(file)
+
+	// Read the opening bracket
+	_, err = decoder.Token()
+	if err != nil {
+		panic(err.Error())
+	}
+
+	// Read the file contents
+	var queries []TracedQuery
+	for decoder.More() {
+		var element TracedQuery
+		err := decoder.Decode(&element)
+		if err != nil {
+			panic(err.Error())
+		}
+		queries = append(queries, element)
+	}
+
+	// Read the closing bracket
+	_, err = decoder.Token()
+	if err != nil {
+		panic(err.Error())
+	}
+
+	sort.Slice(queries, func(i, j int) bool {
+		a, err := strconv.Atoi(queries[i].LineNumber)
+		if err != nil {
+			return false
+		}
+		b, err := strconv.Atoi(queries[j].LineNumber)
+		if err != nil {
+			return false
+		}
+		return a < b
+	})
+
+	return TraceFile{
+		Name:    fileName,
+		Queries: queries,
+	}
+}
+
+const queryPrefix = "Query: "
+
+func limitQueryLength(query string, termWidth int) string {
+	// Process the query string
+	processedQuery := strings.ReplaceAll(query, "\n", " ") // Replace newlines with spaces
+	processedQuery = strings.TrimSpace(processedQuery)     // Trim leading/trailing spaces
+
+	// Calculate available space for query
+	availableSpace := termWidth - len(queryPrefix) - 3 // 3 for ellipsis
+
+	if len(processedQuery) > availableSpace {
+		processedQuery = processedQuery[:availableSpace] + "..."
+	}
+	return processedQuery
+}
+
+func printSummary(file TraceFile) {
+	summary := summarizeTraces(file)
+	termWidth, _, err := term.GetSize(int(os.Stdout.Fd()))
+	if err != nil {
+		termWidth = 80 // default to 80 if we can't get the terminal width
+	}
+	for _, query := range file.Queries {
+		querySummary := summary[query.Query]
+		printQuery(query, termWidth)
+		table := tablewriter.NewWriter(os.Stdout)
+		table.SetAutoFormatHeaders(false)
+		table.SetHeader([]string{"Route Calls", "Rows Sent", "Rows In Memory"})
+		table.Append([]string{strconv.Itoa(querySummary.RouteCalls), strconv.Itoa(querySummary.RowsSent), strconv.Itoa(querySummary.RowsInMemory)})
+		table.Render()
+		fmt.Println()
+	}
+}
+
+func printQuery(q TracedQuery, terminalWidth int) {
+	fmt.Printf("%s", queryPrefix)
+	err := quick.Highlight(os.Stdout, limitQueryLength(q.Query, terminalWidth), "sql", "terminal", "monokai")
+	if err != nil {
+		return
+	}
+	fmt.Printf("\nLine # %s\n", q.LineNumber)
+}
+
+const significantChangeThreshold = 10
+
+func compareTraces(file1, file2 TraceFile) {
+	summary1 := summarizeTraces(file1)
+	summary2 := summarizeTraces(file2)
+
+	termWidth, _, err := term.GetSize(int(os.Stdout.Fd()))
+	if err != nil {
+		termWidth = 80 // default to 80 if we can't get the terminal width
+	}
+
+	var significantChanges, totalQueries int
+	var totalRouteCallsChange, totalDataSentChange, totalMemoryRowsChange float64
+
+	for query, s1 := range summary1 {
+		s2, ok := summary2[query]
+		if !ok {
+			continue
+		}
+		totalQueries++
+
+		printQuery(s1.Q, termWidth)
+		table := tablewriter.NewWriter(os.Stdout)
+		table.SetHeader([]string{"Metric", file1.Name, file2.Name, "Diff", "% Change"})
+		table.SetAutoFormatHeaders(false)
+
+		routeCallsChange := compareMetric(table, "Route Calls", s1.RouteCalls, s2.RouteCalls)
+		if !math.IsNaN(routeCallsChange) {
+			totalRouteCallsChange += routeCallsChange
+		}
+
+		dataSentChange := compareMetric(table, "Rows Sent", s1.RowsSent, s2.RowsSent)
+		if !math.IsNaN(dataSentChange) {
+			totalDataSentChange += dataSentChange
+		}
+
+		memoryRowsChange := compareMetric(table, "Rows In Memory", s1.RowsInMemory, s2.RowsInMemory)
+		if !math.IsNaN(memoryRowsChange) {
+			totalMemoryRowsChange += memoryRowsChange
+		}
+
+		if math.Abs(routeCallsChange) > significantChangeThreshold || math.Abs(dataSentChange) > significantChangeThreshold {
+			significantChanges++
+		}
+
+		table.Render()
+		fmt.Println()
+	}
+
+	// Print summary
+	fmt.Println("Summary:")
+	fmt.Printf("- %d out of %d queries showed significant change\n", significantChanges, totalQueries)
+	fmt.Printf("- Average change in Route Calls: %.2f%%\n", totalRouteCallsChange/float64(totalQueries))
+	fmt.Printf("- Average change in Data Sent: %.2f%%\n", totalDataSentChange/float64(totalQueries))
+	fmt.Printf("- Average change in Rows In Memory: %.2f%%\n", totalMemoryRowsChange/float64(totalQueries))
+}
+
+func compareMetric(table *tablewriter.Table, metricName string, val1, val2 int) float64 {
+	diff := val2 - val1
+	percentChange := float64(diff) / float64(val1) * 100
+	percentChangeStr := fmt.Sprintf("%.2f%%", percentChange)
+	if math.IsInf(percentChange, 0) {
+		percentChangeStr = "∞%"
+		percentChange = 0 // To not skew the average calculation
+	}
+
+	table.Append([]string{
+		metricName,
+		strconv.Itoa(val1),
+		strconv.Itoa(val2),
+		strconv.Itoa(diff),
+		percentChangeStr,
+	})
+
+	return percentChange
+}

src/vitess-tester/tester.go

@@ -384,27 +384,12 @@ func (t *Tester) execute(query query) error {
 
 // trace writes the query and its trace (fetched from VtConn) as a JSON object into traceFile
 func (t *Tester) trace(query query) error {
-	// If there are already written traces, prepend a comma for valid JSON separation
-	if t.alreadyWrittenTraces {
-		if _, err := t.traceFile.Write([]byte(",")); err != nil {
-			return err
-		}
-	}
-
-	// Mark that at least one trace has been written
-	t.alreadyWrittenTraces = true
-
 	// Marshal the query into JSON format for safe embedding
 	queryJSON, err := json.Marshal(query.Query)
 	if err != nil {
 		return err
 	}
 
-	// Write the "Query" part of the JSON entry
-	if _, err := fmt.Fprintf(t.traceFile, `{"Query": %s, "LineNumber": "%d", "Trace": `, queryJSON, query.Line); err != nil {
-		return err
-	}
-
 	// Fetch the trace for the query using "vexplain trace"
 	rs, err := t.curr.VtConn.ExecuteFetch(fmt.Sprintf("vexplain trace %s", query.Query), 10000, false)
 	if err != nil {
@@ -417,13 +402,20 @@ func (t *Tester) trace(query query) error {
 		return err
 	}
 
-	// Write the formatted trace JSON
-	if _, err := t.traceFile.Write(prettyTrace.Bytes()); err != nil {
-		return err
+	// Construct the entire JSON entry in memory
+	var traceEntry bytes.Buffer
+	if t.alreadyWrittenTraces {
+		traceEntry.WriteString(",") // Prepend a comma if there are already written traces
 	}
+	traceEntry.WriteString(fmt.Sprintf(`{"Query": %s, "LineNumber": "%d", "Trace": `, queryJSON, query.Line))
+	traceEntry.Write(prettyTrace.Bytes()) // Add the formatted trace
+	traceEntry.WriteString("}")           // Close the JSON object
+
+	// Mark that at least one trace has been written
+	t.alreadyWrittenTraces = true
 
-	// Close the JSON object for this query/trace pair
-	if _, err := t.traceFile.Write([]byte("}")); err != nil {
+	// Write the fully constructed JSON entry to the file
+	if _, err := t.traceFile.Write(traceEntry.Bytes()); err != nil {
 		return err
 	}