Merge pull request #85 from planetscale/connections

Testdata cleanup

go/data/query_log_parse_test.go

@@ -24,7 +24,7 @@ import (
 )
 
 func TestParseMySQLQueryLog(t *testing.T) {
-	loader := MySQLLogLoader{}.Load("../testdata/mysql.query.log")
+	loader := MySQLLogLoader{}.Load("../testdata/query-logs/mysql.query.log")
 	gotQueries, err := makeSlice(loader)
 	require.NoError(t, err)
 	for _, query := range gotQueries {
@@ -34,7 +34,7 @@ func TestParseMySQLQueryLog(t *testing.T) {
 }
 
 func TestSmallSnippet(t *testing.T) {
-	loader := MySQLLogLoader{}.Load("../testdata/mysql.small-query.log")
+	loader := MySQLLogLoader{}.Load("../testdata/query-logs/mysql.small-query.log")
 	gotQueries, err := makeSlice(loader)
 	require.NoError(t, err)
 	expected := []Query{

go/data/slow_query_log_loader_test.go

@@ -23,7 +23,7 @@ import (
 )
 
 func TestLoadSlowQueryLogWithMetadata(t *testing.T) {
-	loader := SlowQueryLogLoader{}.Load("../testdata/slow_query_log")
+	loader := SlowQueryLogLoader{}.Load("../testdata/query-logs/slow_query_log")
 	queries, err := makeSlice(loader)
 	require.NoError(t, err)
 

go/summarize/utils.go → go/data/utils.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package summarize
+package data
 
 import (
 	"encoding/json"
@@ -23,72 +23,72 @@ import (
 	"os"
 )
 
-type fileType int
+type FileType int
 
 const (
-	unknownFile fileType = iota
-	traceFile
-	keysFile
-	dbInfoFile
-	transactionFile
+	UnknownFile FileType = iota
+	TraceFile
+	KeysFile
+	DBInfoFile
+	TransactionFile
 )
 
-var fileTypeMap = map[string]fileType{ //nolint:gochecknoglobals // this is instead of a const
-	"trace":        traceFile,
-	"keys":         keysFile,
-	"dbinfo":       dbInfoFile,
-	"transactions": transactionFile,
+var fileTypeMap = map[string]FileType{ //nolint:gochecknoglobals // this is instead of a const
+	"trace":        TraceFile,
+	"keys":         KeysFile,
+	"dbinfo":       DBInfoFile,
+	"transactions": TransactionFile,
 }
 
-// getFileType reads the first key-value pair from a JSON file and returns the type of the file
+// GetFileType reads the first key-value pair from a JSON file and returns the type of the file
 // Note:
-func getFileType(filename string) (fileType, error) {
+func GetFileType(filename string) (FileType, error) {
 	file, err := os.Open(filename)
 	if err != nil {
-		return unknownFile, fmt.Errorf("error opening file: %v", err)
+		return UnknownFile, fmt.Errorf("error opening file: %v", err)
 	}
 	defer file.Close()
 
 	decoder := json.NewDecoder(file)
 
 	token, err := decoder.Token()
 	if err != nil {
-		return unknownFile, fmt.Errorf("error reading json token: %v", err)
+		return UnknownFile, fmt.Errorf("error reading json token: %v", err)
 	}
 
 	if delim, ok := token.(json.Delim); !ok || delim != '{' {
-		return unknownFile, errors.New("expected start of object '{'")
+		return UnknownFile, errors.New("expected start of object '{'")
 	}
 
 	if !decoder.More() {
-		return unknownFile, nil
+		return UnknownFile, nil
 	}
 
 	keyToken, err := decoder.Token()
 	if err != nil {
-		return unknownFile, fmt.Errorf("error reading json key token: %v", err)
+		return UnknownFile, fmt.Errorf("error reading json key token: %v", err)
 	}
 
 	key, ok := keyToken.(string)
 	if !ok {
-		return unknownFile, fmt.Errorf("expected key to be a string: %s", keyToken)
+		return UnknownFile, fmt.Errorf("expected key to be a string: %s", keyToken)
 	}
 
 	valueToken, err := decoder.Token()
 	if err != nil {
-		return unknownFile, fmt.Errorf("error reading json value token: %v", err)
+		return UnknownFile, fmt.Errorf("error reading json value token: %v", err)
 	}
 
 	if key == "fileType" {
 		s, ok := valueToken.(string)
 		if !ok {
-			return unknownFile, fmt.Errorf("expected value to be a string: %s", valueToken)
+			return UnknownFile, fmt.Errorf("expected value to be a string: %s", valueToken)
 		}
 		if fileType, ok := fileTypeMap[s]; ok {
 			return fileType, nil
 		}
-		return unknownFile, fmt.Errorf("unknown FileType: %s", valueToken)
+		return UnknownFile, fmt.Errorf("unknown FileType: %s", valueToken)
 	}
 
-	return unknownFile, nil
+	return UnknownFile, nil
 }

go/summarize/utils_test.go → go/data/utils_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package summarize
+package data
 
 import (
 	"testing"
@@ -27,27 +27,23 @@ import (
 func TestGetFileType(t *testing.T) {
 	type testCase struct {
 		filename      string
-		expectedType  fileType
+		expectedType  FileType
 		expectedError string
 	}
-	testCases := []testCase{
-		{
-			filename:     "../testdata/keys-log.json",
-			expectedType: keysFile,
-		},
-		{
-			filename:     "../testdata/sakila-dbinfo.json",
-			expectedType: dbInfoFile,
-		},
-		{
-			filename:      "../testdata/mysql.query.log",
-			expectedType:  unknownFile,
-			expectedError: "error reading token: invalid character '/' looking for beginning of value",
-		},
-	}
+	testCases := []testCase{{
+		filename:     "../testdata/keys-output/keys-log.json",
+		expectedType: KeysFile,
+	}, {
+		filename:     "../testdata/dbinfo-output/sakila-dbinfo.json",
+		expectedType: DBInfoFile,
+	}, {
+		filename:      "../testdata/query-logs/mysql.query.log",
+		expectedType:  UnknownFile,
+		expectedError: "'/' looking for beginning of value",
+	}}
 	for _, tc := range testCases {
 		t.Run(tc.filename, func(t *testing.T) {
-			ft, err := getFileType(tc.filename)
+			ft, err := GetFileType(tc.filename)
 			if tc.expectedError != "" {
 				require.ErrorContains(t, err, tc.expectedError)
 			}

go/data/vtgate_log_parse_test.go

@@ -26,19 +26,19 @@ import (
 )
 
 func TestParseVtGateQueryLogRedacted(t *testing.T) {
-	loader := VtGateLogLoader{NeedsBindVars: true}.Load("../testdata/vtgate.query.log.redacted")
+	loader := VtGateLogLoader{NeedsBindVars: true}.Load("../testdata/query-logs/vtgate.query.log.redacted")
 	_, err := makeSlice(loader)
 	require.EqualError(t, err, "line 1: query has redacted bind variables, cannot parse them")
 }
 
 func TestParseVtGateQueryLog(t *testing.T) {
-	loader := VtGateLogLoader{NeedsBindVars: true}.Load("../testdata/vtgate.query.log")
+	loader := VtGateLogLoader{NeedsBindVars: true}.Load("../testdata/query-logs/vtgate.query.log")
 	gotQueries, err := makeSlice(loader)
 	require.NoError(t, err)
 
 	require.Len(t, gotQueries, 25)
 
-	expect, err := os.ReadFile("../testdata/vtgate.query.log.parsed.bv.txt")
+	expect, err := os.ReadFile("../testdata/golden-tests/vtgate.query.log.parsed.bv.txt")
 	require.NoError(t, err)
 
 	var got []string
@@ -50,12 +50,12 @@ func TestParseVtGateQueryLog(t *testing.T) {
 }
 
 func TestParseVtGateQueryLogNoBindVars(t *testing.T) {
-	loader := VtGateLogLoader{NeedsBindVars: false}.Load("../testdata/vtgate.query.log")
+	loader := VtGateLogLoader{NeedsBindVars: false}.Load("../testdata/query-logs/vtgate.query.log")
 	gotQueries, err := makeSlice(loader)
 	require.NoError(t, err)
 	require.Len(t, gotQueries, 25)
 
-	expect, err := os.ReadFile("../testdata/vtgate.query.log.parsed.txt")
+	expect, err := os.ReadFile("../testdata/golden-tests/vtgate.query.log.parsed.txt")
 	require.NoError(t, err)
 
 	var got []string

go/dbinfo/dbinfo.go

@@ -18,11 +18,14 @@ package dbinfo
 
 import (
 	"encoding/json"
+	"errors"
 	"io"
 	"os"
 	"sort"
 
 	"vitess.io/vitess/go/mysql"
+
+	"github.com/vitessio/vt/go/data"
 )
 
 type Config struct {
@@ -211,6 +214,14 @@ func Get(cfg Config) (*Info, error) {
 }
 
 func Load(fileName string) (*Info, error) {
+	typ, err := data.GetFileType(fileName)
+	if err != nil {
+		return nil, err
+	}
+	if typ != data.DBInfoFile {
+		return nil, errors.New("file is not a dbinfo file")
+	}
+
 	b, err := os.ReadFile(fileName)
 	if err != nil {
 		return nil, err

go/dbinfo/dbinfo_test.go

@@ -28,7 +28,7 @@ import (
 )
 
 func TestDBInfoLoad(t *testing.T) {
-	si, err := Load("../testdata/sakila-dbinfo.json")
+	si, err := Load("../testdata/dbinfo-output/sakila-dbinfo.json")
 	require.NoError(t, err)
 	require.NotNil(t, si)
 
@@ -72,11 +72,11 @@ func TestDBInfoGet(t *testing.T) {
 	err := clusterInstance.StartTopo()
 	require.NoError(t, err)
 
-	schemaFile := "../testdata/sakila-schema-ddls.sql"
+	schemaFile := "../testdata/query-logs/sakila-schema-ddls.sql"
 	schemaBytes, err := os.ReadFile(schemaFile)
 	require.NoError(t, err)
 
-	dataFile := "../testdata/sakila-data.sql"
+	dataFile := "../testdata/query-logs/sakila-data.sql"
 	dataBytes, err := os.ReadFile(dataFile)
 	require.NoError(t, err)
 

go/keys/keys_test.go

@@ -41,21 +41,21 @@ func TestKeys(t *testing.T) {
 		},
 		{
 			cfg: Config{
-				FileName: "../testdata/vtgate.query.log",
+				FileName: "../testdata/query-logs/vtgate.query.log",
 				Loader:   data.VtGateLogLoader{NeedsBindVars: false},
 			},
 			expectedFile: "keys-log-vtgate.json",
 		},
 		{
 			cfg: Config{
-				FileName: "../testdata/slow_query_log",
+				FileName: "../testdata/query-logs/slow_query_log",
 				Loader:   data.SlowQueryLogLoader{},
 			},
 			expectedFile: "slow-query-log.json",
 		},
 		{
 			cfg: Config{
-				FileName: "../testdata/bigger_slow_query_log.log",
+				FileName: "../testdata/query-logs/bigger_slow_query_log.log",
 				Loader:   data.SlowQueryLogLoader{},
 			},
 			expectedFile: "bigger_slow_query_log.json",
@@ -68,7 +68,7 @@ func TestKeys(t *testing.T) {
 			err := run(sb, tcase.cfg)
 			require.NoError(t, err)
 
-			out, err := os.ReadFile("../testdata/" + tcase.expectedFile)
+			out, err := os.ReadFile("../testdata/keys-output/" + tcase.expectedFile)
 			require.NoError(t, err)
 
 			assert.Equal(t, string(out), sb.String())

go/summarize/force-graph.go

@@ -43,15 +43,15 @@ type (
 		Predicates []string `json:"predicates"`
 	}
 
-	data struct {
+	graphData struct {
 		Nodes []node `json:"nodes"`
 		Links []link `json:"links"`
 	}
 
 	forceGraphData struct {
 		maxValue   int
 		maxNumRows int
-		data
+		graphData
 	}
 )
 
@@ -203,9 +203,9 @@ var templateHTML string
 
 // Function to dynamically generate and serve index.html
 func serveIndex(w http.ResponseWriter, data forceGraphData) error {
-	dataBytes, err := json.Marshal(data.data)
+	dataBytes, err := json.Marshal(data.graphData)
 	if err != nil {
-		return fmt.Errorf("could not marshal data: %w", err)
+		return fmt.Errorf("could not marshal graphData: %w", err)
 	}
 
 	tmpl, err := template.New("index").Parse(templateHTML)

go/summarize/reading_test.go

@@ -24,7 +24,7 @@ import (
 )
 
 func TestReadTransaction(t *testing.T) {
-	file, err := readTransactionFile("../testdata/small-slow-query-transactions.json")
+	file, err := readTransactionFile("../testdata/transactions-output/small-slow-query-transactions.json")
 	require.NoError(t, err)
 	assert.NotNil(t, file)
 }

go/summarize/summarize-keys.go

@@ -192,7 +192,7 @@ func summarizeKeysQueries(summary *Summary, queries *keys.Output) error {
 	tableUsageWriteCounts := make(map[string]int)
 	tableUsageReadCounts := make(map[string]int)
 
-	// First pass: collect all data and count occurrences
+	// First pass: collect all graphData and count occurrences
 	for _, query := range queries.Queries {
 		gatherTableInfo(query, tableSummaries, tableUsageWriteCounts, tableUsageReadCounts)
 		checkQueryForHotness(&summary.hotQueries, query, summary.hotQueryFn)

go/summarize/summarize-keys_test.go

@@ -70,10 +70,10 @@ func TestSummarizeKeysFile(t *testing.T) {
 	sb := &strings.Builder{}
 	now := time.Date(2024, time.January, 1, 1, 2, 3, 0, time.UTC)
 
-	fnKeys, err := readKeysFile("../testdata/keys-log.json")
+	fnKeys, err := readKeysFile("../testdata/keys-output/keys-log.json")
 	require.NoError(t, err)
 
-	fnSchemaInfo, err := readDBInfoFile("../testdata/keys-schema-info.json")
+	fnSchemaInfo, err := readDBInfoFile("../testdata/dbinfo-output/keys-schema-info.json")
 	require.NoError(t, err)
 
 	s, err := NewSummary("")
@@ -88,7 +88,7 @@ func TestSummarizeKeysFile(t *testing.T) {
 	err = s.PrintMarkdown(sb, now)
 	require.NoError(t, err)
 
-	expected, err := os.ReadFile("../testdata/keys-summary.md")
+	expected, err := os.ReadFile("../testdata/summarize-output/keys-summary.md")
 	require.NoError(t, err)
 	assert.Equal(t, string(expected), sb.String())
 	if t.Failed() {
@@ -107,7 +107,7 @@ func TestSummarizeKeysWithHotnessFile(t *testing.T) {
 
 	for _, metric := range tests {
 		t.Run(metric, func(t *testing.T) {
-			fn, err := readKeysFile("../testdata/bigger_slow_query_log.json")
+			fn, err := readKeysFile("../testdata/keys-output/bigger_slow_query_log.json")
 			require.NoError(t, err)
 			sb := &strings.Builder{}
 			now := time.Date(2024, time.January, 1, 1, 2, 3, 0, time.UTC)
@@ -121,7 +121,7 @@ func TestSummarizeKeysWithHotnessFile(t *testing.T) {
 			err = s.PrintMarkdown(sb, now)
 			require.NoError(t, err)
 
-			expected, err := os.ReadFile(fmt.Sprintf("../testdata/bigger_slow_log_%s.md", metric))
+			expected, err := os.ReadFile(fmt.Sprintf("../testdata/summarize-output/bigger_slow_log_%s.md", metric))
 			require.NoError(t, err)
 			assert.Equal(t, string(expected), sb.String())
 			if t.Failed() {