From 7b2ae6ea3a6d961d567f0b018d526e3fbff50b8d Mon Sep 17 00:00:00 2001 From: sheyanjie Date: Fri, 15 Dec 2023 12:50:47 +0800 Subject: [PATCH 1/3] fix sql too long --- api/audit.go | 52 +++++++++++++++++++++++++++++++++++++---------- api/audit_test.go | 7 +++---- 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/api/audit.go b/api/audit.go index 78f8ea8..fa9e636 100644 --- a/api/audit.go +++ b/api/audit.go @@ -9,6 +9,8 @@ import ( "net/http" "strings" + // "sync" + "github.com/gin-gonic/gin" "github.com/taosdata/taoskeeper/db" "github.com/taosdata/taoskeeper/infrastructure/config" @@ -19,6 +21,11 @@ var auditLogger = log.GetLogger("audit") const MAX_DETAIL_LEN = 50000 +const MAX_SQL_LEN = 1000 * 1000 + +// var idMap = sync.Map{} +// var duplicateCount = 0 + type Audit struct { username string password string @@ -103,18 +110,14 @@ func (a *Audit) handleBatchFunc() gin.HandlerFunc { return } - sql := parseBatchSql(auditArray.Records) + err = handleBatchRecord(auditArray.Records, a.conn) + if err != nil { - auditLogger.WithError(err).Errorf("## parse sql error") - c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("timestamp format error. %s", err)}) + auditLogger.WithError(err).Errorf("## process records error") + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("process records error. %s", err)}) return } - if _, err = a.conn.Exec(context.Background(), sql); err != nil { - auditLogger.WithError(err).Error("##save audit data error", "sql", sql) - c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("save audit data error: %s", err)}) - return - } c.JSON(http.StatusOK, gin.H{}) } } @@ -179,21 +182,48 @@ func parseSql(audit AuditInfo) string { getTableName(audit), audit.ClusterID, audit.Timestamp, audit.User, audit.Operation, audit.Db, audit.Resource, audit.ClientAdd, details) } -func parseBatchSql(auditArray []AuditInfo) string { +func handleBatchRecord(auditArray []AuditInfo, conn *db.Connector) error { + var builder strings.Builder var head = fmt.Sprintf( "insert into %s using operations tags ('%s') values", getTableName(auditArray[0]), auditArray[0].ClusterID) - builder.WriteString(head) + builder.WriteString(head) for _, audit := range auditArray { + // if _, exists := idMap.Load(audit.Timestamp); exists { + // duplicateCount += 1 + // } + // idMap.Store(audit.Timestamp, true) + details := handleDetails(audit.Details) varluesStr := fmt.Sprintf( "(%s, '%s', '%s', '%s', '%s', '%s', '%s') ", audit.Timestamp, audit.User, audit.Operation, audit.Db, audit.Resource, audit.ClientAdd, details) + + if (builder.Len() + len(varluesStr)) > MAX_SQL_LEN { + sql := builder.String() + if _, err := conn.Exec(context.Background(), sql); err != nil { + return err + } + builder.Reset() + builder.WriteString(head) + } builder.WriteString(varluesStr) } - return builder.String() + + if builder.Len() > len(head) { + sql := builder.String() + if _, err := conn.Exec(context.Background(), sql); err != nil { + return err + } + } + + // if duplicateCount > 0 { + // auditLogger.Error("## Duplicate total: ", duplicateCount) + // } + + return nil } func getTableName(audit AuditInfo) string { diff --git a/api/audit_test.go b/api/audit_test.go index 35d95af..fbd5d41 100644 --- a/api/audit_test.go +++ b/api/audit_test.go @@ -89,8 +89,7 @@ func TestAudit(t *testing.T) { } // test audit batch - input := `{"records": [{"timestamp": "1699839716440000000", "cluster_id": "cluster_id_batch", "user": "user", "operation": "operation", "db":"dbnamea", "resource":"resourcenamea", "client_add": "localhost:30000", "details": "detail"},` + - `{"timestamp": "1699839716441000000", "cluster_id": "cluster_id_batch", "user": "user", "operation": "operation", "db":"dbnamea", "resource":"resourcenamea", "client_add": "localhost:30000", "details": "detail"}]}` + input := `{"records":[{"timestamp":"1702548856940013848","cluster_id":"8468922059162439502","user":"root","operation":"createTable","client_add":"173.50.0.7:45166","db":"test","resource":"","details":"d630302"},{"timestamp":"1702548856939746458","cluster_id":"8468922059162439502","user":"root","operation":"createTable","client_add":"173.50.0.7:45230","db":"test","resource":"","details":"d130277"},{"timestamp":"1702548856939586665","cluster_id":"8468922059162439502","user":"root","operation":"createTable","client_add":"173.50.0.7:50288","db":"test","resource":"","details":"d5268"},{"timestamp":"1702548856939528940","cluster_id":"8468922059162439502","user":"root","operation":"createTable","client_add":"173.50.0.7:50222","db":"test","resource":"","details":"d255282"},{"timestamp":"1702548856939336371","cluster_id":"8468922059162439502","user":"root","operation":"createTable","client_add":"173.50.0.7:45126","db":"test","resource":"","details":"d755297"},{"timestamp":"1702548856939075131","cluster_id":"8468922059162439502","user":"root","operation":"createTable","client_add":"173.50.0.7:45122","db":"test","resource":"","details":"d380325"},{"timestamp":"1702548856938640661","cluster_id":"8468922059162439502","user":"root","operation":"createTable","client_add":"173.50.0.7:45152","db":"test","resource":"","details":"d255281"},{"timestamp":"1702548856938505795","cluster_id":"8468922059162439502","user":"root","operation":"createTable","client_add":"173.50.0.7:45122","db":"test","resource":"","details":"d130276"},{"timestamp":"1702548856938363319","cluster_id":"8468922059162439502","user":"root","operation":"createTable","client_add":"173.50.0.7:45178","db":"test","resource":"","details":"d755296"},{"timestamp":"1702548856938201478","cluster_id":"8468922059162439502","user":"root","operation":"createTable","client_add":"173.50.0.7:45166","db":"test","resource":"","details":"d380324"},{"timestamp":"1702548856937740618","cluster_id":"8468922059162439502","user":"root","operation":"createTable","client_add":"173.50.0.7:50288","db":"test","resource":"","details":"d5266"}]}` t.Run("testbatch", func(t *testing.T) { @@ -109,8 +108,8 @@ func TestAudit(t *testing.T) { router.ServeHTTP(w, req) assert.Equal(t, 200, w.Code) - data, err := conn.Query(context.Background(), "select ts, details from audit.operations where cluster_id='cluster_id_batch'") + data, err := conn.Query(context.Background(), "select ts, details from audit.operations where cluster_id='8468922059162439502'") assert.NoError(t, err) - assert.Equal(t, 2, len(data.Data)) + assert.Equal(t, 11, len(data.Data)) }) } From 28c2f6db862bc9c805bbf969bece56200db79433 Mon Sep 17 00:00:00 2001 From: sheyanjie Date: Fri, 15 Dec 2023 16:21:03 +0800 Subject: [PATCH 2/3] remove comment --- api/audit.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/api/audit.go b/api/audit.go index fa9e636..f69981b 100644 --- a/api/audit.go +++ b/api/audit.go @@ -9,8 +9,6 @@ import ( "net/http" "strings" - // "sync" - "github.com/gin-gonic/gin" "github.com/taosdata/taoskeeper/db" "github.com/taosdata/taoskeeper/infrastructure/config" @@ -23,9 +21,6 @@ const MAX_DETAIL_LEN = 50000 const MAX_SQL_LEN = 1000 * 1000 -// var idMap = sync.Map{} -// var duplicateCount = 0 - type Audit struct { username string password string @@ -191,10 +186,6 @@ func handleBatchRecord(auditArray []AuditInfo, conn *db.Connector) error { builder.WriteString(head) for _, audit := range auditArray { - // if _, exists := idMap.Load(audit.Timestamp); exists { - // duplicateCount += 1 - // } - // idMap.Store(audit.Timestamp, true) details := handleDetails(audit.Details) varluesStr := fmt.Sprintf( @@ -219,10 +210,6 @@ func handleBatchRecord(auditArray []AuditInfo, conn *db.Connector) error { } } - // if duplicateCount > 0 { - // auditLogger.Error("## Duplicate total: ", duplicateCount) - // } - return nil } From e33ee9337ff14d60eca1c91cedacce027e0323b1 Mon Sep 17 00:00:00 2001 From: sheyanjie Date: Fri, 15 Dec 2023 16:45:14 +0800 Subject: [PATCH 3/3] fix misspelling --- api/audit.go | 8 ++++---- api/audit_test.go | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/api/audit.go b/api/audit.go index f69981b..3a84bc2 100644 --- a/api/audit.go +++ b/api/audit.go @@ -19,7 +19,7 @@ var auditLogger = log.GetLogger("audit") const MAX_DETAIL_LEN = 50000 -const MAX_SQL_LEN = 1000 * 1000 +var MAX_SQL_LEN = 1000 * 1000 type Audit struct { username string @@ -188,11 +188,11 @@ func handleBatchRecord(auditArray []AuditInfo, conn *db.Connector) error { for _, audit := range auditArray { details := handleDetails(audit.Details) - varluesStr := fmt.Sprintf( + valuesStr := fmt.Sprintf( "(%s, '%s', '%s', '%s', '%s', '%s', '%s') ", audit.Timestamp, audit.User, audit.Operation, audit.Db, audit.Resource, audit.ClientAdd, details) - if (builder.Len() + len(varluesStr)) > MAX_SQL_LEN { + if (builder.Len() + len(valuesStr)) > MAX_SQL_LEN { sql := builder.String() if _, err := conn.Exec(context.Background(), sql); err != nil { return err @@ -200,7 +200,7 @@ func handleBatchRecord(auditArray []AuditInfo, conn *db.Connector) error { builder.Reset() builder.WriteString(head) } - builder.WriteString(varluesStr) + builder.WriteString(valuesStr) } if builder.Len() > len(head) { diff --git a/api/audit_test.go b/api/audit_test.go index fbd5d41..72f75ab 100644 --- a/api/audit_test.go +++ b/api/audit_test.go @@ -88,6 +88,7 @@ func TestAudit(t *testing.T) { }) } + MAX_SQL_LEN = 300 // test audit batch input := `{"records":[{"timestamp":"1702548856940013848","cluster_id":"8468922059162439502","user":"root","operation":"createTable","client_add":"173.50.0.7:45166","db":"test","resource":"","details":"d630302"},{"timestamp":"1702548856939746458","cluster_id":"8468922059162439502","user":"root","operation":"createTable","client_add":"173.50.0.7:45230","db":"test","resource":"","details":"d130277"},{"timestamp":"1702548856939586665","cluster_id":"8468922059162439502","user":"root","operation":"createTable","client_add":"173.50.0.7:50288","db":"test","resource":"","details":"d5268"},{"timestamp":"1702548856939528940","cluster_id":"8468922059162439502","user":"root","operation":"createTable","client_add":"173.50.0.7:50222","db":"test","resource":"","details":"d255282"},{"timestamp":"1702548856939336371","cluster_id":"8468922059162439502","user":"root","operation":"createTable","client_add":"173.50.0.7:45126","db":"test","resource":"","details":"d755297"},{"timestamp":"1702548856939075131","cluster_id":"8468922059162439502","user":"root","operation":"createTable","client_add":"173.50.0.7:45122","db":"test","resource":"","details":"d380325"},{"timestamp":"1702548856938640661","cluster_id":"8468922059162439502","user":"root","operation":"createTable","client_add":"173.50.0.7:45152","db":"test","resource":"","details":"d255281"},{"timestamp":"1702548856938505795","cluster_id":"8468922059162439502","user":"root","operation":"createTable","client_add":"173.50.0.7:45122","db":"test","resource":"","details":"d130276"},{"timestamp":"1702548856938363319","cluster_id":"8468922059162439502","user":"root","operation":"createTable","client_add":"173.50.0.7:45178","db":"test","resource":"","details":"d755296"},{"timestamp":"1702548856938201478","cluster_id":"8468922059162439502","user":"root","operation":"createTable","client_add":"173.50.0.7:45166","db":"test","resource":"","details":"d380324"},{"timestamp":"1702548856937740618","cluster_id":"8468922059162439502","user":"root","operation":"createTable","client_add":"173.50.0.7:50288","db":"test","resource":"","details":"d5266"}]}`