Skip to content

Commit

Permalink
add support for log events
Browse files Browse the repository at this point in the history
Signed-off-by: Nicola Murino <[email protected]>
  • Loading branch information
drakkan committed May 12, 2023
1 parent 2fc51c5 commit 170bef0
Show file tree
Hide file tree
Showing 15 changed files with 200 additions and 33 deletions.
11 changes: 10 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,15 @@ This is an example configuration.
"share",
"event_action",
"event_rule",
"role"
"role",
"ip_list_entry",
"configs"
],
"log_events": [
1,
2,
3,
4
],
"retry_max_time": 60,
"retry_queue_max_size": 1000
Expand Down Expand Up @@ -86,6 +94,7 @@ The plugin will automatically create the following database tables:

- `eventstore_fs_events`
- `eventstore_provider_events`
- `eventstore_log_events`

Inspect your database for more details.

Expand Down
2 changes: 1 addition & 1 deletion cmd/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (
)

const (
version = "1.0.6-dev"
version = "1.0.7"
envPrefix = "SFTPGO_PLUGIN_EVENTSTORE_"
)

Expand Down
7 changes: 6 additions & 1 deletion db/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,8 @@ func Initialize(driver, dsn string, dbDebug bool) error {
}

sqlDB.SetMaxIdleConns(10)
sqlDB.SetConnMaxIdleTime(3 * time.Minute)
sqlDB.SetConnMaxIdleTime(4 * time.Minute)
sqlDB.SetConnMaxLifetime(2 * time.Minute)

return sqlDB.Ping()
}
Expand Down Expand Up @@ -109,4 +110,8 @@ func Cleanup(timestamp time.Time) {
if err := cleanupProviderEvents(timestamp); err != nil {
logger.AppLogger.Error("unable to delete provider events", "error", err)
}

if err := cleanupLogEvents(timestamp); err != nil {
logger.AppLogger.Error("unable to delete log events", "error", err)
}
}
52 changes: 52 additions & 0 deletions db/logevent.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package db

import (
"time"

"github.com/rs/xid"
"gorm.io/gorm"

"github.com/sftpgo/sftpgo-plugin-eventstore/logger"
)

// LogEvent defines a log event
type LogEvent struct {
ID string `json:"id" gorm:"primaryKey"`
Timestamp int64 `json:"timestamp"`
Event int `json:"event"`
Protocol string `json:"protocol,omitempty"`
Username string `json:"username,omitempty"`
IP string `json:"ip,omitempty"`
Message string `json:"message,omitempty"`
Role string `json:"role,omitempty"`
InstanceID string `json:"instance_id,omitempty"`
}

// TableName defines the database table name
func (ev *LogEvent) TableName() string {
return "eventstore_log_events"
}

// BeforeCreate implements gorm hook
func (ev *LogEvent) BeforeCreate(_ *gorm.DB) (err error) {
ev.ID = xid.New().String()
return
}

// Create persists the object
func (ev *LogEvent) Create(tx *gorm.DB) error {
return tx.Create(ev).Error
}

func cleanupLogEvents(timestamp time.Time) error {
sess, cancel := getSessionWithTimeout(20 * time.Minute)
defer cancel()

logger.AppLogger.Debug("removing log events", "timestamp", timestamp)
sess = sess.Where("timestamp < ?", timestamp.UnixNano()).Delete(&LogEvent{})
err := sess.Error
if err == nil {
logger.AppLogger.Debug("log events deleted", "num", sess.RowsAffected)
}
return err
}
1 change: 1 addition & 0 deletions db/migration/migration.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ func registerMigrations() {
getV3Migration(),
getV4Migration(),
getV5Migration(),
getV6Migration(),
)
}

Expand Down
2 changes: 1 addition & 1 deletion db/migration/v1.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ type fsEventV1 struct {
FileSize int64 `gorm:"size:64"`
Status int `gorm:"size:32;index:idx_fs_events_status"`
Protocol string `gorm:"size:30;not null;index:idx_fs_events_protocol"`
IP string `gorm:"size:50;index:idx_ip"`
IP string `gorm:"size:50;index:idx_fs_events_ip"`
InstanceID string `gorm:"size:60;index:idx_fs_events_instance_id"`
}

Expand Down
2 changes: 1 addition & 1 deletion db/migration/v2.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ type fsEventV2 struct {
Status int `gorm:"size:32;index:idx_fs_events_status"`
Protocol string `gorm:"size:30;not null;index:idx_fs_events_protocol"`
SessionID string `gorm:"size:100"`
IP string `gorm:"size:50;index:idx_ip"`
IP string `gorm:"size:50;index:idx_fs_events_ip"`
InstanceID string `gorm:"size:60;index:idx_fs_events_instance_id"`
}

Expand Down
8 changes: 4 additions & 4 deletions db/migration/v3.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,10 @@ type fsEventV3 struct {
Status int `gorm:"size:32;index:idx_fs_events_status"`
Protocol string `gorm:"size:30;not null;index:idx_fs_events_protocol"`
SessionID string `gorm:"size:100"`
IP string `gorm:"size:50;index:idx_ip"`
FsProvider int `gorm:"size:32;index:idx_fs_provider"`
Bucket string `gorm:"size:512;index:idx_bucket"`
Endpoint string `gorm:"size:512;index:idx_endpoint"`
IP string `gorm:"size:50;index:idx_fs_events_ip"`
FsProvider int `gorm:"size:32;index:idx_fs_events_provider"`
Bucket string `gorm:"size:512;index:idx_fs_events_bucket"`
Endpoint string `gorm:"size:512;index:idx_fs_events_endpoint"`
OpenFlags int `gorm:"size:32"`
InstanceID string `gorm:"size:60;index:idx_fs_events_instance_id"`
}
Expand Down
12 changes: 6 additions & 6 deletions db/migration/v4.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,12 @@ type fsEventV4 struct {
Status int `gorm:"size:32;index:idx_fs_events_status"`
Protocol string `gorm:"size:30;not null;index:idx_fs_events_protocol"`
SessionID string `gorm:"size:100"`
IP string `gorm:"size:50;index:idx_ip"`
FsProvider int `gorm:"size:32;index:idx_fs_provider"`
Bucket string `gorm:"size:512;index:idx_bucket"`
Endpoint string `gorm:"size:512;index:idx_endpoint"`
IP string `gorm:"size:50;index:idx_fs_events_ip"`
FsProvider int `gorm:"size:32;index:idx_fs_events_provider"`
Bucket string `gorm:"size:512;index:idx_fs_events_bucket"`
Endpoint string `gorm:"size:512;index:idx_fs_events_endpoint"`
OpenFlags int `gorm:"size:32"`
Role string `gorm:"size:255;index:idx_role"`
Role string `gorm:"size:255;index:idx_fs_events_role"`
InstanceID string `gorm:"size:60;index:idx_fs_events_instance_id"`
}

Expand All @@ -45,7 +45,7 @@ type providerEventV4 struct {
ObjectType string `gorm:"size:50;index:idx_provider_events_object_type"`
ObjectName string `gorm:"size:255;index:idx_provider_events_object_name"`
ObjectData []byte
Role string `gorm:"size:255;index:idx_role"`
Role string `gorm:"size:255;index:idx_provider_events_role"`
InstanceID string `gorm:"size:60;index:idx_provider_events_instance_id"`
}

Expand Down
10 changes: 5 additions & 5 deletions db/migration/v5.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,12 @@ type fsEventV5 struct {
Status int `gorm:"size:32;index:idx_fs_events_status"`
Protocol string `gorm:"size:30;not null;index:idx_fs_events_protocol"`
SessionID string `gorm:"size:100"`
IP string `gorm:"size:50;index:idx_ip"`
FsProvider int `gorm:"size:32;index:idx_fs_provider"`
Bucket string `gorm:"size:512;index:idx_bucket"`
Endpoint string `gorm:"size:512;index:idx_endpoint"`
IP string `gorm:"size:50;index:idx_fs_events_ip"`
FsProvider int `gorm:"size:32;index:idx_fs_events_provider"`
Bucket string `gorm:"size:512;index:idx_fs_events_bucket"`
Endpoint string `gorm:"size:512;index:idx_fs_events_endpoint"`
OpenFlags int `gorm:"size:32"`
Role string `gorm:"size:255;index:idx_role"`
Role string `gorm:"size:255;index:idx_fs_events_role"`
InstanceID string `gorm:"size:60;index:idx_fs_events_instance_id"`
}

Expand Down
52 changes: 52 additions & 0 deletions db/migration/v6.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package migration

import (
"github.com/go-gormigrate/gormigrate/v2"
"gorm.io/gorm"
)

const (
mignationV6ID = "6"
)

type logEventV1 struct {
ID string `gorm:"primaryKey;size:36"`
Timestamp int64 `gorm:"size:64;not null;index:idx_log_events_timestamp"`
Event int `gorm:"size:32;not null;index:idx_log_events_event"`
Protocol string `gorm:"size:30;index:idx_log_events_protocol"`
Username string `gorm:"size:255;index:idx_log_events_username"`
IP string `gorm:"size:50;index:idx_log_events_ip"`
Message string
Role string `gorm:"size:255;index:idx_log_events_role"`
InstanceID string `gorm:"size:60;index:idx_log_events_instance_id"`
}

func (ev *logEventV1) TableName() string {
return "eventstore_log_events"
}

func v6Up(tx *gorm.DB) error {
modelsToMigrate := []interface{}{
&logEventV1{},
}
return tx.AutoMigrate(modelsToMigrate...)
}

func v6Down(tx *gorm.DB) error {
modelsToMigrate := []interface{}{
&logEventV1{},
}
return tx.Migrator().DropTable(modelsToMigrate...)
}

func getV6Migration() *gormigrate.Migration {
return &gormigrate.Migration{
ID: mignationV6ID,
Migrate: func(tx *gorm.DB) error {
return v6Up(tx)
},
Rollback: func(tx *gorm.DB) error {
return v6Down(tx)
},
}
}
22 changes: 22 additions & 0 deletions db/notifier.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,3 +67,25 @@ func (n *Notifier) NotifyProviderEvent(event *notifier.ProviderEvent) error {
}
return nil
}

func (n *Notifier) NotifyLogEvent(event *notifier.LogEvent) error {
ev := &LogEvent{
Timestamp: event.Timestamp,
Event: int(event.Event),
Protocol: event.Protocol,
Username: event.Username,
IP: event.IP,
Message: event.Message,
Role: event.Role,
InstanceID: n.InstanceID,
}
sess, cancel := GetDefaultSession()
defer cancel()

err := ev.Create(sess)
if err != nil {
logger.AppLogger.Warn("unable to save log event", "event", event.Event, "error", err)
return err
}
return nil
}
28 changes: 27 additions & 1 deletion db/notifier_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"github.com/stretchr/testify/assert"
)

func TestFsEvent(t *testing.T) {
func TestNotifyEvents(t *testing.T) {
n := Notifier{
InstanceID: "sftpgo1",
}
Expand Down Expand Up @@ -99,6 +99,32 @@ func TestFsEvent(t *testing.T) {
assert.Equal(t, providerEvent.Role, providerEv.Role)
assert.Equal(t, providerEvent.ObjectData, providerEv.ObjectData)

logEvent := &notifier.LogEvent{
Timestamp: time.Now().UnixNano(),
Event: 1,
Protocol: "SSH",
Username: "user1",
IP: "127.0.0.1",
Message: "error desc",
Role: "role1",
}
err = n.NotifyLogEvent(logEvent)
assert.NoError(t, err)

var logEv LogEvent
err = sess.First(&logEv).Error
assert.NoError(t, err)

assert.Equal(t, n.InstanceID, logEv.InstanceID)
assert.NotEmpty(t, logEv.ID)
assert.Equal(t, logEvent.Timestamp, logEv.Timestamp)
assert.Equal(t, int(logEvent.Event), logEv.Event)
assert.Equal(t, logEvent.Protocol, logEv.Protocol)
assert.Equal(t, logEvent.Username, logEv.Username)
assert.Equal(t, logEvent.IP, logEv.IP)
assert.Equal(t, logEvent.Message, logEv.Message)
assert.Equal(t, logEvent.Role, logEv.Role)

// test cleanup
Cleanup(time.Now().Add(-24 * time.Hour))
// the data must not be deleted
Expand Down
8 changes: 4 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,12 @@ require (
github.com/hashicorp/go-hclog v1.5.0
github.com/hashicorp/go-plugin v1.4.10-0.20230403150917-e889c1ba1044
github.com/rs/xid v1.5.0
github.com/sftpgo/sdk v0.1.3
github.com/sftpgo/sdk v0.1.4-0.20230512160325-38e59551f700
github.com/stretchr/testify v1.8.2
github.com/urfave/cli/v2 v2.25.3
gorm.io/driver/mysql v1.5.0
gorm.io/driver/postgres v1.5.0
gorm.io/gorm v1.25.0
gorm.io/gorm v1.25.1
)

require (
Expand All @@ -36,8 +36,8 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
golang.org/x/crypto v0.8.0 // indirect
golang.org/x/net v0.9.0 // indirect
golang.org/x/crypto v0.9.0 // indirect
golang.org/x/net v0.10.0 // indirect
golang.org/x/sys v0.8.0 // indirect
golang.org/x/text v0.9.0 // indirect
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
Expand Down
16 changes: 8 additions & 8 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,8 @@ github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sftpgo/sdk v0.1.3 h1:o/9herRbrDH6sQwfpKlV3AV0R7qJgOe/x4yQnEIWIHk=
github.com/sftpgo/sdk v0.1.3/go.mod h1:gDxDaU3rhp9Y92ddsE7SbQ8jdBNNWK1DKlp5eHXrsb8=
github.com/sftpgo/sdk v0.1.4-0.20230512160325-38e59551f700 h1:jL6mfKAaFv862AnBUxIfTH9wmnuPjbWyjHQUGDo+Xt0=
github.com/sftpgo/sdk v0.1.4-0.20230512160325-38e59551f700/go.mod h1:gDxDaU3rhp9Y92ddsE7SbQ8jdBNNWK1DKlp5eHXrsb8=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
Expand All @@ -103,8 +103,8 @@ golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ=
golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE=
golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
Expand All @@ -113,8 +113,8 @@ golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
Expand Down Expand Up @@ -176,5 +176,5 @@ gorm.io/driver/postgres v1.5.0/go.mod h1:FUZXzO+5Uqg5zzwzv4KK49R8lvGIyscBOqYrtI1
gorm.io/driver/sqlite v1.3.2 h1:nWTy4cE52K6nnMhv23wLmur9Y3qWbZvOBz+V4PrGAxg=
gorm.io/driver/sqlserver v1.3.2 h1:yYt8f/xdAKLY7lCCyXxIUEgZ/WsURos3dHrx8MKFGAk=
gorm.io/gorm v1.24.7-0.20230306060331-85eaf9eeda11/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
gorm.io/gorm v1.25.0 h1:+KtYtb2roDz14EQe4bla8CbQlmb9dN3VejSai3lprfU=
gorm.io/gorm v1.25.0/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
gorm.io/gorm v1.25.1 h1:nsSALe5Pr+cM3V1qwwQ7rOkw+6UeLrX5O4v3llhHa64=
gorm.io/gorm v1.25.1/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=

0 comments on commit 170bef0

Please sign in to comment.