Save reports to storage backend

internal/schema/postgres/migrations/20231205072045_validation_reports.up.pgsql

@@ -5,7 +5,8 @@ CREATE TABLE tl_validation_reports (
     created_at timestamp without time zone DEFAULT now() NOT NULL,
     updated_at timestamp without time zone DEFAULT now() NOT NULL,
     reported_at timestamp without time zone NOT NULL,
-    feed_version_id bigint REFERENCES feed_versions(id) NOT NULL
+    feed_version_id bigint REFERENCES feed_versions(id) NOT NULL,
+    file text
 );
 CREATE INDEX ON tl_validation_reports(feed_version_id);
 

internal/schema/sqlite.sql

@@ -696,7 +696,8 @@ CREATE TABLE tl_validation_reports (
 "feed_version_id" int,
 "created_at" datetime DEFAULT CURRENT_TIMESTAMP NOT NULL,
 "updated_at" datetime DEFAULT CURRENT_TIMESTAMP NOT NULL,
-"reported_at" datetime NOT NULL
+"reported_at" datetime NOT NULL,
+"file" varchar(255)
 );
 
 CREATE TABLE tl_validation_trip_update_stats (

validator/result.go

@@ -1,10 +1,15 @@
 package validator
 
 import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
 	"time"
 
 	"github.com/interline-io/transitland-lib/copier"
 	"github.com/interline-io/transitland-lib/dmfr"
+	"github.com/interline-io/transitland-lib/dmfr/store"
 	"github.com/interline-io/transitland-lib/rt"
 	"github.com/interline-io/transitland-lib/tl"
 	"github.com/interline-io/transitland-lib/tl/tt"
@@ -38,14 +43,39 @@ type RealtimeResult struct {
 	Errors               []error
 }
 
-func SaveValidationReport(atx tldb.Adapter, result *Result, reportedAt time.Time, fvid int, saveStatic bool, saveRealtimeStats bool) error {
+func (r *Result) Key() string {
+	return fmt.Sprintf("report-%s-%d.json", r.SHA1, time.Now().In(time.UTC).Unix())
+}
+
+func SaveValidationReport(atx tldb.Adapter, result *Result, reportedAt time.Time, fvid int, saveStatic bool, saveRealtimeStats bool, reportStorage string) error {
 	// Save validation reports
 	validationReport := ValidationReport{}
 	validationReport.FeedVersionID = fvid
 	validationReport.ReportedAt = tt.NewTime(reportedAt)
+
+	// Save JSON
+	if reportStorage != "" {
+		validationReport.File = tt.NewString(result.Key())
+		store, err := store.GetStore(reportStorage)
+		if err != nil {
+			return err
+		}
+		jj, err := json.Marshal(result)
+		if err != nil {
+			return err
+		}
+		jb := bytes.NewReader(jj)
+		if err := store.Upload(context.Background(), validationReport.File.Val, tl.Secret{}, jb); err != nil {
+			return err
+		}
+	}
+
+	// Save record
 	if _, err := atx.Insert(&validationReport); err != nil {
 		return err
 	}
+
+	// Save additional stats
 	if saveRealtimeStats {
 		for _, r := range result.Realtime {
 			for _, s := range r.TripUpdateStats {
@@ -81,6 +111,7 @@ func SaveValidationReport(atx tldb.Adapter, result *Result, reportedAt time.Time
 
 type ValidationReport struct {
 	ReportedAt tt.Time
+	File       tt.String
 	tl.BaseEntity
 }
 

validator/result_test.go

@@ -5,7 +5,6 @@ import (
 	"net/http/httptest"
 	"os"
 	"path/filepath"
-	"strings"
 	"testing"
 	"time"
 
@@ -31,10 +30,8 @@ func TestSaveValidationReport(t *testing.T) {
 	tz, _ := time.LoadLocation("America/Los_Angeles")
 	now := time.Date(2023, 11, 7, 17, 05, 0, 0, tz)
 	opts := Options{
-		SaveStaticValidationReport:   true,
-		SaveRealtimeValidationReport: true,
-		IncludeRealtimeJson:          true,
-		EvaluateAt:                   now,
+		IncludeRealtimeJson: true,
+		EvaluateAt:          now,
 		ValidateRealtimeMessages: []string{
 			ts.URL + "/ct-trip-updates.pb.json",
 			ts.URL + "/ct-vehicle-positions.pb.json",
@@ -47,26 +44,9 @@ func TestSaveValidationReport(t *testing.T) {
 		t.Fatal(err)
 	}
 	testdb.TempSqlite(func(atx tldb.Adapter) error {
-		if err := SaveValidationReport(atx, result, now, 1, true, true); err != nil {
+		if err := SaveValidationReport(atx, result, now, 1, true, true, ""); err != nil {
 			t.Fatal(err)
 		}
 		return nil
 	})
 }
-
-func getFiles(path string) ([]string, error) {
-	files := []string{}
-	if err := filepath.Walk(path,
-		func(path string, info os.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
-			if strings.HasSuffix(info.Name(), ".pb") {
-				files = append(files, path)
-			}
-			return nil
-		}); err != nil {
-		return nil, err
-	}
-	return files, nil
-}

validator/validate_cmd.go

@@ -16,13 +16,16 @@ import (
 
 // Command
 type Command struct {
-	Options    Options
-	rtFiles    cli.ArrayFlags
-	OutputFile string
-	DBURL      string
-	FVID       int
-	extensions cli.ArrayFlags
-	readerPath string
+	Options                      Options
+	rtFiles                      cli.ArrayFlags
+	OutputFile                   string
+	DBURL                        string
+	FVID                         int
+	extensions                   cli.ArrayFlags
+	SaveStaticValidationReport   bool
+	SaveRealtimeValidationReport bool
+	SaveStorage                  string
+	readerPath                   string
 }
 
 func (cmd *Command) Parse(args []string) error {
@@ -35,8 +38,9 @@ func (cmd *Command) Parse(args []string) error {
 	fl.StringVar(&cmd.OutputFile, "o", "", "Write validation report as JSON to file")
 	fl.BoolVar(&cmd.Options.BestPractices, "best-practices", false, "Include Best Practices validations")
 	fl.BoolVar(&cmd.Options.IncludeRealtimeJson, "rt-json", false, "Include GTFS-RT proto messages as JSON in validation report")
-	fl.BoolVar(&cmd.Options.SaveStaticValidationReport, "save-static-report", false, "Save static validation report in database")
-	fl.BoolVar(&cmd.Options.SaveRealtimeValidationReport, "save-rt-report", false, "Save RT validation report in database")
+	fl.BoolVar(&cmd.SaveStaticValidationReport, "save-static-report", false, "Save static validation report in database")
+	fl.BoolVar(&cmd.SaveRealtimeValidationReport, "save-rt-report", false, "Save RT validation report in database")
+	fl.StringVar(&cmd.SaveStorage, "save-storage", "", "Storage path for saving validation report JSON")
 	fl.IntVar(&cmd.FVID, "save-fvid", 0, "Save report to feed version ID")
 	fl.Var(&cmd.rtFiles, "rt", "Include GTFS-RT proto message in validation report")
 	fl.IntVar(&cmd.Options.ErrorLimit, "error-limit", 1000, "Max number of detailed errors per error group")
@@ -89,15 +93,15 @@ func (cmd *Command) Run() error {
 	}
 
 	// Save to database
-	if cmd.Options.SaveRealtimeValidationReport || cmd.Options.SaveStaticValidationReport {
+	if cmd.SaveRealtimeValidationReport || cmd.SaveStaticValidationReport {
 		log.Infof("Saving validation report to feed version: %d", cmd.FVID)
 		writer, err := tldb.OpenWriter(cmd.DBURL, true)
 		if err != nil {
 			return err
 		}
 		atx := writer.Adapter
 		defer atx.Close()
-		if err := SaveValidationReport(atx, result, time.Now(), cmd.FVID, true, true); err != nil {
+		if err := SaveValidationReport(atx, result, time.Now(), cmd.FVID, cmd.SaveStaticValidationReport, cmd.SaveRealtimeValidationReport, cmd.SaveStorage); err != nil {
 			return err
 		}
 	}

validator/validator.go

@@ -38,18 +38,16 @@ var defaultMaxFileRows = map[string]int64{
 
 // Options defines options for the Validator.
 type Options struct {
-	BestPractices                bool
-	CheckFileLimits              bool
-	IncludeServiceLevels         bool
-	IncludeEntities              bool
-	IncludeEntitiesLimit         int
-	IncludeRouteGeometries       bool
-	ValidateRealtimeMessages     []string
-	IncludeRealtimeJson          bool
-	MaxRTMessageSize             uint64
-	EvaluateAt                   time.Time
-	SaveStaticValidationReport   bool
-	SaveRealtimeValidationReport bool
+	BestPractices            bool
+	CheckFileLimits          bool
+	IncludeServiceLevels     bool
+	IncludeEntities          bool
+	IncludeEntitiesLimit     int
+	IncludeRouteGeometries   bool
+	ValidateRealtimeMessages []string
+	IncludeRealtimeJson      bool
+	MaxRTMessageSize         uint64
+	EvaluateAt               time.Time
 	copier.Options
 }