diff --git a/cmd/insights/commands/upload.go b/cmd/insights/commands/upload.go index 6f680c3..aae5175 100644 --- a/cmd/insights/commands/upload.go +++ b/cmd/insights/commands/upload.go @@ -4,6 +4,7 @@ import ( "log/slog" "github.com/spf13/cobra" + "github.com/ubuntu/ubuntu-insights/internal/constants" ) type uploadConfig struct { @@ -16,7 +17,7 @@ type uploadConfig struct { var defaultUploadConfig = uploadConfig{ sources: []string{""}, - server: "https://metrics.ubuntu.com", + server: constants.DefaultServerURL, minAge: 604800, force: false, dryRun: false, diff --git a/internal/consent/consent.go b/internal/consent/consent.go index d13a145..b1c0749 100644 --- a/internal/consent/consent.go +++ b/internal/consent/consent.go @@ -30,12 +30,12 @@ func New(path string) *Manager { return &Manager{path: path} } -// GetConsentState gets the consent state for the given source. +// GetState gets the consent state for the given source. // If the source do not have a consent file, it will be considered as a false state. // If the source is an empty string, then the global consent state will be returned. // If the target consent file does not exist, it will not be created. -func (cm Manager) GetConsentState(source string) (bool, error) { - sourceConsent, err := readConsentFile(cm.getConsentFile(source)) +func (cm Manager) GetState(source string) (bool, error) { + sourceConsent, err := readFile(cm.getFile(source)) if err != nil { slog.Error("Error reading source consent file", "source", source, "error", err) return false, err @@ -46,20 +46,33 @@ func (cm Manager) GetConsentState(source string) (bool, error) { var consentSourceFilePattern = `%s` + constants.ConsentSourceBaseSeparator + constants.GlobalFileName -// SetConsentState updates the consent state for the given source. +// SetState updates the consent state for the given source. // If the source is an empty string, then the global consent state will be set. // If the target consent file does not exist, it will be created. -func (cm Manager) SetConsentState(source string, state bool) (err error) { +func (cm Manager) SetState(source string, state bool) (err error) { defer decorate.OnError(&err, "could not set consent state") consent := consentFile{ConsentState: state} - return consent.write(cm.getConsentFile(source)) + return consent.write(cm.getFile(source)) } -// getConsentFile returns the expected path to the consent file for the given source. +// HasConsent returns true if there is consent for the given source, based on the hierarchy rules. +// If the source has a consent file, its value is returned. +// Otherwise, the global consent state is returned. +func (cm Manager) HasConsent(source string) (bool, error) { + consent, err := cm.GetState(source) + if err != nil { + slog.Warn("Could not get source specific consent state, falling back to global consent state", "source", source, "error", err) + return cm.GetState("") + } + + return consent, nil +} + +// getFile returns the expected path to the consent file for the given source. // If source is blank, it returns the path to the global consent file. // It does not check if the file exists, or if it is valid. -func (cm Manager) getConsentFile(source string) string { +func (cm Manager) getFile(source string) string { p := filepath.Join(cm.path, constants.GlobalFileName) if source != "" { p = filepath.Join(cm.path, fmt.Sprintf(consentSourceFilePattern, source)) @@ -69,7 +82,7 @@ func (cm Manager) getConsentFile(source string) string { } // getSourceConsentFiles returns a map of all paths to validly named consent files in the folder, other than the global file. -func (cm Manager) getConsentFiles() (map[string]string, error) { +func (cm Manager) getFiles() (map[string]string, error) { sourceFiles := make(map[string]string) entries, err := os.ReadDir(cm.path) @@ -94,7 +107,7 @@ func (cm Manager) getConsentFiles() (map[string]string, error) { return sourceFiles, nil } -func readConsentFile(path string) (consentFile, error) { +func readFile(path string) (consentFile, error) { var consent consentFile _, err := toml.DecodeFile(path, &consent) slog.Debug("Read consent file", "file", path, "consent", consent.ConsentState) diff --git a/internal/consent/consent_test.go b/internal/consent/consent_test.go index fc1568c..612fdb5 100644 --- a/internal/consent/consent_test.go +++ b/internal/consent/consent_test.go @@ -11,7 +11,7 @@ import ( "github.com/ubuntu/ubuntu-insights/internal/testutils" ) -func TestGetConsentState(t *testing.T) { +func TestGetState(t *testing.T) { t.Parallel() tests := map[string]struct { @@ -55,10 +55,9 @@ func TestGetConsentState(t *testing.T) { t.Parallel() dir, err := setupTmpConsentFiles(t, tc.globalFile) require.NoError(t, err, "Setup: failed to setup temporary consent files") - defer testutils.CleanupDir(t, dir) cm := consent.New(dir) - got, err := cm.GetConsentState(tc.source) + got, err := cm.GetState(tc.source) if tc.wantErr { require.Error(t, err, "expected an error but got none") return @@ -71,7 +70,7 @@ func TestGetConsentState(t *testing.T) { } } -func TestSetConsentStates(t *testing.T) { +func TestSetState(t *testing.T) { t.Parallel() tests := map[string]struct { @@ -112,10 +111,9 @@ func TestSetConsentStates(t *testing.T) { t.Parallel() dir, err := setupTmpConsentFiles(t, tc.globalFile) require.NoError(t, err, "Setup: failed to setup temporary consent files") - defer testutils.CleanupDir(t, dir) cm := consent.New(dir) - err = cm.SetConsentState(tc.writeSource, tc.writeState) + err = cm.SetState(tc.writeSource, tc.writeState) if tc.wantErr { require.Error(t, err, "expected an error but got none") return @@ -135,23 +133,69 @@ func TestSetConsentStates(t *testing.T) { } } +func TestHasConsent(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + source string + + globalFile string + + want bool + wantErr bool + }{ + "True Global-True Source": {source: "valid_true", globalFile: "valid_true-consent.toml", want: true}, + "True Global-False Source": {source: "valid_false", globalFile: "valid_true-consent.toml", want: false}, + "True Global-Invalid Value Source": {source: "invalid_value", globalFile: "valid_true-consent.toml", want: true}, + "True Global-Invalid File Source": {source: "invalid_file", globalFile: "valid_true-consent.toml", want: true}, + "True Global-Not A File Source": {source: "not_a_file", globalFile: "valid_true-consent.toml", want: true}, + + "False Global-True Source": {source: "valid_true", globalFile: "valid_false-consent.toml", want: true}, + "False Global-False Source": {source: "valid_false", globalFile: "valid_false-consent.toml", want: false}, + "False Global-Invalid Value Source": {source: "invalid_value", globalFile: "valid_false-consent.toml", want: false}, + "False Global-Invalid File Source": {source: "invalid_file", globalFile: "valid_false-consent.toml", want: false}, + "False Global-Not A File Source": {source: "not_a_file", globalFile: "valid_false-consent.toml", want: false}, + + "No Global-True Source": {source: "valid_true", want: true}, + "No Global-False Source": {source: "valid_false", want: false}, + "No Global-Invalid Value Source": {source: "invalid_value", wantErr: true}, + "No Global-Invalid File Source": {source: "invalid_file", wantErr: true}, + "No Global-Not A File Source": {source: "not_a_file", wantErr: true}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + dir, err := setupTmpConsentFiles(t, tc.globalFile) + require.NoError(t, err, "Setup: failed to setup temporary consent files") + cm := consent.New(dir) + + got, err := cm.HasConsent(tc.source) + if tc.wantErr { + require.Error(t, err, "expected an error but got none") + return + } + require.NoError(t, err, "got an unexpected error") + + require.Equal(t, tc.want, got, "HasConsent should return expected consent state") + }) + } +} + func setupTmpConsentFiles(t *testing.T, globalFile string) (string, error) { t.Helper() // Setup temporary directory var err error - dir, err := os.MkdirTemp("", "consent-files") - if err != nil { - return dir, fmt.Errorf("failed to create temporary directory: %v", err) - } + dir := t.TempDir() - if err = testutils.CopyDir(filepath.Join("testdata", "consent_files"), dir); err != nil { + if err = testutils.CopyDir(t, filepath.Join("testdata", "consent_files"), dir); err != nil { return dir, fmt.Errorf("failed to copy testdata directory to temporary directory: %v", err) } // Setup globalFile if provided if globalFile != "" { - if err = testutils.CopyFile(filepath.Join(dir, globalFile), filepath.Join(dir, "consent.toml")); err != nil { + if err = testutils.CopyFile(t, filepath.Join(dir, globalFile), filepath.Join(dir, "consent.toml")); err != nil { return dir, fmt.Errorf("failed to copy requested global consent file: %v", err) } } diff --git a/internal/consent/export_test.go b/internal/consent/export_test.go index f887256..8f88838 100644 --- a/internal/consent/export_test.go +++ b/internal/consent/export_test.go @@ -4,14 +4,14 @@ package consent // It does not get the global consent state. // If continueOnErr is true, it will continue to the next source if an error occurs. func (cm Manager) GetAllSourceConsentStates(continueOnErr bool) (map[string]bool, error) { - p, err := cm.getConsentFiles() + p, err := cm.getFiles() if err != nil { return nil, err } consentStates := make(map[string]bool) for source, path := range p { - consent, err := readConsentFile(path) + consent, err := readFile(path) if err != nil && !continueOnErr { return nil, err } diff --git a/internal/consent/testdata/TestGetConsentState/golden/invalid_value_global_file,_valid_false_source b/internal/consent/testdata/TestGetState/golden/invalid_value_global_file,_valid_false_source similarity index 100% rename from internal/consent/testdata/TestGetConsentState/golden/invalid_value_global_file,_valid_false_source rename to internal/consent/testdata/TestGetState/golden/invalid_value_global_file,_valid_false_source diff --git a/internal/consent/testdata/TestGetConsentState/golden/invalid_value_global_file,_valid_true_source b/internal/consent/testdata/TestGetState/golden/invalid_value_global_file,_valid_true_source similarity index 100% rename from internal/consent/testdata/TestGetConsentState/golden/invalid_value_global_file,_valid_true_source rename to internal/consent/testdata/TestGetState/golden/invalid_value_global_file,_valid_true_source diff --git a/internal/consent/testdata/TestGetConsentState/golden/no_global_file,_valid_false_source b/internal/consent/testdata/TestGetState/golden/no_global_file,_valid_false_source similarity index 100% rename from internal/consent/testdata/TestGetConsentState/golden/no_global_file,_valid_false_source rename to internal/consent/testdata/TestGetState/golden/no_global_file,_valid_false_source diff --git a/internal/consent/testdata/TestGetConsentState/golden/no_global_file,_valid_true_source b/internal/consent/testdata/TestGetState/golden/no_global_file,_valid_true_source similarity index 100% rename from internal/consent/testdata/TestGetConsentState/golden/no_global_file,_valid_true_source rename to internal/consent/testdata/TestGetState/golden/no_global_file,_valid_true_source diff --git a/internal/consent/testdata/TestGetConsentState/golden/valid_false_global_file b/internal/consent/testdata/TestGetState/golden/valid_false_global_file similarity index 100% rename from internal/consent/testdata/TestGetConsentState/golden/valid_false_global_file rename to internal/consent/testdata/TestGetState/golden/valid_false_global_file diff --git a/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file b/internal/consent/testdata/TestGetState/golden/valid_true_global_file similarity index 100% rename from internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file rename to internal/consent/testdata/TestGetState/golden/valid_true_global_file diff --git a/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_false_source b/internal/consent/testdata/TestGetState/golden/valid_true_global_file,_valid_false_source similarity index 100% rename from internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_false_source rename to internal/consent/testdata/TestGetState/golden/valid_true_global_file,_valid_false_source diff --git a/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_true_source b/internal/consent/testdata/TestGetState/golden/valid_true_global_file,_valid_true_source similarity index 100% rename from internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_true_source rename to internal/consent/testdata/TestGetState/golden/valid_true_global_file,_valid_true_source diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_false b/internal/consent/testdata/TestSetState/golden/new_file,_write_global_false similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_false rename to internal/consent/testdata/TestSetState/golden/new_file,_write_global_false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_true b/internal/consent/testdata/TestSetState/golden/new_file,_write_global_true similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_true rename to internal/consent/testdata/TestSetState/golden/new_file,_write_global_true diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_false b/internal/consent/testdata/TestSetState/golden/new_file,_write_source_false similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_false rename to internal/consent/testdata/TestSetState/golden/new_file,_write_source_false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_true b/internal/consent/testdata/TestSetState/golden/new_file,_write_source_true similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_true rename to internal/consent/testdata/TestSetState/golden/new_file,_write_source_true diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_false b/internal/consent/testdata/TestSetState/golden/overwrite_file,_write_diff_global_false similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_false rename to internal/consent/testdata/TestSetState/golden/overwrite_file,_write_diff_global_false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_true b/internal/consent/testdata/TestSetState/golden/overwrite_file,_write_diff_global_true similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_true rename to internal/consent/testdata/TestSetState/golden/overwrite_file,_write_diff_global_true diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_false b/internal/consent/testdata/TestSetState/golden/overwrite_file,_write_diff_source_false similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_false rename to internal/consent/testdata/TestSetState/golden/overwrite_file,_write_diff_source_false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_true b/internal/consent/testdata/TestSetState/golden/overwrite_file,_write_diff_source_true similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_true rename to internal/consent/testdata/TestSetState/golden/overwrite_file,_write_diff_source_true diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_false b/internal/consent/testdata/TestSetState/golden/overwrite_file,_write_global_false similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_false rename to internal/consent/testdata/TestSetState/golden/overwrite_file,_write_global_false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_true b/internal/consent/testdata/TestSetState/golden/overwrite_file,_write_global_true similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_true rename to internal/consent/testdata/TestSetState/golden/overwrite_file,_write_global_true diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_false b/internal/consent/testdata/TestSetState/golden/overwrite_file,_write_source_false similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_false rename to internal/consent/testdata/TestSetState/golden/overwrite_file,_write_source_false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_true b/internal/consent/testdata/TestSetState/golden/overwrite_file,_write_source_true similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_true rename to internal/consent/testdata/TestSetState/golden/overwrite_file,_write_source_true diff --git a/internal/constants/constants.go b/internal/constants/constants.go index 448720d..6f9acf6 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -18,6 +18,15 @@ const ( // DefaultLogLevel is the default log level selected without any verbosity flags. DefaultLogLevel = slog.LevelInfo + // DefaultServerURL is the default base URL for the server that reports are uploaded to. + DefaultServerURL = "https://metrics.ubuntu.com" + + // LocalFolder is the default name of the local collected reports folder. + LocalFolder = "local" + + // UploadedFolder is the default name of the uploaded reports folder. + UploadedFolder = "uploaded" + // GlobalFileName is the default base name of the consent state files. GlobalFileName = "consent.toml" @@ -28,6 +37,9 @@ const ( ReportExt = ".json" ) +// OptOutJSON is the data sent in case of Opt-Out choice. +var OptOutJSON = struct{ OptOut bool }{true} + type options struct { baseDir func() (string, error) } diff --git a/internal/fileutils/fileutils.go b/internal/fileutils/fileutils.go new file mode 100644 index 0000000..b113a2f --- /dev/null +++ b/internal/fileutils/fileutils.go @@ -0,0 +1,38 @@ +// Package fileutils provides utility functions for handling files. +package fileutils + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" +) + +// AtomicWrite writes data to a file atomically. +// If the file already exists, then it will be overwritten. +// Not atomic on Windows. +func AtomicWrite(path string, data []byte) error { + tmp, err := os.CreateTemp(filepath.Dir(path), "tmp-*.tmp") + if err != nil { + return fmt.Errorf("could not create temporary file: %v", err) + } + defer func() { + _ = tmp.Close() + if err := os.Remove(tmp.Name()); err != nil && !os.IsNotExist(err) { + slog.Warn("Failed to remove temporary file", "file", tmp.Name(), "error", err) + } + }() + + if _, err := tmp.Write(data); err != nil { + return fmt.Errorf("could not write to temporary file: %v", err) + } + + if err := tmp.Close(); err != nil { + return fmt.Errorf("could not close temporary file: %v", err) + } + + if err := os.Rename(tmp.Name(), path); err != nil { + return fmt.Errorf("could not rename temporary file: %v", err) + } + return nil +} diff --git a/internal/fileutils/fileutils_test.go b/internal/fileutils/fileutils_test.go new file mode 100644 index 0000000..9308134 --- /dev/null +++ b/internal/fileutils/fileutils_test.go @@ -0,0 +1,81 @@ +package fileutils_test + +import ( + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/require" + "github.com/ubuntu/ubuntu-insights/internal/fileutils" +) + +func TestAtomicWrite(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + data []byte + fileExists bool + fileExistsPerms os.FileMode + invalidDir bool + + wantError bool + }{ + "Empty file": {data: []byte{}}, + "Non-empty file": {data: []byte("data")}, + "Override file": {data: []byte("data"), fileExistsPerms: 0600, fileExists: true}, + "Override empty file": {data: []byte{}, fileExistsPerms: 0600, fileExists: true}, + + "Existing empty file": {data: []byte{}, fileExistsPerms: 0600, fileExists: true}, + "Existing non-empty file": {data: []byte("data"), fileExistsPerms: 0600, fileExists: true}, + + "Override read-only file": {data: []byte("data"), fileExistsPerms: 0400, fileExists: true, wantError: runtime.GOOS == "windows"}, + "Override No Perms file": {data: []byte("data"), fileExistsPerms: 0000, fileExists: true, wantError: runtime.GOOS == "windows"}, + "Invalid Dir": {data: []byte("data"), invalidDir: true, wantError: true}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + oldFile := []byte("Old File!") + tempDir := t.TempDir() + path := filepath.Join(tempDir, "file") + if tc.invalidDir { + path = filepath.Join(path, "fake_dir") + } + + if tc.fileExists { + err := os.WriteFile(path, oldFile, tc.fileExistsPerms) + require.NoError(t, err, "Setup: WriteFile should not return an error") + t.Cleanup(func() { _ = os.Chmod(path, 0600) }) + } + + err := fileutils.AtomicWrite(path, tc.data) + if tc.wantError { + require.Error(t, err, "AtomicWrite should return an error") + + // Check that the file was not overwritten + if !tc.fileExists { + return + } + + if tc.invalidDir { + path = filepath.Dir(path) + } + + data, err := os.ReadFile(path) + require.NoError(t, err, "ReadFile should not return an error") + require.Equal(t, oldFile, data, "AtomicWrite should not overwrite the file") + + return + } + require.NoError(t, err, "AtomicWrite should not return an error") + + // Check that the file was written + data, err := os.ReadFile(path) + require.NoError(t, err, "ReadFile should not return an error") + require.Equal(t, tc.data, data, "AtomicWrite should write the data to the file") + }) + } +} diff --git a/internal/report/export_test.go b/internal/report/export_test.go new file mode 100644 index 0000000..6a8629f --- /dev/null +++ b/internal/report/export_test.go @@ -0,0 +1,10 @@ +package report + +type ReportStash = reportStash + +// ReportStash returns the reportStash of the report. +// +//nolint:revive // This is a false positive as we returned a typed alias and not the private type. +func (r Report) ReportStash() ReportStash { + return r.reportStash +} diff --git a/internal/report/report.go b/internal/report/report.go new file mode 100644 index 0000000..6116851 --- /dev/null +++ b/internal/report/report.go @@ -0,0 +1,260 @@ +// Package report provides utility functions for handling reports. +package report + +// package report + +import ( + "encoding/json" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/ubuntu/ubuntu-insights/internal/constants" + "github.com/ubuntu/ubuntu-insights/internal/fileutils" +) + +var ( + // ErrInvalidPeriod is returned when a function requiring a period, received an invalid, period that isn't a non-negative integer. + ErrInvalidPeriod = errors.New("invalid period, period should be a positive integer") + + // ErrInvalidReportExt is returned when a report file has an invalid extension. + ErrInvalidReportExt = errors.New("invalid report file extension") + + // ErrInvalidReportName is returned when a report file has an invalid name that can't be parsed. + ErrInvalidReportName = errors.New("invalid report file name") +) + +// Report represents a report file. +type Report struct { + Path string // Path is the path to the report file. + Name string // Name is the name of the report file, including extension. + TimeStamp int64 // TimeStamp is the timestamp of the report. + + reportStash reportStash +} + +// reportStash is a helper struct to store a report and its data for movement. +type reportStash struct { + Path string + Data []byte +} + +// New creates a new Report object from a path. +// It does not write to the file system, or validate the path. +func New(path string) (Report, error) { + if filepath.Ext(path) != constants.ReportExt { + return Report{}, ErrInvalidReportExt + } + + rTime, err := getReportTime(filepath.Base(path)) + if err != nil { + return Report{}, err + } + + return Report{Path: path, Name: filepath.Base(path), TimeStamp: rTime}, nil +} + +// ReadJSON reads the JSON data from the report file. +func (r Report) ReadJSON() ([]byte, error) { + // Read the report file + data, err := os.ReadFile(r.Path) + if err != nil { + return nil, fmt.Errorf("failed to read report file: %v", err) + } + + if !json.Valid(data) { + return nil, fmt.Errorf("invalid JSON data in report file") + } + + return data, nil +} + +// MarkAsProcessed moves the report to a destination directory, and writes the data to the report. +// The original report is removed. +// +// The new report is returned, and the original data is stashed for use with UndoProcessed. +// Note that calling MarkAsProcessed multiple times on the same report will overwrite the stashed data. +func (r Report) MarkAsProcessed(dest string, data []byte) (Report, error) { + origData, err := r.ReadJSON() + if err != nil { + return Report{}, fmt.Errorf("failed to read original report: %v", err) + } + + newReport := Report{Path: filepath.Join(dest, r.Name), Name: r.Name, TimeStamp: r.TimeStamp, + reportStash: reportStash{Path: r.Path, Data: origData}} + + if err := fileutils.AtomicWrite(newReport.Path, data); err != nil { + return Report{}, fmt.Errorf("failed to write report: %v", err) + } + + if err := os.Remove(r.Path); err != nil { + return Report{}, fmt.Errorf("failed to remove report: %v", err) + } + + return newReport, nil +} + +// UndoProcessed moves the report back to the original directory, and writes the original data to the report. +// The new report is returned, and the original data is removed. +func (r Report) UndoProcessed() (Report, error) { + if r.reportStash.Path == "" { + return Report{}, errors.New("no stashed data to restore") + } + + if err := fileutils.AtomicWrite(r.reportStash.Path, r.reportStash.Data); err != nil { + return Report{}, fmt.Errorf("failed to write report: %v", err) + } + + if err := os.Remove(r.Path); err != nil { + return Report{}, fmt.Errorf("failed to remove report: %v", err) + } + + newReport := Report{Path: r.reportStash.Path, Name: r.Name, TimeStamp: r.TimeStamp} + return newReport, nil +} + +// getReportTime returns a int64 representation of the report time from the report path. +func getReportTime(path string) (int64, error) { + fileName := filepath.Base(path) + i, err := strconv.ParseInt(strings.TrimSuffix(fileName, filepath.Ext(fileName)), 10, 64) + if err != nil { + return i, fmt.Errorf("%w: %v", ErrInvalidReportName, err) + } + return i, nil +} + +// GetPeriodStart returns the start of the period window for a given period in seconds. +func GetPeriodStart(period int, t time.Time) (int64, error) { + if period <= 0 { + return 0, ErrInvalidPeriod + } + return t.Unix() - (t.Unix() % int64(period)), nil +} + +// GetForPeriod returns the most recent report within a period window for a given directory. +// Not inclusive of the period end (periodStart + period). +// +// For example, given reports 1 and 7, with time 2 and period 7, the function will return the path for report 1. +func GetForPeriod(dir string, t time.Time, period int) (Report, error) { + periodStart, err := GetPeriodStart(period, t) + if err != nil { + return Report{}, err + } + periodEnd := periodStart + int64(period) + + // Reports names are utc timestamps. Get the most recent report within the period window. + var report Report + err = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return fmt.Errorf("failed to access path: %v", err) + } + + // Skip subdirectories. + if d.IsDir() && path != dir { + return filepath.SkipDir + } + + r, err := New(path) + if errors.Is(err, ErrInvalidReportExt) || errors.Is(err, ErrInvalidReportName) { + slog.Info("Skipping non-report file", "file", d.Name(), "error", err) + return nil + } else if err != nil { + return fmt.Errorf("failed to create report object: %v", err) + } + + if r.TimeStamp < periodStart { + return nil + } + if r.TimeStamp >= periodEnd { + return filepath.SkipDir + } + + report = r + return nil + }) + + if err != nil { + return Report{}, err + } + + return report, nil +} + +// GetPerPeriod returns the latest report within each period window for a given directory. +// The key of the map is the start of the period window, and the value is a Report object. +// +// If period is 1, then all reports in the dir are returned. +func GetPerPeriod(dir string, period int) (map[int64]Report, error) { + if period <= 0 { + return nil, ErrInvalidPeriod + } + + reports := make(map[int64]Report) + err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return fmt.Errorf("failed to access path: %v", err) + } + + if d.IsDir() && path != dir { + return filepath.SkipDir + } + + r, err := New(path) + if errors.Is(err, ErrInvalidReportExt) || errors.Is(err, ErrInvalidReportName) { + slog.Info("Skipping non-report file", "file", d.Name(), "error", err) + return nil + } else if err != nil { + return fmt.Errorf("failed to create report object: %v", err) + } + + periodStart := r.TimeStamp - (r.TimeStamp % int64(period)) + if existingReport, ok := reports[periodStart]; !ok || existingReport.TimeStamp < r.TimeStamp { + reports[periodStart] = r + } + + return nil + }) + + if err != nil { + return nil, err + } + + return reports, nil +} + +// GetAll returns all reports in a given directory. +// Reports are expected to have the correct file extension, and have a name which can be parsed by a timestamp. +// Does not traverse subdirectories. +func GetAll(dir string) ([]Report, error) { + reports := make([]Report, 0) + err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return fmt.Errorf("failed to access path: %v", err) + } + + if d.IsDir() && path != dir { + return filepath.SkipDir + } + + r, err := New(path) + if errors.Is(err, ErrInvalidReportExt) || errors.Is(err, ErrInvalidReportName) { + slog.Info("Skipping non-report file", "file", d.Name(), "error", err) + return nil + } else if err != nil { + return fmt.Errorf("failed to create report object: %v", err) + } + + reports = append(reports, r) + return nil + }) + if err != nil { + return nil, err + } + + return reports, nil +} diff --git a/internal/report/report_test.go b/internal/report/report_test.go new file mode 100644 index 0000000..0764e69 --- /dev/null +++ b/internal/report/report_test.go @@ -0,0 +1,568 @@ +package report_test + +import ( + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/ubuntu/ubuntu-insights/internal/report" + "github.com/ubuntu/ubuntu-insights/internal/testutils" +) + +func TestGetPeriodStart(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + period int + time int64 + + wantErr error + }{ + "Valid Period": {period: 500, time: 100000}, + "Negative Time:": {period: 500, time: -100000}, + "Non-Multiple Time": {period: 500, time: 1051}, + + "Invalid Negative Period": {period: -500, wantErr: report.ErrInvalidPeriod}, + "Invalid Zero Period": {period: 0, wantErr: report.ErrInvalidPeriod}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + got, err := report.GetPeriodStart(tc.period, time.Unix(tc.time, 0)) + if tc.wantErr != nil { + require.ErrorIs(t, err, tc.wantErr) + return + } + require.NoError(t, err, "got an unexpected error") + + require.IsType(t, int64(0), got) + want := testutils.LoadWithUpdateFromGoldenYAML(t, got) + require.Equal(t, want, got, "GetPeriodStart should return the expect start of the period window") + }) + } +} + +func TestNew(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + path string + + wantErr bool + }{ + "Valid Report": {path: "1627847285.json"}, + "Valid Report with Path": {path: "/some/dir/1627847285.json"}, + + "Empty File Name": {path: ".json", wantErr: true}, + "Invalid Report Time": {path: "invalid.json", wantErr: true}, + "Invalid Report Mixed": {path: "i-1.json", wantErr: true}, + "Invalid Report Time with Path": {path: "/123/123/invalid.json", wantErr: true}, + "Alt Extension": {path: "1627847285.txt", wantErr: true}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + got, err := report.New(tc.path) + if tc.wantErr { + require.Error(t, err, "expected an error but got none") + return + } + require.NoError(t, err, "got an unexpected error") + + want := testutils.LoadWithUpdateFromGoldenYAML(t, got) + require.Equal(t, want, got, "New should return a new report object") + }) + } +} + +func TestGetForPeriod(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + files []string + subDir string + subDirFiles []string + time int64 + period int + invalidDir bool + + wantSpecificErr error + wantGenericErr bool + }{ + "Empty Directory": {time: 1, period: 500}, + "Files in subDir": {subDir: "subdir", subDirFiles: []string{"1.json", "2.json"}, time: 1, period: 500}, + "Empty subDir": {subDir: "subdir", time: 1, period: 500}, + "Invalid File Extension": {files: []string{"1.txt", "2.txt"}, time: 1, period: 500}, + "Invalid File Names": {files: []string{"i-1.json", "i-2.json", "i-3.json", "test.json", "one.json"}, time: -100, period: 500}, + + "Specific Time Single Valid Report": {files: []string{"1.json", "2.json"}, time: 2, period: 1}, + "Negative Timestamp": {files: []string{"-100.json", "-101.json"}, time: -150, period: 100}, + "Not Inclusive Period": {files: []string{"1.json", "7.json"}, time: 2, period: 7}, + + "Invalid Negative Period": {files: []string{"1.json", "7.json"}, time: 2, period: -7, wantSpecificErr: report.ErrInvalidPeriod}, + "Invalid Zero Period": {files: []string{"1.json", "7.json"}, time: 2, period: 0, wantSpecificErr: report.ErrInvalidPeriod}, + + "Invalid Dir": {period: 1, invalidDir: true, wantGenericErr: true}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + dir, err := setupNoDataDir(t, tc.files, tc.subDir, tc.subDirFiles) + require.NoError(t, err, "Setup: failed to setup temporary directory") + if tc.invalidDir { + dir = filepath.Join(dir, "invalid dir") + } + + r, err := report.GetForPeriod(dir, time.Unix(tc.time, 0), tc.period) + if tc.wantSpecificErr != nil { + require.ErrorIs(t, err, tc.wantSpecificErr) + return + } + if tc.wantGenericErr { + require.Error(t, err, "expected an error but got none") + return + } + require.NoError(t, err, "got an unexpected error") + + got := sanitizeReportPath(t, r, dir) + + want := testutils.LoadWithUpdateFromGoldenYAML(t, got) + require.Equal(t, want, got, "GetReportPath should return the most recent report within the period window") + }) + } +} + +func TestGetPerPeriod(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + files []string + subDir string + subDirFiles []string + period int + invalidDir bool + + wantSpecificErr error + wantGenericErr bool + }{ + "Empty Directory": {period: 500}, + "Files in subDir": {subDir: "subdir", subDirFiles: []string{"1.json", "2.json"}, period: 500}, + + "Invalid File Extension": {files: []string{"1.txt", "2.txt"}, period: 500}, + "Invalid File Names": {files: []string{"i-1.json", "i-2.json", "i-3.json", "test.json", "one.json"}, period: 500}, + "Mix of Valid and Invalid": {files: []string{"1.json", "2.json", "i-1.json", "i-2.json", "i-3.json", "test.json", "five.json"}, period: 500}, + + "Get Newest of Period": {files: []string{"1.json", "7.json"}, period: 100}, + "Multiple Consecutive Windows": {files: []string{"1.json", "7.json", "101.json", "107.json", "201.json", "207.json"}, period: 100}, + "Multiple Non-Consecutive Windows": {files: []string{"1.json", "7.json", "101.json", "107.json", "251.json", "257.json"}, period: 50}, + "Get All Reports": {files: []string{"1.json", "2.json", "3.json", "101.json", "107.json", "251.json", "257.json"}, period: 1}, + + "Invalid Negative Period": {files: []string{"1.json", "7.json"}, period: -7, wantSpecificErr: report.ErrInvalidPeriod}, + "Invalid Zero Period": {files: []string{"1.json", "7.json"}, period: 0, wantSpecificErr: report.ErrInvalidPeriod}, + + "Invalid Dir": {period: 1, invalidDir: true, wantGenericErr: true}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + dir, err := setupNoDataDir(t, tc.files, tc.subDir, tc.subDirFiles) + require.NoError(t, err, "Setup: failed to setup temporary directory") + if tc.invalidDir { + dir = filepath.Join(dir, "invalid dir") + } + + reports, err := report.GetPerPeriod(dir, tc.period) + if tc.wantSpecificErr != nil { + require.ErrorIs(t, err, tc.wantSpecificErr) + return + } + if tc.wantGenericErr { + require.Error(t, err, "expected an error but got none") + return + } + require.NoError(t, err, "got an unexpected error") + + got := make(map[int64]report.Report, len(reports)) + for n, r := range reports { + got[n] = sanitizeReportPath(t, r, dir) + } + want := testutils.LoadWithUpdateFromGoldenYAML(t, got) + require.Equal(t, want, got, "GetReports should return the most recent report within each period window") + }) + } +} + +func TestGetAll(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + files []string + subDir string + subDirFiles []string + invalidDir bool + + wantErr bool + }{ + "Empty Directory": {}, + "Files in subDir": {files: []string{"1.json", "2.json"}, subDir: "subdir", subDirFiles: []string{"1.json", "2.json"}}, + "Invalid File Extension": {files: []string{"1.txt", "2.txt"}}, + "Invalid File Names": {files: []string{"i-1.json", "i-2.json", "i-3.json", "test.json", "one.json"}}, + "Mix of Valid and Invalid": {files: []string{"1.json", "2.json", "500.json", "i-1.json", "i-2.json", "i-3.json", "test.json", "five.json"}}, + + "Invalid Dir": {invalidDir: true, wantErr: true}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + dir, err := setupNoDataDir(t, tc.files, tc.subDir, tc.subDirFiles) + require.NoError(t, err, "Setup: failed to setup temporary directory") + if tc.invalidDir { + dir = filepath.Join(dir, "invalid dir") + } + + reports, err := report.GetAll(dir) + if tc.wantErr { + require.Error(t, err, "expected an error but got none") + return + } + require.NoError(t, err, "got an unexpected error") + + got := make([]report.Report, 0, len(reports)) + for _, r := range reports { + got = append(got, sanitizeReportPath(t, r, dir)) + } + want := testutils.LoadWithUpdateFromGoldenYAML(t, got) + require.Equal(t, want, got, "GetAllReports should return all reports in the directory") + }) + } +} + +func TestMarkAsProcessed(t *testing.T) { + t.Parallel() + + type got struct { + Report report.Report + SrcFiles map[string]string + DstFiles map[string]string + } + + tests := map[string]struct { + srcFile map[string]string + dstFile map[string]string + + fileName string + data []byte + srcFilePerms os.FileMode + dstFilePerms os.FileMode + + wantErr bool + }{ + "Basic Move": { + srcFile: map[string]string{"1.json": `{"test": true}`}, + dstFile: map[string]string{}, + fileName: "1.json", + data: []byte(`{"test": true}`), + srcFilePerms: os.FileMode(0o600), + dstFilePerms: os.FileMode(0o600), + wantErr: false, + }, + "Basic Move New Data": { + srcFile: map[string]string{"1.json": `{"test": true}`}, + dstFile: map[string]string{}, + fileName: "1.json", + data: []byte("new data"), + srcFilePerms: os.FileMode(0o600), + dstFilePerms: os.FileMode(0o600), + wantErr: false, + }, + "Basic Move Overwrite": { + srcFile: map[string]string{"1.json": `{"test": true}`}, + dstFile: map[string]string{"1.json": "old data"}, + fileName: "1.json", + data: []byte("new data"), + srcFilePerms: os.FileMode(0o600), + dstFilePerms: os.FileMode(0o600), + wantErr: false, + }, "SrcPerm None": { + srcFile: map[string]string{"1.json": `{"test": true}`}, + dstFile: map[string]string{}, + fileName: "1.json", + data: []byte(`{"test": true}`), + srcFilePerms: os.FileMode(000), + dstFilePerms: os.FileMode(0o600), + wantErr: runtime.GOOS != "windows", + }, "DstPerm None": { + srcFile: map[string]string{"1.json": `{"test": true}`}, + dstFile: map[string]string{"1.json": "old data"}, + fileName: "1.json", + data: []byte("new data"), + srcFilePerms: os.FileMode(0o600), + dstFilePerms: os.FileMode(000), + wantErr: runtime.GOOS == "windows", + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + rootDir, srcDir, dstDir := setupProcessingDirs(t) + + setupBasicDir(t, tc.srcFile, tc.srcFilePerms, srcDir) + setupBasicDir(t, tc.dstFile, tc.dstFilePerms, dstDir) + + r, err := report.New(filepath.Join(srcDir, tc.fileName)) + require.NoError(t, err, "Setup: failed to create report object") + + r, err = r.MarkAsProcessed(dstDir, tc.data) + if tc.wantErr { + require.Error(t, err, "expected an error but got none") + return + } + require.NoError(t, err, "got an unexpected error") + + dstDirContents, err := testutils.GetDirContents(t, dstDir, 2) + require.NoError(t, err, "failed to get directory contents") + + srcDirContents, err := testutils.GetDirContents(t, srcDir, 2) + require.NoError(t, err, "failed to get directory contents") + + r = sanitizeReportPath(t, r, rootDir) + got := got{Report: r, SrcFiles: srcDirContents, DstFiles: dstDirContents} + want := testutils.LoadWithUpdateFromGoldenYAML(t, got) + require.EqualExportedValues(t, want, got, "MarkAsProcessed should move the report to the processed directory") + }) + } +} + +func TestUndoProcessed(t *testing.T) { + t.Parallel() + + type got struct { + Report report.Report + SrcFiles map[string]string + DstFiles map[string]string + } + + tests := map[string]struct { + srcFile map[string]string + dstFile map[string]string + + fileName string + data []byte + + wantErr bool + }{ + "Basic Move": { + srcFile: map[string]string{"1.json": `{"test": true}`}, + dstFile: map[string]string{}, + fileName: "1.json", + data: []byte(`"new data"`), + wantErr: false, + }, + "Basic Move New Data": { + srcFile: map[string]string{"1.json": `{"test": true}`}, + dstFile: map[string]string{}, + fileName: "1.json", + data: []byte("new data"), + wantErr: false, + }, + "Basic Move Overwrite": { + srcFile: map[string]string{"1.json": `{"test": true}`}, + dstFile: map[string]string{"1.json": "old data"}, + fileName: "1.json", + data: []byte("new data"), + wantErr: false, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + rootDir, srcDir, dstDir := setupProcessingDirs(t) + + setupBasicDir(t, tc.srcFile, 0600, srcDir) + setupBasicDir(t, tc.dstFile, 0600, dstDir) + + r, err := report.New(filepath.Join(srcDir, tc.fileName)) + require.NoError(t, err, "Setup: failed to create report object") + + r, err = r.MarkAsProcessed(dstDir, tc.data) + require.NoError(t, err, "Setup: failed to mark report as processed") + + r, err = r.UndoProcessed() + if tc.wantErr { + require.Error(t, err, "expected an error but got none") + return + } + require.NoError(t, err, "got an unexpected error") + + dstDirContents, err := testutils.GetDirContents(t, dstDir, 2) + require.NoError(t, err, "failed to get directory contents") + + srcDirContents, err := testutils.GetDirContents(t, srcDir, 2) + require.NoError(t, err, "failed to get directory contents") + + r = sanitizeReportPath(t, r, rootDir) + got := got{Report: r, SrcFiles: srcDirContents, DstFiles: dstDirContents} + want := testutils.LoadWithUpdateFromGoldenYAML(t, got) + require.EqualExportedValues(t, want, got, "UndoProcessed should move the report to the processed directory") + }) + } +} + +func TestUndoProcessedNoStash(t *testing.T) { + t.Parallel() + + r, err := report.New("1.json") + require.NoError(t, err, "Setup: failed to create report object") + + _, err = r.UndoProcessed() + require.Error(t, err, "UndoProcessed should return an error if the report has not been marked as processed") +} + +func TestMarkAsProcessedNoFile(t *testing.T) { + t.Parallel() + + _, srcDir, dstDir := setupProcessingDirs(t) + r, err := report.New(filepath.Join(srcDir, "1.json")) + require.NoError(t, err, "Setup: failed to create report object") + + _, err = r.MarkAsProcessed(dstDir, []byte(`"new data"`)) + require.Error(t, err, "MarkAsProcessed should return an error if the report file does not exist") +} + +func TestUndoProcessedNoFile(t *testing.T) { + t.Parallel() + + _, srcDir, dstDir := setupProcessingDirs(t) + reportPath := filepath.Join(srcDir, "1.json") + require.NoError(t, os.WriteFile(reportPath, []byte(`{"test": true}`), 0600), "Setup: failed to write report file") + r, err := report.New(reportPath) + require.NoError(t, err, "Setup: failed to create report object") + + r, err = r.MarkAsProcessed(dstDir, []byte(`"new data"`)) + require.NoError(t, err, "Setup: failed to mark report as processed") + + require.NoError(t, os.Remove(r.Path), "Setup: failed to remove report file") + + _, err = r.UndoProcessed() + require.Error(t, err, "UndoProcessed should return an error if the report file does not exist") +} + +func TestReadJSON(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + files map[string]string + file string + + wantErr bool + }{ + "Basic Read": {files: map[string]string{"1.json": `{"test": true}`}, file: "1.json"}, + "Multiple Files": {files: map[string]string{"1.json": `{"test": true}`, "2.json": `{"test": false}`}, file: "1.json"}, + + "Empty File": {files: map[string]string{"1.json": ""}, file: "1.json", wantErr: true}, + "Invalid JSON": {files: map[string]string{"1.json": `{"test":::`}, file: "1.json", wantErr: true}, + "No File": {files: map[string]string{}, file: "1.json", wantErr: true}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + _, srcDir, _ := setupProcessingDirs(t) + + setupBasicDir(t, tc.files, 0600, srcDir) + + r, err := report.New(filepath.Join(srcDir, tc.file)) + require.NoError(t, err, "Setup: failed to create report object") + + data, err := r.ReadJSON() + if tc.wantErr { + require.Error(t, err, "expected an error but got none") + return + } + require.NoError(t, err, "got an unexpected error") + + got := string(data) + want := testutils.LoadWithUpdateFromGolden(t, got) + require.Equal(t, want, got, "ReadJSON should return the data from the report file") + }) + } +} + +func setupProcessingDirs(t *testing.T) (rootDir, srcDir, dstDir string) { + t.Helper() + rootDir = t.TempDir() + srcDir = filepath.Join(rootDir, "src") + dstDir = filepath.Join(rootDir, "dst") + err := os.MkdirAll(srcDir, 0700) + require.NoError(t, err, "Setup: failed to create source directory") + err = os.MkdirAll(dstDir, 0700) + require.NoError(t, err, "Setup: failed to create destination directory") + return rootDir, srcDir, dstDir +} + +func setupBasicDir(t *testing.T, files map[string]string, perms os.FileMode, dir string) { + t.Helper() + for file, data := range files { + path := filepath.Join(dir, file) + err := os.WriteFile(path, []byte(data), perms) + require.NoError(t, err, "Setup: failed to write file") + t.Cleanup(func() { + _ = os.Chmod(path, os.FileMode(0600)) + }) + } +} + +func setupNoDataDir(t *testing.T, files []string, subDir string, subDirFiles []string) (string, error) { + t.Helper() + + dir := t.TempDir() + for _, file := range files { + path := filepath.Join(dir, file) + if err := os.WriteFile(path, []byte{}, 0600); err != nil { + return "", err + } + } + + if subDir != "" { + subDirPath := filepath.Join(dir, subDir) + if err := os.Mkdir(subDirPath, 0700); err != nil { + return "", err + } + + for _, file := range subDirFiles { + path := filepath.Join(subDirPath, file) + if err := os.WriteFile(path, []byte{}, 0600); err != nil { + return "", err + } + } + } + + return dir, nil +} + +func sanitizeReportPath(t *testing.T, r report.Report, dir string) report.Report { + t.Helper() + if r.Path == "" { + return r + } + + fp, err := filepath.Rel(dir, r.Path) + if err != nil { + require.NoError(t, err, "failed to get relative path") + return report.Report{} + } + return report.Report{Path: filepath.ToSlash(fp), Name: r.Name, TimeStamp: r.TimeStamp} +} diff --git a/internal/report/testdata/TestGetAll/golden/empty_directory b/internal/report/testdata/TestGetAll/golden/empty_directory new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/internal/report/testdata/TestGetAll/golden/empty_directory @@ -0,0 +1 @@ +[] diff --git a/internal/report/testdata/TestGetAll/golden/files_in_subdir b/internal/report/testdata/TestGetAll/golden/files_in_subdir new file mode 100644 index 0000000..a079365 --- /dev/null +++ b/internal/report/testdata/TestGetAll/golden/files_in_subdir @@ -0,0 +1,6 @@ +- path: 1.json + name: 1.json + timestamp: 1 +- path: 2.json + name: 2.json + timestamp: 2 diff --git a/internal/report/testdata/TestGetAll/golden/invalid_file_extension b/internal/report/testdata/TestGetAll/golden/invalid_file_extension new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/internal/report/testdata/TestGetAll/golden/invalid_file_extension @@ -0,0 +1 @@ +[] diff --git a/internal/report/testdata/TestGetAll/golden/invalid_file_names b/internal/report/testdata/TestGetAll/golden/invalid_file_names new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/internal/report/testdata/TestGetAll/golden/invalid_file_names @@ -0,0 +1 @@ +[] diff --git a/internal/report/testdata/TestGetAll/golden/mix_of_valid_and_invalid b/internal/report/testdata/TestGetAll/golden/mix_of_valid_and_invalid new file mode 100644 index 0000000..cd1e267 --- /dev/null +++ b/internal/report/testdata/TestGetAll/golden/mix_of_valid_and_invalid @@ -0,0 +1,9 @@ +- path: 1.json + name: 1.json + timestamp: 1 +- path: 2.json + name: 2.json + timestamp: 2 +- path: 500.json + name: 500.json + timestamp: 500 diff --git a/internal/report/testdata/TestGetForPeriod/golden/empty_directory b/internal/report/testdata/TestGetForPeriod/golden/empty_directory new file mode 100644 index 0000000..ec9107d --- /dev/null +++ b/internal/report/testdata/TestGetForPeriod/golden/empty_directory @@ -0,0 +1,3 @@ +path: "" +name: "" +timestamp: 0 diff --git a/internal/report/testdata/TestGetForPeriod/golden/empty_subdir b/internal/report/testdata/TestGetForPeriod/golden/empty_subdir new file mode 100644 index 0000000..ec9107d --- /dev/null +++ b/internal/report/testdata/TestGetForPeriod/golden/empty_subdir @@ -0,0 +1,3 @@ +path: "" +name: "" +timestamp: 0 diff --git a/internal/report/testdata/TestGetForPeriod/golden/files_in_subdir b/internal/report/testdata/TestGetForPeriod/golden/files_in_subdir new file mode 100644 index 0000000..ec9107d --- /dev/null +++ b/internal/report/testdata/TestGetForPeriod/golden/files_in_subdir @@ -0,0 +1,3 @@ +path: "" +name: "" +timestamp: 0 diff --git a/internal/report/testdata/TestGetForPeriod/golden/invalid_file_extension b/internal/report/testdata/TestGetForPeriod/golden/invalid_file_extension new file mode 100644 index 0000000..ec9107d --- /dev/null +++ b/internal/report/testdata/TestGetForPeriod/golden/invalid_file_extension @@ -0,0 +1,3 @@ +path: "" +name: "" +timestamp: 0 diff --git a/internal/report/testdata/TestGetForPeriod/golden/invalid_file_names b/internal/report/testdata/TestGetForPeriod/golden/invalid_file_names new file mode 100644 index 0000000..ec9107d --- /dev/null +++ b/internal/report/testdata/TestGetForPeriod/golden/invalid_file_names @@ -0,0 +1,3 @@ +path: "" +name: "" +timestamp: 0 diff --git a/internal/report/testdata/TestGetForPeriod/golden/negative_timestamp b/internal/report/testdata/TestGetForPeriod/golden/negative_timestamp new file mode 100644 index 0000000..431aa05 --- /dev/null +++ b/internal/report/testdata/TestGetForPeriod/golden/negative_timestamp @@ -0,0 +1,3 @@ +path: -100.json +name: -100.json +timestamp: -100 diff --git a/internal/report/testdata/TestGetForPeriod/golden/not_inclusive_period b/internal/report/testdata/TestGetForPeriod/golden/not_inclusive_period new file mode 100644 index 0000000..56c8ba5 --- /dev/null +++ b/internal/report/testdata/TestGetForPeriod/golden/not_inclusive_period @@ -0,0 +1,3 @@ +path: 1.json +name: 1.json +timestamp: 1 diff --git a/internal/report/testdata/TestGetForPeriod/golden/specific_time_single_valid_report b/internal/report/testdata/TestGetForPeriod/golden/specific_time_single_valid_report new file mode 100644 index 0000000..7c4d2c9 --- /dev/null +++ b/internal/report/testdata/TestGetForPeriod/golden/specific_time_single_valid_report @@ -0,0 +1,3 @@ +path: 2.json +name: 2.json +timestamp: 2 diff --git a/internal/report/testdata/TestGetPerPeriod/golden/empty_directory b/internal/report/testdata/TestGetPerPeriod/golden/empty_directory new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/internal/report/testdata/TestGetPerPeriod/golden/empty_directory @@ -0,0 +1 @@ +{} diff --git a/internal/report/testdata/TestGetPerPeriod/golden/files_in_subdir b/internal/report/testdata/TestGetPerPeriod/golden/files_in_subdir new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/internal/report/testdata/TestGetPerPeriod/golden/files_in_subdir @@ -0,0 +1 @@ +{} diff --git a/internal/report/testdata/TestGetPerPeriod/golden/get_all_reports b/internal/report/testdata/TestGetPerPeriod/golden/get_all_reports new file mode 100644 index 0000000..e3e067a --- /dev/null +++ b/internal/report/testdata/TestGetPerPeriod/golden/get_all_reports @@ -0,0 +1,28 @@ +1: + path: 1.json + name: 1.json + timestamp: 1 +2: + path: 2.json + name: 2.json + timestamp: 2 +3: + path: 3.json + name: 3.json + timestamp: 3 +101: + path: 101.json + name: 101.json + timestamp: 101 +107: + path: 107.json + name: 107.json + timestamp: 107 +251: + path: 251.json + name: 251.json + timestamp: 251 +257: + path: 257.json + name: 257.json + timestamp: 257 diff --git a/internal/report/testdata/TestGetPerPeriod/golden/get_newest_of_period b/internal/report/testdata/TestGetPerPeriod/golden/get_newest_of_period new file mode 100644 index 0000000..d6fdc70 --- /dev/null +++ b/internal/report/testdata/TestGetPerPeriod/golden/get_newest_of_period @@ -0,0 +1,4 @@ +0: + path: 7.json + name: 7.json + timestamp: 7 diff --git a/internal/report/testdata/TestGetPerPeriod/golden/invalid_file_extension b/internal/report/testdata/TestGetPerPeriod/golden/invalid_file_extension new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/internal/report/testdata/TestGetPerPeriod/golden/invalid_file_extension @@ -0,0 +1 @@ +{} diff --git a/internal/report/testdata/TestGetPerPeriod/golden/invalid_file_names b/internal/report/testdata/TestGetPerPeriod/golden/invalid_file_names new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/internal/report/testdata/TestGetPerPeriod/golden/invalid_file_names @@ -0,0 +1 @@ +{} diff --git a/internal/report/testdata/TestGetPerPeriod/golden/mix_of_valid_and_invalid b/internal/report/testdata/TestGetPerPeriod/golden/mix_of_valid_and_invalid new file mode 100644 index 0000000..5f45d53 --- /dev/null +++ b/internal/report/testdata/TestGetPerPeriod/golden/mix_of_valid_and_invalid @@ -0,0 +1,4 @@ +0: + path: 2.json + name: 2.json + timestamp: 2 diff --git a/internal/report/testdata/TestGetPerPeriod/golden/multiple_consecutive_windows b/internal/report/testdata/TestGetPerPeriod/golden/multiple_consecutive_windows new file mode 100644 index 0000000..d777da2 --- /dev/null +++ b/internal/report/testdata/TestGetPerPeriod/golden/multiple_consecutive_windows @@ -0,0 +1,12 @@ +0: + path: 7.json + name: 7.json + timestamp: 7 +100: + path: 107.json + name: 107.json + timestamp: 107 +200: + path: 207.json + name: 207.json + timestamp: 207 diff --git a/internal/report/testdata/TestGetPerPeriod/golden/multiple_non-consecutive_windows b/internal/report/testdata/TestGetPerPeriod/golden/multiple_non-consecutive_windows new file mode 100644 index 0000000..9fb936e --- /dev/null +++ b/internal/report/testdata/TestGetPerPeriod/golden/multiple_non-consecutive_windows @@ -0,0 +1,12 @@ +0: + path: 7.json + name: 7.json + timestamp: 7 +100: + path: 107.json + name: 107.json + timestamp: 107 +250: + path: 257.json + name: 257.json + timestamp: 257 diff --git a/internal/report/testdata/TestGetPeriodStart/golden/negative_time b/internal/report/testdata/TestGetPeriodStart/golden/negative_time new file mode 100644 index 0000000..c6730fa --- /dev/null +++ b/internal/report/testdata/TestGetPeriodStart/golden/negative_time @@ -0,0 +1 @@ +-100000 diff --git a/internal/report/testdata/TestGetPeriodStart/golden/non-multiple_time b/internal/report/testdata/TestGetPeriodStart/golden/non-multiple_time new file mode 100644 index 0000000..83b33d2 --- /dev/null +++ b/internal/report/testdata/TestGetPeriodStart/golden/non-multiple_time @@ -0,0 +1 @@ +1000 diff --git a/internal/report/testdata/TestGetPeriodStart/golden/valid_period b/internal/report/testdata/TestGetPeriodStart/golden/valid_period new file mode 100644 index 0000000..f7393e8 --- /dev/null +++ b/internal/report/testdata/TestGetPeriodStart/golden/valid_period @@ -0,0 +1 @@ +100000 diff --git a/internal/report/testdata/TestMarkAsProcessed/golden/basic_move b/internal/report/testdata/TestMarkAsProcessed/golden/basic_move new file mode 100644 index 0000000..fa8bd51 --- /dev/null +++ b/internal/report/testdata/TestMarkAsProcessed/golden/basic_move @@ -0,0 +1,7 @@ +report: + path: dst/1.json + name: 1.json + timestamp: 1 +srcfiles: {} +dstfiles: + 1.json: '{"test": true}' diff --git a/internal/report/testdata/TestMarkAsProcessed/golden/basic_move_new_data b/internal/report/testdata/TestMarkAsProcessed/golden/basic_move_new_data new file mode 100644 index 0000000..0948d76 --- /dev/null +++ b/internal/report/testdata/TestMarkAsProcessed/golden/basic_move_new_data @@ -0,0 +1,7 @@ +report: + path: dst/1.json + name: 1.json + timestamp: 1 +srcfiles: {} +dstfiles: + 1.json: new data diff --git a/internal/report/testdata/TestMarkAsProcessed/golden/basic_move_overwrite b/internal/report/testdata/TestMarkAsProcessed/golden/basic_move_overwrite new file mode 100644 index 0000000..0948d76 --- /dev/null +++ b/internal/report/testdata/TestMarkAsProcessed/golden/basic_move_overwrite @@ -0,0 +1,7 @@ +report: + path: dst/1.json + name: 1.json + timestamp: 1 +srcfiles: {} +dstfiles: + 1.json: new data diff --git a/internal/report/testdata/TestMarkAsProcessed/golden/dstperm_none b/internal/report/testdata/TestMarkAsProcessed/golden/dstperm_none new file mode 100644 index 0000000..0948d76 --- /dev/null +++ b/internal/report/testdata/TestMarkAsProcessed/golden/dstperm_none @@ -0,0 +1,7 @@ +report: + path: dst/1.json + name: 1.json + timestamp: 1 +srcfiles: {} +dstfiles: + 1.json: new data diff --git a/internal/report/testdata/TestMarkAsProcessed/golden/srcperm_none b/internal/report/testdata/TestMarkAsProcessed/golden/srcperm_none new file mode 100644 index 0000000..fa8bd51 --- /dev/null +++ b/internal/report/testdata/TestMarkAsProcessed/golden/srcperm_none @@ -0,0 +1,7 @@ +report: + path: dst/1.json + name: 1.json + timestamp: 1 +srcfiles: {} +dstfiles: + 1.json: '{"test": true}' diff --git a/internal/report/testdata/TestNew/golden/valid_report b/internal/report/testdata/TestNew/golden/valid_report new file mode 100644 index 0000000..7ce2121 --- /dev/null +++ b/internal/report/testdata/TestNew/golden/valid_report @@ -0,0 +1,3 @@ +path: 1627847285.json +name: 1627847285.json +timestamp: 1627847285 diff --git a/internal/report/testdata/TestNew/golden/valid_report_with_path b/internal/report/testdata/TestNew/golden/valid_report_with_path new file mode 100644 index 0000000..252b842 --- /dev/null +++ b/internal/report/testdata/TestNew/golden/valid_report_with_path @@ -0,0 +1,3 @@ +path: /some/dir/1627847285.json +name: 1627847285.json +timestamp: 1627847285 diff --git a/internal/report/testdata/TestReadJSON/golden/basic_read b/internal/report/testdata/TestReadJSON/golden/basic_read new file mode 100644 index 0000000..fb25aa1 --- /dev/null +++ b/internal/report/testdata/TestReadJSON/golden/basic_read @@ -0,0 +1 @@ +{"test": true} \ No newline at end of file diff --git a/internal/report/testdata/TestReadJSON/golden/multiple_files b/internal/report/testdata/TestReadJSON/golden/multiple_files new file mode 100644 index 0000000..fb25aa1 --- /dev/null +++ b/internal/report/testdata/TestReadJSON/golden/multiple_files @@ -0,0 +1 @@ +{"test": true} \ No newline at end of file diff --git a/internal/report/testdata/TestUndoProcessed/golden/basic_move b/internal/report/testdata/TestUndoProcessed/golden/basic_move new file mode 100644 index 0000000..3330bd0 --- /dev/null +++ b/internal/report/testdata/TestUndoProcessed/golden/basic_move @@ -0,0 +1,7 @@ +report: + path: src/1.json + name: 1.json + timestamp: 1 +srcfiles: + 1.json: '{"test": true}' +dstfiles: {} diff --git a/internal/report/testdata/TestUndoProcessed/golden/basic_move_new_data b/internal/report/testdata/TestUndoProcessed/golden/basic_move_new_data new file mode 100644 index 0000000..3330bd0 --- /dev/null +++ b/internal/report/testdata/TestUndoProcessed/golden/basic_move_new_data @@ -0,0 +1,7 @@ +report: + path: src/1.json + name: 1.json + timestamp: 1 +srcfiles: + 1.json: '{"test": true}' +dstfiles: {} diff --git a/internal/report/testdata/TestUndoProcessed/golden/basic_move_overwrite b/internal/report/testdata/TestUndoProcessed/golden/basic_move_overwrite new file mode 100644 index 0000000..3330bd0 --- /dev/null +++ b/internal/report/testdata/TestUndoProcessed/golden/basic_move_overwrite @@ -0,0 +1,7 @@ +report: + path: src/1.json + name: 1.json + timestamp: 1 +srcfiles: + 1.json: '{"test": true}' +dstfiles: {} diff --git a/internal/testutils/files.go b/internal/testutils/files.go index 84a0cf1..6ac9cb7 100644 --- a/internal/testutils/files.go +++ b/internal/testutils/files.go @@ -3,22 +3,19 @@ package testutils import ( + "bytes" + "fmt" "io" + "io/fs" "os" "path/filepath" "testing" - - "github.com/stretchr/testify/assert" ) -// CleanupDir removes the temporary directory including its contents. -func CleanupDir(t *testing.T, dir string) { +// CopyFile copies a file from source to destination. +func CopyFile(t *testing.T, src, dst string) error { t.Helper() - assert.NoError(t, os.RemoveAll(dir), "Cleanup: failed to remove temporary directory") -} -// CopyFile copies a file from source to destination. -func CopyFile(src, dst string) error { sourceFile, err := os.Open(src) if err != nil { return err @@ -36,8 +33,9 @@ func CopyFile(src, dst string) error { } // CopyDir copies the contents of a directory to another directory. -func CopyDir(srcDir, dstDir string) error { - return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error { +func CopyDir(t *testing.T, srcDir, dstDir string) error { + t.Helper() + return filepath.WalkDir(srcDir, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } @@ -46,9 +44,52 @@ func CopyDir(srcDir, dstDir string) error { return err } dstPath := filepath.Join(dstDir, relPath) - if info.IsDir() { - return os.MkdirAll(dstPath, info.Mode()) + if d.IsDir() { + return os.MkdirAll(dstPath, 0700) + } + return CopyFile(t, path, dstPath) + }) +} + +// GetDirContents returns the contents of a directory as a map of file paths to file contents. +// The contents are read as strings. +// The maxDepth parameter limits the depth of the directory tree to read. +func GetDirContents(t *testing.T, dir string, maxDepth uint) (map[string]string, error) { + t.Helper() + + files := make(map[string]string) + + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if path == dir { + return nil + } + + relPath, err := filepath.Rel(dir, path) + if err != nil { + return err } - return CopyFile(path, dstPath) + + depth := uint(len(filepath.SplitList(relPath))) + if depth > maxDepth { + return fmt.Errorf("max depth %d exceeded at %s", maxDepth, relPath) + } + + if !d.IsDir() { + content, err := os.ReadFile(path) + if err != nil { + return err + } + // Normalize content between Windows and Linux + content = bytes.ReplaceAll(content, []byte("\r\n"), []byte("\n")) + files[filepath.ToSlash(relPath)] = string(content) + } + + return nil }) + + return files, err } diff --git a/internal/uploader/export_test.go b/internal/uploader/export_test.go new file mode 100644 index 0000000..a9a4c3f --- /dev/null +++ b/internal/uploader/export_test.go @@ -0,0 +1,32 @@ +package uploader + +import "time" + +type MockTimeProvider struct { + CurrentTime int64 +} + +func (m MockTimeProvider) Now() time.Time { + return time.Unix(m.CurrentTime, 0) +} + +// WithCachePath sets the cache path for the uploader. +func WithCachePath(path string) Options { + return func(o *options) { + o.cachePath = path + } +} + +// WithBaseServerURL sets the base server URL for the uploader. +func WithBaseServerURL(url string) Options { + return func(o *options) { + o.baseServerURL = url + } +} + +// WithTimeProvider sets the time provider for the uploader. +func WithTimeProvider(tp timeProvider) Options { + return func(o *options) { + o.timeProvider = tp + } +} diff --git a/internal/uploader/internal_test.go b/internal/uploader/internal_test.go new file mode 100644 index 0000000..6ececef --- /dev/null +++ b/internal/uploader/internal_test.go @@ -0,0 +1,139 @@ +package uploader + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "github.com/ubuntu/ubuntu-insights/internal/constants" + "github.com/ubuntu/ubuntu-insights/internal/fileutils" + "github.com/ubuntu/ubuntu-insights/internal/report" +) + +func TestUploadBadFile(t *testing.T) { + t.Parallel() + basicContent := `{"Content":true, "string": "string"}` + badContent := `bad content` + + tests := map[string]struct { + fName string + fileContents string + missingFile bool + fileIsDir bool + url string + consent bool + + rNewErr bool + wantErr bool + }{ + "Ok": {fName: "0.json", fileContents: basicContent, wantErr: false}, + "Missing File": {fName: "0.json", fileContents: basicContent, missingFile: true, wantErr: true}, + "File Is Dir": {fName: "0.json", fileIsDir: true, wantErr: true}, + "Non-numeric": {fName: "not-numeric.json", fileContents: basicContent, rNewErr: true}, + "Bad File Ext": {fName: "0.txt", fileContents: basicContent, rNewErr: true}, + "Bad Contents": {fName: "0.json", fileContents: badContent, wantErr: true}, + "Bad URL": {fName: "0.json", fileContents: basicContent, url: "http://bad host:1234", wantErr: true}, + + "Ok Consent": {fName: "0.json", fileContents: basicContent, consent: true, wantErr: false}, + "Missing File Consent": {fName: "0.json", fileContents: basicContent, missingFile: true, consent: true, wantErr: true}, + "File Is Dir Consent": {fName: "0.json", fileIsDir: true, consent: true, wantErr: true}, + "Non-numeric Consent": {fName: "not-numeric.json", fileContents: basicContent, consent: true, rNewErr: true}, + "Bad File Ext Consent": {fName: "0.txt", fileContents: basicContent, consent: true, rNewErr: true}, + "Bad Contents Consent": {fName: "0.json", fileContents: badContent, consent: true, wantErr: true}, + "Bad URL Consent": {fName: "0.json", fileContents: basicContent, url: "http://bad host:1234", consent: true, wantErr: true}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + require.False(t, tc.missingFile && tc.fileIsDir, "Test case cannot have both missing file and file is dir") + + dir := t.TempDir() + + um := &Uploader{ + collectedDir: filepath.Join(dir, constants.LocalFolder), + uploadedDir: filepath.Join(dir, constants.UploadedFolder), + minAge: 0, + timeProvider: MockTimeProvider{CurrentTime: 0}, + } + + require.NoError(t, os.Mkdir(um.collectedDir, 0750), "Setup: failed to create uploaded folder") + require.NoError(t, os.Mkdir(um.uploadedDir, 0750), "Setup: failed to create collected folder") + fPath := filepath.Join(um.collectedDir, tc.fName) + + if !tc.missingFile && !tc.fileIsDir { + require.NoError(t, fileutils.AtomicWrite(fPath, []byte(tc.fileContents)), "Setup: failed to create report file") + } + if tc.fileIsDir { + require.NoError(t, os.Mkdir(fPath, 0750), "Setup: failed to create directory") + } + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(func() { ts.Close() }) + + if tc.url == "" { + tc.url = ts.URL + } + r, err := report.New(fPath) + if tc.rNewErr { + require.Error(t, err, "Setup: failed to create report object") + return + } + require.NoError(t, err, "Setup: failed to create report object") + err = um.upload(r, tc.url, tc.consent, false) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + }) + } +} + +func TestSend(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + url string + noServer bool + serverResponse int + + wantErr bool + }{ + "No Server": {noServer: true, wantErr: true}, + "Bad URL": {url: "http://local host:1234", serverResponse: http.StatusOK, wantErr: true}, + "Bad Response": {serverResponse: http.StatusForbidden, wantErr: true}, + + "Success": {serverResponse: http.StatusOK}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tc.serverResponse) + })) + t.Cleanup(func() { ts.Close() }) + + if tc.url == "" { + tc.url = ts.URL + } + if tc.noServer { + ts.Close() + } + + err := send(tc.url, []byte("payload")) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + }) + } +} diff --git a/internal/uploader/testdata/TestUpload/golden/bad_content b/internal/uploader/testdata/TestUpload/golden/bad_content new file mode 100644 index 0000000..9f97665 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/bad_content @@ -0,0 +1 @@ +source/local/1.json: bad content diff --git a/internal/uploader/testdata/TestUpload/golden/bad_response b/internal/uploader/testdata/TestUpload/golden/bad_response new file mode 100644 index 0000000..ec0b658 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/bad_response @@ -0,0 +1 @@ +source/local/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/bad_url b/internal/uploader/testdata/TestUpload/golden/bad_url new file mode 100644 index 0000000..ec0b658 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/bad_url @@ -0,0 +1 @@ +source/local/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/consent_manager_false b/internal/uploader/testdata/TestUpload/golden/consent_manager_false new file mode 100644 index 0000000..587b133 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/consent_manager_false @@ -0,0 +1 @@ +source/uploaded/1.json: '{"OptOut":true}' diff --git a/internal/uploader/testdata/TestUpload/golden/consent_manager_source_error b/internal/uploader/testdata/TestUpload/golden/consent_manager_source_error new file mode 100644 index 0000000..ec0b658 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/consent_manager_source_error @@ -0,0 +1 @@ +source/local/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/consent_manager_source_error_with_true b/internal/uploader/testdata/TestUpload/golden/consent_manager_source_error_with_true new file mode 100644 index 0000000..ec0b658 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/consent_manager_source_error_with_true @@ -0,0 +1 @@ +source/local/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/dry_run b/internal/uploader/testdata/TestUpload/golden/dry_run new file mode 100644 index 0000000..ec0b658 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/dry_run @@ -0,0 +1 @@ +source/local/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/duplicate_upload b/internal/uploader/testdata/TestUpload/golden/duplicate_upload new file mode 100644 index 0000000..e37f517 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/duplicate_upload @@ -0,0 +1,2 @@ +source/local/1.json: '{"Content":"normal content"}' +source/uploaded/1.json: bad content diff --git a/internal/uploader/testdata/TestUpload/golden/force_cm_false b/internal/uploader/testdata/TestUpload/golden/force_cm_false new file mode 100644 index 0000000..587b133 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/force_cm_false @@ -0,0 +1 @@ +source/uploaded/1.json: '{"OptOut":true}' diff --git a/internal/uploader/testdata/TestUpload/golden/force_duplicate b/internal/uploader/testdata/TestUpload/golden/force_duplicate new file mode 100644 index 0000000..f436ae8 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/force_duplicate @@ -0,0 +1 @@ +source/uploaded/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/force_min_age b/internal/uploader/testdata/TestUpload/golden/force_min_age new file mode 100644 index 0000000..e21264b --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/force_min_age @@ -0,0 +1,2 @@ +source/uploaded/1.json: '{"Content":"normal content"}' +source/uploaded/9.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/future_timestamp b/internal/uploader/testdata/TestUpload/golden/future_timestamp new file mode 100644 index 0000000..ce033c1 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/future_timestamp @@ -0,0 +1,2 @@ +source/local/11.json: '{"Content":"normal content"}' +source/uploaded/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/min_age b/internal/uploader/testdata/TestUpload/golden/min_age new file mode 100644 index 0000000..85afcf0 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/min_age @@ -0,0 +1,2 @@ +source/local/9.json: '{"Content":"normal content"}' +source/uploaded/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/multi_upload b/internal/uploader/testdata/TestUpload/golden/multi_upload new file mode 100644 index 0000000..91fd905 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/multi_upload @@ -0,0 +1,2 @@ +source/uploaded/1.json: '{"Content":"normal content"}' +source/uploaded/5.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/no_directory b/internal/uploader/testdata/TestUpload/golden/no_directory new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/no_directory @@ -0,0 +1 @@ +{} diff --git a/internal/uploader/testdata/TestUpload/golden/no_reports b/internal/uploader/testdata/TestUpload/golden/no_reports new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/no_reports @@ -0,0 +1 @@ +{} diff --git a/internal/uploader/testdata/TestUpload/golden/no_reports_with_dummy b/internal/uploader/testdata/TestUpload/golden/no_reports_with_dummy new file mode 100644 index 0000000..dcd1986 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/no_reports_with_dummy @@ -0,0 +1,33 @@ +-1.json: bad contents +dummy.json: |- + { + "OS": "something" + } +dummy/dummy.json: |- + { + "OS": "something" + } +dummy/empty-dummy.json: "" +empty-dummy.json: "" +source/local/-1.json: bad contents +source/local/dummy.json: |- + { + "OS": "something" + } +source/local/dummy/dummy.json: |- + { + "OS": "something" + } +source/local/dummy/empty-dummy.json: "" +source/local/empty-dummy.json: "" +source/uploaded/-1.json: bad contents +source/uploaded/dummy.json: |- + { + "OS": "something" + } +source/uploaded/dummy/dummy.json: |- + { + "OS": "something" + } +source/uploaded/dummy/empty-dummy.json: "" +source/uploaded/empty-dummy.json: "" diff --git a/internal/uploader/testdata/TestUpload/golden/offline_server b/internal/uploader/testdata/TestUpload/golden/offline_server new file mode 100644 index 0000000..ec0b658 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/offline_server @@ -0,0 +1 @@ +source/local/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/optout_payload_cm_false b/internal/uploader/testdata/TestUpload/golden/optout_payload_cm_false new file mode 100644 index 0000000..587b133 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/optout_payload_cm_false @@ -0,0 +1 @@ +source/uploaded/1.json: '{"OptOut":true}' diff --git a/internal/uploader/testdata/TestUpload/golden/optout_payload_cm_true b/internal/uploader/testdata/TestUpload/golden/optout_payload_cm_true new file mode 100644 index 0000000..587b133 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/optout_payload_cm_true @@ -0,0 +1 @@ +source/uploaded/1.json: '{"OptOut":true}' diff --git a/internal/uploader/testdata/TestUpload/golden/single_upload b/internal/uploader/testdata/TestUpload/golden/single_upload new file mode 100644 index 0000000..f436ae8 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/single_upload @@ -0,0 +1 @@ +source/uploaded/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/test_source/-1.json b/internal/uploader/testdata/test_source/-1.json new file mode 100644 index 0000000..89d4cda --- /dev/null +++ b/internal/uploader/testdata/test_source/-1.json @@ -0,0 +1 @@ +bad contents \ No newline at end of file diff --git a/internal/uploader/testdata/test_source/dummy.json b/internal/uploader/testdata/test_source/dummy.json new file mode 100644 index 0000000..6012d3c --- /dev/null +++ b/internal/uploader/testdata/test_source/dummy.json @@ -0,0 +1,3 @@ +{ + "OS": "something" +} \ No newline at end of file diff --git a/internal/uploader/testdata/test_source/dummy/dummy.json b/internal/uploader/testdata/test_source/dummy/dummy.json new file mode 100644 index 0000000..6012d3c --- /dev/null +++ b/internal/uploader/testdata/test_source/dummy/dummy.json @@ -0,0 +1,3 @@ +{ + "OS": "something" +} \ No newline at end of file diff --git a/internal/uploader/testdata/test_source/dummy/empty-dummy.json b/internal/uploader/testdata/test_source/dummy/empty-dummy.json new file mode 100644 index 0000000..e69de29 diff --git a/internal/uploader/testdata/test_source/empty-dummy.json b/internal/uploader/testdata/test_source/empty-dummy.json new file mode 100644 index 0000000..e69de29 diff --git a/internal/uploader/upload.go b/internal/uploader/upload.go new file mode 100644 index 0000000..c6e86bb --- /dev/null +++ b/internal/uploader/upload.go @@ -0,0 +1,160 @@ +package uploader + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "sync" + "time" + + "github.com/ubuntu/ubuntu-insights/internal/constants" + "github.com/ubuntu/ubuntu-insights/internal/report" +) + +var ( + // ErrReportNotMature is returned when a report is not mature enough to be uploaded. + ErrReportNotMature = errors.New("report is not mature enough to be uploaded") +) + +// Upload uploads the reports corresponding to the source to the configured server. +// +// It will only upload reports that are mature enough, and have not been uploaded before. +// If force is true, maturity and duplicate check will be skipped. +func (um Uploader) Upload(force bool) error { + slog.Debug("Uploading reports") + if err := um.makeDirs(); err != nil { + return err + } + + consent, err := um.consentM.HasConsent(um.source) + if err != nil { + return fmt.Errorf("upload failed to get consent state: %v", err) + } + + reports, err := report.GetAll(um.collectedDir) + if err != nil { + return fmt.Errorf("failed to get reports: %v", err) + } + + url, err := um.getURL() + if err != nil { + return fmt.Errorf("failed to get URL: %v", err) + } + + var wg sync.WaitGroup + for _, r := range reports { + wg.Add(1) + go func(r report.Report) { + defer wg.Done() + err := um.upload(r, url, consent, force) + if errors.Is(err, ErrReportNotMature) { + slog.Debug("Skipped report upload, not mature enough", "file", r.Name, "source", um.source) + } else if err != nil { + slog.Warn("Failed to upload report", "file", r.Name, "source", um.source, "error", err) + } + }(r) + } + wg.Wait() + + return nil +} + +// upload uploads an individual report to the server. It returns an error if the report is not mature enough to be uploaded, or if the upload fails. +// It also moves the report to the uploaded directory after a successful upload. +func (um Uploader) upload(r report.Report, url string, consent, force bool) error { + slog.Debug("Uploading report", "file", r.Name, "consent", consent, "force", force) + + if um.timeProvider.Now().Add(-um.minAge).Before(time.Unix(r.TimeStamp, 0)) && !force { + return ErrReportNotMature + } + + // Check for duplicate reports. + _, err := os.Stat(filepath.Join(um.uploadedDir, r.Name)) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to check if report has already been uploaded: %v", err) + } + if err == nil && !force { + return fmt.Errorf("report has already been uploaded") + } + + origData, err := r.ReadJSON() + if err != nil { + return fmt.Errorf("failed to read report: %v", err) + } + data := origData + if !consent { + data, err = json.Marshal(constants.OptOutJSON) + if err != nil { + return fmt.Errorf("failed to marshal opt-out JSON data: %v", err) + } + } + slog.Debug("Uploading", "payload", data) + + if um.dryRun { + slog.Debug("Dry run, skipping upload") + return nil + } + + // Move report first to avoid the situation where the report is sent, but not marked as sent. + r, err = r.MarkAsProcessed(um.uploadedDir, data) + if err != nil { + return fmt.Errorf("failed to mark report as processed: %v", err) + } + if err := send(url, data); err != nil { + if _, err := r.UndoProcessed(); err != nil { + return fmt.Errorf("failed to send data: %v, and failed to restore the original report: %v", err, err) + } + return fmt.Errorf("failed to send data: %v", err) + } + + return nil +} + +func (um Uploader) getURL() (string, error) { + u, err := url.Parse(um.baseServerURL) + if err != nil { + return "", fmt.Errorf("failed to parse base server URL %s: %v", um.baseServerURL, err) + } + u.Path = path.Join(u.Path, um.source) + return u.String(), nil +} + +func send(url string, data []byte) error { + slog.Debug("Sending data to server", "url", url, "data", data) + req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(data)) + if err != nil { + return fmt.Errorf("failed to create request: %v", err) + } + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{Timeout: time.Second * 10} + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("failed to send HTTP request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("server returned status code %d", resp.StatusCode) + } + + return nil +} + +// makeDirs creates the directories for the collected and uploaded reports if they don't already exist. +func (um Uploader) makeDirs() error { + if err := os.MkdirAll(um.collectedDir, 0750); err != nil { + return fmt.Errorf("failed to create collected directory: %v", err) + } + if err := os.MkdirAll(um.uploadedDir, 0750); err != nil { + return fmt.Errorf("failed to create uploaded directory: %v", err) + } + return nil +} diff --git a/internal/uploader/uploader.go b/internal/uploader/uploader.go index b1c189e..b2aa83f 100644 --- a/internal/uploader/uploader.go +++ b/internal/uploader/uploader.go @@ -1,3 +1,83 @@ // Package uploader implements the uploader component. // The uploader component is responsible for uploading reports to the Ubuntu Insights server. package uploader + +import ( + "fmt" + "log/slog" + "path/filepath" + "time" + + "github.com/ubuntu/ubuntu-insights/internal/constants" +) + +type timeProvider interface { + Now() time.Time +} + +type realTimeProvider struct{} + +func (realTimeProvider) Now() time.Time { + return time.Now() +} + +// Uploader is an abstraction of the uploader component. +type Uploader struct { + source string + consentM consentManager + minAge time.Duration + dryRun bool + + baseServerURL string + collectedDir string + uploadedDir string + timeProvider timeProvider +} + +type options struct { + // Private members exported for tests. + baseServerURL string + cachePath string + timeProvider timeProvider +} + +// Options represents an optional function to override Upload Manager default values. +type Options func(*options) + +type consentManager interface { + HasConsent(source string) (bool, error) +} + +// New returns a new UploaderManager. +func New(cm consentManager, source string, minAge uint, dryRun bool, args ...Options) (Uploader, error) { + slog.Debug("Creating new uploader manager", "source", source, "minAge", minAge, "dryRun", dryRun) + + if source == "" { + return Uploader{}, fmt.Errorf("source cannot be an empty string") + } + + if minAge > (1<<63-1)/uint(time.Second) { + return Uploader{}, fmt.Errorf("min age %d is too large, would overflow", minAge) + } + + opts := options{ + baseServerURL: constants.DefaultServerURL, + cachePath: constants.GetDefaultCachePath(), + timeProvider: realTimeProvider{}, + } + for _, opt := range args { + opt(&opts) + } + + return Uploader{ + source: source, + consentM: cm, + minAge: time.Duration(minAge) * time.Second, + dryRun: dryRun, + timeProvider: opts.timeProvider, + + baseServerURL: opts.baseServerURL, + collectedDir: filepath.Join(opts.cachePath, source, constants.LocalFolder), + uploadedDir: filepath.Join(opts.cachePath, source, constants.UploadedFolder), + }, nil +} diff --git a/internal/uploader/uploader_test.go b/internal/uploader/uploader_test.go new file mode 100644 index 0000000..45a5757 --- /dev/null +++ b/internal/uploader/uploader_test.go @@ -0,0 +1,228 @@ +package uploader_test + +import ( + "encoding/json" + "fmt" + "math" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/require" + "github.com/ubuntu/ubuntu-insights/internal/constants" + "github.com/ubuntu/ubuntu-insights/internal/fileutils" + "github.com/ubuntu/ubuntu-insights/internal/testutils" + "github.com/ubuntu/ubuntu-insights/internal/uploader" +) + +type reportType any + +var ( + normal reportType = struct{ Content string }{Content: "normal content"} + optOut = constants.OptOutJSON + badContent = `bad content` +) + +var ( + cTrue = testConsentChecker{consent: true} + cFalse = testConsentChecker{consent: false} + cErr = testConsentChecker{err: fmt.Errorf("consent error")} + cErrTrue = testConsentChecker{consent: true, err: fmt.Errorf("consent error")} +) + +func TestNew(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + consent testConsentChecker + source string + minAge uint + dryRun bool + + wantErr bool + }{ + "Valid": {consent: cTrue, source: "source", minAge: 5, dryRun: true}, + "Zero Min Age": {consent: cTrue, source: "source", minAge: 0}, + + "Empty Source": {consent: cTrue, source: "", wantErr: true}, + "Minage Overflow": {consent: cTrue, source: "source", minAge: math.MaxUint64, wantErr: true}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + _, err := uploader.New(tc.consent, tc.source, tc.minAge, tc.dryRun) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + }) + } +} + +func TestUpload(t *testing.T) { + t.Parallel() + + const ( + mockTime = 10 + defaultResponse = http.StatusOK + source = "source" + ) + + tests := map[string]struct { + lFiles, uFiles map[string]reportType + dummy bool + serverResponse int + serverOffline bool + url string + rmLocal bool + noPerms bool + + consent testConsentChecker + minAge uint + dryRun bool + force bool + + skipContentCheck bool + wantErr bool + }{ + "No Reports": {consent: cTrue}, + "No Reports with Dummy": {dummy: true, consent: cTrue}, + "Single Upload": {lFiles: map[string]reportType{"1.json": normal}, consent: cTrue}, + "Multi Upload": {lFiles: map[string]reportType{"1.json": normal, "5.json": normal}, consent: cTrue}, + "Min Age": {lFiles: map[string]reportType{"1.json": normal, "9.json": normal}, consent: cTrue, minAge: 5}, + "Future Timestamp": {lFiles: map[string]reportType{"1.json": normal, "11.json": normal}, consent: cTrue}, + "Duplicate Upload": {lFiles: map[string]reportType{"1.json": normal}, uFiles: map[string]reportType{"1.json": badContent}, consent: cTrue}, + "Bad Content": {lFiles: map[string]reportType{"1.json": badContent}, consent: cTrue}, + "No Directory": {lFiles: map[string]reportType{"1.json": normal}, consent: cTrue, rmLocal: true}, + + "Consent Manager Source Error": {lFiles: map[string]reportType{"1.json": normal}, consent: cErr, wantErr: true}, + "Consent Manager Source Error with True": {lFiles: map[string]reportType{"1.json": normal}, consent: cErrTrue, wantErr: true}, + "Consent Manager False": {lFiles: map[string]reportType{"1.json": normal}, consent: cFalse}, + + "Force CM False": {lFiles: map[string]reportType{"1.json": normal}, consent: cFalse, force: true}, + "Force Min Age": {lFiles: map[string]reportType{"1.json": normal, "9.json": normal}, consent: cTrue, minAge: 5, force: true}, + "Force Duplicate": {lFiles: map[string]reportType{"1.json": normal}, uFiles: map[string]reportType{"1.json": badContent}, consent: cTrue, force: true}, + + "OptOut Payload CM True": {lFiles: map[string]reportType{"1.json": optOut}, consent: cTrue}, + "OptOut Payload CM False": {lFiles: map[string]reportType{"1.json": optOut}, consent: cFalse}, + + "Dry run": {lFiles: map[string]reportType{"1.json": normal}, consent: cTrue, dryRun: true}, + + "Bad URL": {lFiles: map[string]reportType{"1.json": normal}, consent: cTrue, url: "http://a b.com/", wantErr: true}, + "Bad Response": {lFiles: map[string]reportType{"1.json": normal}, consent: cTrue, serverResponse: http.StatusForbidden}, + "Offline Server": {lFiles: map[string]reportType{"1.json": normal}, consent: cTrue, serverOffline: true}, + "No Permissions": {lFiles: map[string]reportType{"1.json": normal}, consent: cTrue, noPerms: true, wantErr: runtime.GOOS != "windows", skipContentCheck: true}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + dir := setupTmpDir(t, tc.lFiles, tc.uFiles, source, tc.dummy) + + if tc.serverResponse == 0 { + tc.serverResponse = defaultResponse + } + + if !tc.serverOffline { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tc.serverResponse) + })) + t.Cleanup(func() { ts.Close() }) + if tc.url == "" { + tc.url = ts.URL + } + } + + localDir := filepath.Join(dir, source, constants.LocalFolder) + if tc.rmLocal { + require.NoError(t, os.RemoveAll(localDir), "Setup: failed to remove local directory") + } else if tc.noPerms { + require.NoError(t, os.Chmod(localDir, 0), "Setup: failed to remove local directory") + t.Cleanup(func() { require.NoError(t, os.Chmod(localDir, 0750), "Cleanup: failed to restore permissions") }) //nolint:gosec //0750 is fine for folders + } + + mgr, err := uploader.New(tc.consent, source, tc.minAge, tc.dryRun, + uploader.WithBaseServerURL(tc.url), uploader.WithCachePath(dir), uploader.WithTimeProvider(uploader.MockTimeProvider{CurrentTime: mockTime})) + require.NoError(t, err, "Setup: failed to create new uploader manager") + + err = mgr.Upload(tc.force) + if tc.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + if tc.noPerms { + //nolint:gosec //0750 is fine for folders + require.NoError(t, os.Chmod(localDir, 0750), "Post: failed to restore permissions") + } + + if tc.skipContentCheck { + return + } + + got, err := testutils.GetDirContents(t, dir, 3) + require.NoError(t, err) + want := testutils.LoadWithUpdateFromGoldenYAML(t, got) + require.EqualValues(t, want, got) + }) + } +} + +func setupTmpDir(t *testing.T, localFiles, uploadedFiles map[string]reportType, source string, dummy bool) string { + t.Helper() + dir := t.TempDir() + + localDir := filepath.Join(dir, source, constants.LocalFolder) + uploadedDir := filepath.Join(dir, source, constants.UploadedFolder) + require.NoError(t, os.MkdirAll(localDir, 0750), "Setup: failed to create local directory") + require.NoError(t, os.MkdirAll(uploadedDir, 0750), "Setup: failed to create uploaded directory") + + if dummy { + copyDummyData(t, "testdata/test_source", dir, localDir, uploadedDir) + } + + writeFiles(t, localDir, localFiles) + writeFiles(t, uploadedDir, uploadedFiles) + + return dir +} + +func copyDummyData(t *testing.T, sourceDir, dir, localDir, uploadedDir string) { + t.Helper() + require.NoError(t, testutils.CopyDir(t, sourceDir, dir), "Setup: failed to copy dummy data to temporary directory") + require.NoError(t, testutils.CopyDir(t, sourceDir, localDir), "Setup: failed to copy dummy data to local") + require.NoError(t, testutils.CopyDir(t, sourceDir, uploadedDir), "Setup: failed to copy dummy data to uploaded") +} + +func writeFiles(t *testing.T, targetDir string, files map[string]reportType) { + t.Helper() + for file, content := range files { + var data []byte + var err error + + switch v := content.(type) { + case string: + data = []byte(v) + default: + data, err = json.Marshal(content) + require.NoError(t, err, "Setup: failed to marshal sample data") + } + require.NoError(t, fileutils.AtomicWrite(filepath.Join(targetDir, file), data), "Setup: failed to write file") + } +} + +type testConsentChecker struct { + consent bool + err error +} + +func (m testConsentChecker) HasConsent(source string) (bool, error) { + return m.consent, m.err +}