From e8b8d195dc0bf4f89ca9a4bef3ac2ac2ba929994 Mon Sep 17 00:00:00 2001 From: kat <28567881+hk21702@users.noreply.github.com> Date: Mon, 13 Jan 2025 14:35:09 -0500 Subject: [PATCH 01/26] Remove flock, change consent write to be atomic, measure file count in set test --- go.mod | 6 +- go.sum | 8 +- internal/consent/consent.go | 45 ++--- internal/consent/consent_test.go | 167 +++++------------- .../golden/get_consent_states | 25 --- ...ile,_no_locks => invalid_file_global_file} | 0 ...le,_no_locks => invalid_value_global_file} | 0 ...o_global_file,_no_locks => no_global_file} | 0 ...file,_no_locks => valid_false_global_file} | 0 ...lobal_read_lock => valid_true_global_file} | 0 ..._file,_2_multiple_sources_(vtrue,_vfalse)} | 0 ...,_3_multiple_sources_(vtrue,_vfalse,_naf)} | 0 ...,_dual_source_write_lock,_global_read_lock | 29 --- .../valid_true_global_file,_global_write_lock | 29 --- ...lid_true_global_file,_invalid_file_source} | 0 ...id_true_global_file,_invalid_value_source} | 0 ...=> valid_true_global_file,_no_file_source} | 0 .../golden/valid_true_global_file,_no_locks | 29 --- .../valid_true_global_file,_source_read_lock | 29 --- ...l_file,_source_read_lock,_global_read_lock | 29 --- .../valid_true_global_file,_source_write_lock | 29 --- ..._file,_source_write_lock,_global_read_lock | 29 --- ...alid_true_global_file,_valid_false_source} | 0 ...valid_true_global_file,_valid_true_source} | 0 .../new_file,_no_locks,_write_global_false | 29 --- .../new_file,_no_locks,_write_global_true | 29 --- .../new_file,_no_locks,_write_source_false | 33 ---- .../new_file,_no_locks,_write_source_true | 33 ---- .../golden/new_file,_write_global_false | 31 ++++ .../golden/new_file,_write_global_true | 31 ++++ .../golden/new_file,_write_source_false | 35 ++++ .../golden/new_file,_write_source_true | 35 ++++ ...te_file,_no_locks,_write_diff_global_false | 29 --- ...ite_file,_no_locks,_write_diff_global_true | 29 --- ...te_file,_no_locks,_write_diff_source_false | 29 --- ...ite_file,_no_locks,_write_diff_source_true | 29 --- ...erwrite_file,_no_locks,_write_global_false | 29 --- ...verwrite_file,_no_locks,_write_global_true | 29 --- ...erwrite_file,_no_locks,_write_source_false | 29 --- ...verwrite_file,_no_locks,_write_source_true | 29 --- .../overwrite_file,_write_diff_global_false | 31 ++++ .../overwrite_file,_write_diff_global_true | 31 ++++ .../overwrite_file,_write_diff_source_false | 31 ++++ .../overwrite_file,_write_diff_source_true | 31 ++++ .../golden/overwrite_file,_write_global_false | 31 ++++ .../golden/overwrite_file,_write_global_true | 31 ++++ .../golden/overwrite_file,_write_source_false | 31 ++++ .../golden/overwrite_file,_write_source_true | 31 ++++ 48 files changed, 450 insertions(+), 740 deletions(-) delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/get_consent_states rename internal/consent/testdata/TestGetConsentStates/golden/{invalid_file_global_file,_no_locks => invalid_file_global_file} (100%) rename internal/consent/testdata/TestGetConsentStates/golden/{invalid_value_global_file,_no_locks => invalid_value_global_file} (100%) rename internal/consent/testdata/TestGetConsentStates/golden/{no_global_file,_no_locks => no_global_file} (100%) rename internal/consent/testdata/TestGetConsentStates/golden/{valid_false_global_file,_no_locks => valid_false_global_file} (100%) rename internal/consent/testdata/TestGetConsentStates/golden/{valid_true_global_file,_global_read_lock => valid_true_global_file} (100%) rename internal/consent/testdata/TestGetConsentStates/golden/{valid_true_global_file,_no_locks,_2_multiple_sources_(vtrue,_vfalse),_no_locks => valid_true_global_file,_2_multiple_sources_(vtrue,_vfalse)} (100%) rename internal/consent/testdata/TestGetConsentStates/golden/{valid_true_global_file,_no_locks,_3_multiple_sources_(vtrue,_vfalse,_naf),_no_locks => valid_true_global_file,_3_multiple_sources_(vtrue,_vfalse,_naf)} (100%) delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_dual_source_write_lock,_global_read_lock delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_global_write_lock rename internal/consent/testdata/TestGetConsentStates/golden/{valid_true_global_file,_no_locks,_invalid_file_source,_no_locks => valid_true_global_file,_invalid_file_source} (100%) rename internal/consent/testdata/TestGetConsentStates/golden/{valid_true_global_file,_no_locks,_invalid_value_source,_no_locks => valid_true_global_file,_invalid_value_source} (100%) rename internal/consent/testdata/TestGetConsentStates/golden/{valid_true_global_file,_no_locks,_no_file_source,_no_locks => valid_true_global_file,_no_file_source} (100%) delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_locks delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_source_read_lock delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_source_read_lock,_global_read_lock delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_source_write_lock delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_source_write_lock,_global_read_lock rename internal/consent/testdata/TestGetConsentStates/golden/{valid_true_global_file,_no_locks,_valid_false_source,_no_locks => valid_true_global_file,_valid_false_source} (100%) rename internal/consent/testdata/TestGetConsentStates/golden/{valid_true_global_file,_no_locks,_valid_true_source,_no_locks => valid_true_global_file,_valid_true_source} (100%) delete mode 100644 internal/consent/testdata/TestSetConsentStates/golden/new_file,_no_locks,_write_global_false delete mode 100644 internal/consent/testdata/TestSetConsentStates/golden/new_file,_no_locks,_write_global_true delete mode 100644 internal/consent/testdata/TestSetConsentStates/golden/new_file,_no_locks,_write_source_false delete mode 100644 internal/consent/testdata/TestSetConsentStates/golden/new_file,_no_locks,_write_source_true create mode 100644 internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_false create mode 100644 internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_true create mode 100644 internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_false create mode 100644 internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_true delete mode 100644 internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_diff_global_false delete mode 100644 internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_diff_global_true delete mode 100644 internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_diff_source_false delete mode 100644 internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_diff_source_true delete mode 100644 internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_global_false delete mode 100644 internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_global_true delete mode 100644 internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_source_false delete mode 100644 internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_source_true create mode 100644 internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_false create mode 100644 internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_true create mode 100644 internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_false create mode 100644 internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_true create mode 100644 internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_false create mode 100644 internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_true create mode 100644 internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_false create mode 100644 internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_true diff --git a/go.mod b/go.mod index 581939f..954ae7b 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.23.4 require ( github.com/BurntSushi/toml v1.4.0 - github.com/gofrs/flock v0.12.1 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.10.0 @@ -14,8 +13,7 @@ require ( require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/kr/text v0.2.0 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/rogpeppe/go-internal v1.9.0 // indirect - golang.org/x/sys v0.22.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index bc97dc5..11637b2 100644 --- a/go.sum +++ b/go.sum @@ -4,14 +4,16 @@ github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E= -github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= @@ -23,8 +25,6 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/internal/consent/consent.go b/internal/consent/consent.go index 7bdecfc..fcf129e 100644 --- a/internal/consent/consent.go +++ b/internal/consent/consent.go @@ -10,7 +10,6 @@ import ( "strings" "github.com/BurntSushi/toml" - "github.com/gofrs/flock" "github.com/ubuntu/ubuntu-insights/internal/constants" ) @@ -160,42 +159,34 @@ func readConsentFile(filePath string) (*consentFile, error) { if _, err := os.Stat(filePath); os.IsNotExist(err) { return &consent, nil } - - lock := flock.New(filePath + ".lock") - lockAcquired, err := lock.TryRLock() - if err != nil { - return &consent, err - } - if !lockAcquired { - return &consent, fmt.Errorf("could not acquire lock on %s", filePath) - } - defer lock.Unlock() - - _, err = toml.DecodeFile(filePath, &consent) + _, err := toml.DecodeFile(filePath, &consent) return &consent, err } -// writeConsentFile writes the given consent file to the given path, replacing it if it already exists. -func writeConsentFile(filePath string, consent *consentFile) error { - lock := flock.New(filePath + ".lock") - lockAcquired, err := lock.TryLock() +// writeConsentFile writes the given consent file to the given path atomically, replacing it if it already exists. +// Not atomic in Windows. +func writeConsentFile(filePath string, consent *consentFile) (err error) { + dir := filepath.Dir(filePath) + tempFile, err := os.CreateTemp(dir, "consent-*.tmp") if err != nil { - return err + return fmt.Errorf("could not create temporary file: %w", err) } - if !lockAcquired { - return fmt.Errorf("could not acquire lock on %s", filePath) + defer func() { + tempFile.Close() + os.Remove(tempFile.Name()) + }() + + if err := toml.NewEncoder(tempFile).Encode(consent); err != nil { + return fmt.Errorf("could not encode consent file: %w", err) } - defer lock.Unlock() - file, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) - if err != nil { - return err + if err := tempFile.Close(); err != nil { + return fmt.Errorf("could not close temporary file: %w", err) } - defer file.Close() - if err := toml.NewEncoder(file).Encode(consent); err != nil { - return err + if err := os.Rename(tempFile.Name(), filePath); err != nil { + return fmt.Errorf("could not rename temporary file: %w", err) } return nil diff --git a/internal/consent/consent_test.go b/internal/consent/consent_test.go index 5bbc171..434b23c 100644 --- a/internal/consent/consent_test.go +++ b/internal/consent/consent_test.go @@ -7,71 +7,49 @@ import ( "path/filepath" "testing" - "github.com/gofrs/flock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/ubuntu/ubuntu-insights/internal/consent" "github.com/ubuntu/ubuntu-insights/internal/testutils" ) -type lockType uint - -const ( - noLock lockType = iota - readLock - writeLock -) - // consentDir is a struct that holds a test's temporary directory and its locks. // It should be cleaned up after the test is done. type consentDir struct { - dir string - sourceLocks map[string]*flock.Flock - globalLock *flock.Flock + dir string } func TestGetConsentStates(t *testing.T) { t.Parallel() tests := map[string]struct { - sources []string - lockSources map[string]lockType - globalFile string - lockGlobal lockType + sources []string + globalFile string wantErr bool }{ - "No Global File, No Locks": {}, + "No Global File": {}, // Global File Tests - "Valid True Global File, No Locks": {globalFile: "valid_true-consent.toml"}, - "Valid False Global File, No Locks": {globalFile: "valid_false-consent.toml"}, - "Valid True Global File, Global Read Lock": {globalFile: "valid_true-consent.toml", lockGlobal: readLock}, - "Valid True Global File, Global Write Lock": {globalFile: "valid_true-consent.toml", lockGlobal: writeLock}, - "Invalid Value Global File, No Locks": {globalFile: "invalid_value-consent.toml"}, - "Invalid File Global File, No Locks": {globalFile: "invalid_file-consent.toml"}, - - // Lock sources - "Valid True Global File, Source Read Lock": {globalFile: "valid_true-consent.toml", lockSources: map[string]lockType{"valid_true": readLock}}, - "Valid True Global File, Source Write Lock": {globalFile: "valid_true-consent.toml", lockSources: map[string]lockType{"valid_true": writeLock}}, - "Valid True Global File, Source Read Lock, Global Read Lock": {globalFile: "valid_true-consent.toml", lockSources: map[string]lockType{"valid_true": readLock}, lockGlobal: readLock}, - "Valid True Global File, Source Write Lock, Global Read Lock": {globalFile: "valid_true-consent.toml", lockSources: map[string]lockType{"valid_true": writeLock}, lockGlobal: readLock}, - "Valid True Global File, Dual Source Write Lock, Global Read Lock": {globalFile: "valid_true-consent.toml", lockSources: map[string]lockType{"valid_true": writeLock, "invalid_value": writeLock}, lockGlobal: readLock}, + "Valid True Global File": {globalFile: "valid_true-consent.toml"}, + "Valid False Global File": {globalFile: "valid_false-consent.toml"}, + "Invalid Value Global File": {globalFile: "invalid_value-consent.toml"}, + "Invalid File Global File": {globalFile: "invalid_file-consent.toml"}, // Source Specific Tests - "Valid True Global File, No Locks, Valid True Source, No Locks": {globalFile: "valid_true-consent.toml", sources: []string{"valid_true"}}, - "Valid True Global File, No Locks, Valid False Source, No Locks": {globalFile: "valid_true-consent.toml", sources: []string{"valid_false"}}, - "Valid True Global File, No Locks, Invalid Value Source, No Locks": {globalFile: "valid_true-consent.toml", sources: []string{"invalid_value"}}, - "Valid True Global File, No Locks, Invalid File Source, No Locks": {globalFile: "valid_true-consent.toml", sources: []string{"invalid_file"}}, - "Valid True Global File, No Locks, No File Source, No Locks": {globalFile: "valid_true-consent.toml", sources: []string{"not_a_file"}}, - "Valid True Global File, No Locks, 2 Multiple Sources (VTrue, VFalse), No Locks": {globalFile: "valid_true-consent.toml", sources: []string{"valid_true", "valid_false"}}, - "Valid True Global File, No Locks, 3 Multiple Sources (VTrue, VFalse, NAF), No Locks": {globalFile: "valid_true-consent.toml", sources: []string{"valid_true", "valid_false", "not_a_file"}}, + "Valid True Global File, Valid True Source": {globalFile: "valid_true-consent.toml", sources: []string{"valid_true"}}, + "Valid True Global File, Valid False Source": {globalFile: "valid_true-consent.toml", sources: []string{"valid_false"}}, + "Valid True Global File, Invalid Value Source": {globalFile: "valid_true-consent.toml", sources: []string{"invalid_value"}}, + "Valid True Global File, Invalid File Source": {globalFile: "valid_true-consent.toml", sources: []string{"invalid_file"}}, + "Valid True Global File, No File Source": {globalFile: "valid_true-consent.toml", sources: []string{"not_a_file"}}, + "Valid True Global File, 2 Multiple Sources (VTrue, VFalse)": {globalFile: "valid_true-consent.toml", sources: []string{"valid_true", "valid_false"}}, + "Valid True Global File, 3 Multiple Sources (VTrue, VFalse, NAF)": {globalFile: "valid_true-consent.toml", sources: []string{"valid_true", "valid_false", "not_a_file"}}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { t.Parallel() - cdir, err := setupTmpConsentFiles(t, tc.lockSources, tc.globalFile, tc.lockGlobal) + cdir, err := setupTmpConsentFiles(t, tc.globalFile) require.NoError(t, err, "failed to setup temporary consent files") defer cdir.cleanup(t) cm := consent.New(cdir.dir) @@ -95,50 +73,41 @@ func TestSetConsentStates(t *testing.T) { tests := map[string]struct { sources []string consentStates map[string]bool - lockSources map[string]lockType globalFile string - lockGlobal lockType writeSource string writeState bool wantErr bool }{ - // New File Tests No Locks - "New File, No Locks, Write Global False": {}, - "New File, No Locks, Write Global True": {writeState: true}, - "New File, No Locks, Write Source True": {writeSource: "new_true", writeState: true}, - "New File, No Locks, Write Source False": {writeSource: "new_false"}, - - // Overwrite File, No Locks, Different State - "Overwrite File, No Locks, Write Diff Global False": {globalFile: "valid_true-consent.toml", writeState: false}, - "Overwrite File, No Locks, Write Diff Global True": {globalFile: "valid_false-consent.toml", writeState: true}, - "Overwrite File, No Locks, Write Diff Source True": {globalFile: "valid_true-consent.toml", writeSource: "valid_false", writeState: true}, - "Overwrite File, No Locks, Write Diff Source False": {globalFile: "valid_true-consent.toml", writeSource: "valid_true", writeState: false}, - - // Overwrite File, No Locks, Same State - "Overwrite File, No Locks, Write Global True": {globalFile: "valid_true-consent.toml", writeState: true}, - "Overwrite File, No Locks, Write Global False": {globalFile: "valid_false-consent.toml", writeState: false}, - "Overwrite File, No Locks, Write Source True": {globalFile: "valid_true-consent.toml", writeSource: "valid_true", writeState: true}, - "Overwrite File, No Locks, Write Source False": {globalFile: "valid_false-consent.toml", writeSource: "valid_false"}, - - // Overwrite File, Read Locks, Different State - "Overwrite File, Source Read Lock, Write Global False": {globalFile: "valid_true-consent.toml", lockGlobal: readLock, writeState: false, wantErr: true}, - "Overwrite File, Source Read Lock, Write Global True": {globalFile: "valid_false-consent.toml", lockGlobal: readLock, writeState: true, wantErr: true}, - "Overwrite File, Source Read Lock, Write Source True": {globalFile: "valid_true-consent.toml", lockSources: map[string]lockType{"valid_false": readLock}, writeSource: "valid_false", writeState: true, wantErr: true}, - "Overwrite File, Source Read Lock, Write Source False": {globalFile: "valid_false-consent.toml", lockSources: map[string]lockType{"valid_true": readLock}, writeSource: "valid_true", writeState: false, wantErr: true}, - - // Overwrite File, Write Locks, Different State - "Overwrite File, Source Write Lock, Write Global False": {globalFile: "valid_true-consent.toml", lockGlobal: writeLock, writeState: false, wantErr: true}, - "Overwrite File, Source Write Lock, Write Global True": {globalFile: "valid_false-consent.toml", lockGlobal: writeLock, writeState: true, wantErr: true}, - "Overwrite File, Source Write Lock, Write Source True": {globalFile: "valid_true-consent.toml", lockSources: map[string]lockType{"valid_false": writeLock}, writeSource: "valid_false", writeState: true, wantErr: true}, - "Overwrite File, Source Write Lock, Write Source False": {globalFile: "valid_false-consent.toml", lockSources: map[string]lockType{"valid_true": writeLock}, writeSource: "valid_true", writeState: false, wantErr: true}, + // New File Tests + "New File, Write Global False": {}, + "New File, Write Global True": {writeState: true}, + "New File, Write Source True": {writeSource: "new_true", writeState: true}, + "New File, Write Source False": {writeSource: "new_false"}, + + // Overwrite File, Different State + "Overwrite File, Write Diff Global False": {globalFile: "valid_true-consent.toml", writeState: false}, + "Overwrite File, Write Diff Global True": {globalFile: "valid_false-consent.toml", writeState: true}, + "Overwrite File, Write Diff Source True": {globalFile: "valid_true-consent.toml", writeSource: "valid_false", writeState: true}, + "Overwrite File, Write Diff Source False": {globalFile: "valid_true-consent.toml", writeSource: "valid_true", writeState: false}, + + // Overwrite File, Same State + "Overwrite File, Write Global True": {globalFile: "valid_true-consent.toml", writeState: true}, + "Overwrite File, Write Global False": {globalFile: "valid_false-consent.toml", writeState: false}, + "Overwrite File, Write Source True": {globalFile: "valid_true-consent.toml", writeSource: "valid_true", writeState: true}, + "Overwrite File, Write Source False": {globalFile: "valid_false-consent.toml", writeSource: "valid_false"}, + } + + type goldenFile struct { + States *consent.States + FileCount uint } for name, tc := range tests { t.Run(name, func(t *testing.T) { t.Parallel() - cdir, err := setupTmpConsentFiles(t, tc.lockSources, tc.globalFile, tc.lockGlobal) + cdir, err := setupTmpConsentFiles(t, tc.globalFile) require.NoError(t, err, "failed to setup temporary consent files") defer cdir.cleanup(t) cm := consent.New(cdir.dir) @@ -150,9 +119,13 @@ func TestSetConsentStates(t *testing.T) { } require.NoError(t, err, "got an unexpected error") - got, err := cm.GetConsentStates(tc.sources) + states, err := cm.GetConsentStates(tc.sources) require.NoError(t, err, "got an unexpected error while getting consent states") + d, err := os.ReadDir(cdir.dir) + require.NoError(t, err, "failed to read temporary directory") + got := goldenFile{States: states, FileCount: uint(len(d))} + want := testutils.LoadWithUpdateFromGoldenYAML(t, got) require.Equal(t, want, got, "GetConsentStates should return expected consent states") }) @@ -162,14 +135,6 @@ func TestSetConsentStates(t *testing.T) { // cleanup unlocks all the locks and removes the temporary directory including its contents. func (cdir consentDir) cleanup(t *testing.T) { t.Helper() - for i := range cdir.sourceLocks { - assert.NoError(t, cdir.sourceLocks[i].Unlock(), "failed to unlock source lock") - } - - if cdir.globalLock != nil { - assert.NoError(t, cdir.globalLock.Unlock(), "failed to unlock global lock") - } - assert.NoError(t, os.RemoveAll(cdir.dir), "failed to remove temporary directory") } @@ -207,9 +172,9 @@ func copyDir(srcDir, dstDir string) error { }) } -func setupTmpConsentFiles(t *testing.T, lockSources map[string]lockType, globalFile string, lockGlobal lockType) (*consentDir, error) { +func setupTmpConsentFiles(t *testing.T, globalFile string) (*consentDir, error) { t.Helper() - cdir := consentDir{sourceLocks: make(map[string]*flock.Flock)} + cdir := consentDir{} // Setup temporary directory var err error @@ -229,45 +194,5 @@ func setupTmpConsentFiles(t *testing.T, lockSources map[string]lockType, globalF } } - // Setup lock files - for source, lock := range lockSources { - if lock == noLock { - continue - } - - lockPath := filepath.Join(cdir.dir, source+"-consent.toml.lock") - cdir.sourceLocks[source] = flock.New(lockPath) - switch lock { - case readLock: - err = cdir.sourceLocks[source].RLock() - case writeLock: - err = cdir.sourceLocks[source].Lock() - } - - if err != nil { - return &cdir, fmt.Errorf("failed to acquire lock on consent file for source %s: %v", source, err) - } - } - - // Setup global lock file - if lockGlobal != noLock { - if globalFile == "" { - return &cdir, fmt.Errorf("global file must be provided if global lock is requested") - } - - lockPath := filepath.Join(cdir.dir, "consent.toml.lock") - cdir.globalLock = flock.New(lockPath) - switch lockGlobal { - case readLock: - err = cdir.globalLock.RLock() - case writeLock: - err = cdir.globalLock.Lock() - } - - if err != nil { - return &cdir, fmt.Errorf("failed to acquire lock on global consent file: %v", err) - } - } - return &cdir, nil } diff --git a/internal/consent/testdata/TestGetConsentStates/golden/get_consent_states b/internal/consent/testdata/TestGetConsentStates/golden/get_consent_states deleted file mode 100644 index 0eb4214..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/get_consent_states +++ /dev/null @@ -1,25 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: "" - state: false - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/invalid_file_global_file,_no_locks b/internal/consent/testdata/TestGetConsentStates/golden/invalid_file_global_file similarity index 100% rename from internal/consent/testdata/TestGetConsentStates/golden/invalid_file_global_file,_no_locks rename to internal/consent/testdata/TestGetConsentStates/golden/invalid_file_global_file diff --git a/internal/consent/testdata/TestGetConsentStates/golden/invalid_value_global_file,_no_locks b/internal/consent/testdata/TestGetConsentStates/golden/invalid_value_global_file similarity index 100% rename from internal/consent/testdata/TestGetConsentStates/golden/invalid_value_global_file,_no_locks rename to internal/consent/testdata/TestGetConsentStates/golden/invalid_value_global_file diff --git a/internal/consent/testdata/TestGetConsentStates/golden/no_global_file,_no_locks b/internal/consent/testdata/TestGetConsentStates/golden/no_global_file similarity index 100% rename from internal/consent/testdata/TestGetConsentStates/golden/no_global_file,_no_locks rename to internal/consent/testdata/TestGetConsentStates/golden/no_global_file diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_false_global_file,_no_locks b/internal/consent/testdata/TestGetConsentStates/golden/valid_false_global_file similarity index 100% rename from internal/consent/testdata/TestGetConsentStates/golden/valid_false_global_file,_no_locks rename to internal/consent/testdata/TestGetConsentStates/golden/valid_false_global_file diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_global_read_lock b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file similarity index 100% rename from internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_global_read_lock rename to internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_locks,_2_multiple_sources_(vtrue,_vfalse),_no_locks b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_2_multiple_sources_(vtrue,_vfalse) similarity index 100% rename from internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_locks,_2_multiple_sources_(vtrue,_vfalse),_no_locks rename to internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_2_multiple_sources_(vtrue,_vfalse) diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_locks,_3_multiple_sources_(vtrue,_vfalse,_naf),_no_locks b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_3_multiple_sources_(vtrue,_vfalse,_naf) similarity index 100% rename from internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_locks,_3_multiple_sources_(vtrue,_vfalse,_naf),_no_locks rename to internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_3_multiple_sources_(vtrue,_vfalse,_naf) diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_dual_source_write_lock,_global_read_lock b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_dual_source_write_lock,_global_read_lock deleted file mode 100644 index 91d8dee..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_dual_source_write_lock,_global_read_lock +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: false - readerr: true -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_global_write_lock b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_global_write_lock deleted file mode 100644 index 2ae9869..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_global_write_lock +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: false - readerr: true diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_locks,_invalid_file_source,_no_locks b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_invalid_file_source similarity index 100% rename from internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_locks,_invalid_file_source,_no_locks rename to internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_invalid_file_source diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_locks,_invalid_value_source,_no_locks b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_invalid_value_source similarity index 100% rename from internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_locks,_invalid_value_source,_no_locks rename to internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_invalid_value_source diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_locks,_no_file_source,_no_locks b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_file_source similarity index 100% rename from internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_locks,_no_file_source,_no_locks rename to internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_file_source diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_locks b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_locks deleted file mode 100644 index 7ff6725..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_locks +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_source_read_lock b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_source_read_lock deleted file mode 100644 index 7ff6725..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_source_read_lock +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_source_read_lock,_global_read_lock b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_source_read_lock,_global_read_lock deleted file mode 100644 index 7ff6725..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_source_read_lock,_global_read_lock +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_source_write_lock b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_source_write_lock deleted file mode 100644 index 91d8dee..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_source_write_lock +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: false - readerr: true -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_source_write_lock,_global_read_lock b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_source_write_lock,_global_read_lock deleted file mode 100644 index 91d8dee..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_source_write_lock,_global_read_lock +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: false - readerr: true -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_locks,_valid_false_source,_no_locks b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_valid_false_source similarity index 100% rename from internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_locks,_valid_false_source,_no_locks rename to internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_valid_false_source diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_locks,_valid_true_source,_no_locks b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_valid_true_source similarity index 100% rename from internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_locks,_valid_true_source,_no_locks rename to internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_valid_true_source diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_no_locks,_write_global_false b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_no_locks,_write_global_false deleted file mode 100644 index 8a2bbfd..0000000 --- a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_no_locks,_write_global_false +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: false - readerr: false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_no_locks,_write_global_true b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_no_locks,_write_global_true deleted file mode 100644 index 7ff6725..0000000 --- a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_no_locks,_write_global_true +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_no_locks,_write_source_false b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_no_locks,_write_source_false deleted file mode 100644 index c080dff..0000000 --- a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_no_locks,_write_source_false +++ /dev/null @@ -1,33 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - new_false: - source: new_false - state: false - readerr: false - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: "" - state: false - readerr: false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_no_locks,_write_source_true b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_no_locks,_write_source_true deleted file mode 100644 index d9168bc..0000000 --- a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_no_locks,_write_source_true +++ /dev/null @@ -1,33 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - new_true: - source: new_true - state: true - readerr: false - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: "" - state: false - readerr: false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_false b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_false new file mode 100644 index 0000000..28a95b6 --- /dev/null +++ b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_false @@ -0,0 +1,31 @@ +states: + sourcestates: + empty: + source: empty + state: false + readerr: false + extra_key: + source: extra_key + state: false + readerr: false + invalid_file: + source: invalid_file + state: false + readerr: true + invalid_value: + source: invalid_value + state: false + readerr: true + valid_false: + source: valid_false + state: false + readerr: false + valid_true: + source: valid_true + state: true + readerr: false + globalstate: + source: global + state: false + readerr: false +filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_true b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_true new file mode 100644 index 0000000..bbe6efd --- /dev/null +++ b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_true @@ -0,0 +1,31 @@ +states: + sourcestates: + empty: + source: empty + state: false + readerr: false + extra_key: + source: extra_key + state: false + readerr: false + invalid_file: + source: invalid_file + state: false + readerr: true + invalid_value: + source: invalid_value + state: false + readerr: true + valid_false: + source: valid_false + state: false + readerr: false + valid_true: + source: valid_true + state: true + readerr: false + globalstate: + source: global + state: true + readerr: false +filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_false b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_false new file mode 100644 index 0000000..d50e411 --- /dev/null +++ b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_false @@ -0,0 +1,35 @@ +states: + sourcestates: + empty: + source: empty + state: false + readerr: false + extra_key: + source: extra_key + state: false + readerr: false + invalid_file: + source: invalid_file + state: false + readerr: true + invalid_value: + source: invalid_value + state: false + readerr: true + new_false: + source: new_false + state: false + readerr: false + valid_false: + source: valid_false + state: false + readerr: false + valid_true: + source: valid_true + state: true + readerr: false + globalstate: + source: "" + state: false + readerr: false +filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_true b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_true new file mode 100644 index 0000000..0491904 --- /dev/null +++ b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_true @@ -0,0 +1,35 @@ +states: + sourcestates: + empty: + source: empty + state: false + readerr: false + extra_key: + source: extra_key + state: false + readerr: false + invalid_file: + source: invalid_file + state: false + readerr: true + invalid_value: + source: invalid_value + state: false + readerr: true + new_true: + source: new_true + state: true + readerr: false + valid_false: + source: valid_false + state: false + readerr: false + valid_true: + source: valid_true + state: true + readerr: false + globalstate: + source: "" + state: false + readerr: false +filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_diff_global_false b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_diff_global_false deleted file mode 100644 index 8a2bbfd..0000000 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_diff_global_false +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: false - readerr: false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_diff_global_true b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_diff_global_true deleted file mode 100644 index 7ff6725..0000000 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_diff_global_true +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_diff_source_false b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_diff_source_false deleted file mode 100644 index 0d67d24..0000000 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_diff_source_false +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: false - readerr: false -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_diff_source_true b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_diff_source_true deleted file mode 100644 index 05ab2c5..0000000 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_diff_source_true +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: true - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_global_false b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_global_false deleted file mode 100644 index 8a2bbfd..0000000 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_global_false +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: false - readerr: false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_global_true b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_global_true deleted file mode 100644 index 7ff6725..0000000 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_global_true +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_source_false b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_source_false deleted file mode 100644 index 8a2bbfd..0000000 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_source_false +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: false - readerr: false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_source_true b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_source_true deleted file mode 100644 index 7ff6725..0000000 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_no_locks,_write_source_true +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_false b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_false new file mode 100644 index 0000000..28a95b6 --- /dev/null +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_false @@ -0,0 +1,31 @@ +states: + sourcestates: + empty: + source: empty + state: false + readerr: false + extra_key: + source: extra_key + state: false + readerr: false + invalid_file: + source: invalid_file + state: false + readerr: true + invalid_value: + source: invalid_value + state: false + readerr: true + valid_false: + source: valid_false + state: false + readerr: false + valid_true: + source: valid_true + state: true + readerr: false + globalstate: + source: global + state: false + readerr: false +filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_true b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_true new file mode 100644 index 0000000..bbe6efd --- /dev/null +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_true @@ -0,0 +1,31 @@ +states: + sourcestates: + empty: + source: empty + state: false + readerr: false + extra_key: + source: extra_key + state: false + readerr: false + invalid_file: + source: invalid_file + state: false + readerr: true + invalid_value: + source: invalid_value + state: false + readerr: true + valid_false: + source: valid_false + state: false + readerr: false + valid_true: + source: valid_true + state: true + readerr: false + globalstate: + source: global + state: true + readerr: false +filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_false b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_false new file mode 100644 index 0000000..e97680a --- /dev/null +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_false @@ -0,0 +1,31 @@ +states: + sourcestates: + empty: + source: empty + state: false + readerr: false + extra_key: + source: extra_key + state: false + readerr: false + invalid_file: + source: invalid_file + state: false + readerr: true + invalid_value: + source: invalid_value + state: false + readerr: true + valid_false: + source: valid_false + state: false + readerr: false + valid_true: + source: valid_true + state: false + readerr: false + globalstate: + source: global + state: true + readerr: false +filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_true b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_true new file mode 100644 index 0000000..d035451 --- /dev/null +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_true @@ -0,0 +1,31 @@ +states: + sourcestates: + empty: + source: empty + state: false + readerr: false + extra_key: + source: extra_key + state: false + readerr: false + invalid_file: + source: invalid_file + state: false + readerr: true + invalid_value: + source: invalid_value + state: false + readerr: true + valid_false: + source: valid_false + state: true + readerr: false + valid_true: + source: valid_true + state: true + readerr: false + globalstate: + source: global + state: true + readerr: false +filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_false b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_false new file mode 100644 index 0000000..28a95b6 --- /dev/null +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_false @@ -0,0 +1,31 @@ +states: + sourcestates: + empty: + source: empty + state: false + readerr: false + extra_key: + source: extra_key + state: false + readerr: false + invalid_file: + source: invalid_file + state: false + readerr: true + invalid_value: + source: invalid_value + state: false + readerr: true + valid_false: + source: valid_false + state: false + readerr: false + valid_true: + source: valid_true + state: true + readerr: false + globalstate: + source: global + state: false + readerr: false +filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_true b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_true new file mode 100644 index 0000000..bbe6efd --- /dev/null +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_true @@ -0,0 +1,31 @@ +states: + sourcestates: + empty: + source: empty + state: false + readerr: false + extra_key: + source: extra_key + state: false + readerr: false + invalid_file: + source: invalid_file + state: false + readerr: true + invalid_value: + source: invalid_value + state: false + readerr: true + valid_false: + source: valid_false + state: false + readerr: false + valid_true: + source: valid_true + state: true + readerr: false + globalstate: + source: global + state: true + readerr: false +filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_false b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_false new file mode 100644 index 0000000..28a95b6 --- /dev/null +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_false @@ -0,0 +1,31 @@ +states: + sourcestates: + empty: + source: empty + state: false + readerr: false + extra_key: + source: extra_key + state: false + readerr: false + invalid_file: + source: invalid_file + state: false + readerr: true + invalid_value: + source: invalid_value + state: false + readerr: true + valid_false: + source: valid_false + state: false + readerr: false + valid_true: + source: valid_true + state: true + readerr: false + globalstate: + source: global + state: false + readerr: false +filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_true b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_true new file mode 100644 index 0000000..bbe6efd --- /dev/null +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_true @@ -0,0 +1,31 @@ +states: + sourcestates: + empty: + source: empty + state: false + readerr: false + extra_key: + source: extra_key + state: false + readerr: false + invalid_file: + source: invalid_file + state: false + readerr: true + invalid_value: + source: invalid_value + state: false + readerr: true + valid_false: + source: valid_false + state: false + readerr: false + valid_true: + source: valid_true + state: true + readerr: false + globalstate: + source: global + state: true + readerr: false +filecount: 9 From bd400fc514cfe47b4af8d6aa213133b2bd000150 Mon Sep 17 00:00:00 2001 From: kat <28567881+hk21702@users.noreply.github.com> Date: Mon, 13 Jan 2025 14:44:56 -0500 Subject: [PATCH 02/26] Re-add golangci --- .golangci.yaml | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 .golangci.yaml diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 0000000..62d954a --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,63 @@ +# This is for linting. To run it, please use: +# golangci-lint run ${MODULE}/... [--fix] + +linters: + # linters to run in addition to default ones + enable: + - copyloopvar + - dupl + - durationcheck + - errname + - errorlint + - forbidigo + - forcetypeassert + - gci + - godot + - gofmt + - gosec + - misspell + - nakedret + - nolintlint + - revive + - thelper + - tparallel + - unconvert + - unparam + - whitespace + +run: + timeout: 5m + +# Get all linter issues, even if duplicated +issues: + exclude-use-default: false + max-issues-per-linter: 0 + max-same-issues: 0 + fix: false # we don’t want this in CI + exclude: + # EXC0001 errcheck: most errors are in defer calls, which are safe to ignore and idiomatic Go (would be good to only ignore defer ones though) + - 'Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv|w\.Stop). is not checked' + # EXC0008 gosec: duplicated of errcheck + - (G104|G307) + # EXC0010 gosec: False positive is triggered by 'src, err := ioutil.ReadFile(filename)' + - Potential file inclusion via variable + # We don't wrap errors on purpose + - non-wrapping format verb for fmt.Errorf. Use `%w` to format errors + # We want named parameters even if unused, as they help better document the function + - unused-parameter + # Sometimes it is more readable it do a `if err:=a(); err != nil` tha simpy `return a()` + - if-return + +nolintlint: + require-explanation: true + require-specific: true + +linters-settings: + # Forbid the usage of deprecated ioutil and debug prints + forbidigo: + forbid: + - ioutil\. + - ^print.*$ + # Never have naked return ever + nakedret: + max-func-lines: 1 \ No newline at end of file From d4464e848d5c7b91734f6a5b4534dda4a3b6305b Mon Sep 17 00:00:00 2001 From: kat <28567881+hk21702@users.noreply.github.com> Date: Mon, 13 Jan 2025 16:11:25 -0500 Subject: [PATCH 03/26] fixup! Remove flock, change consent write to be atomic, measure file count in set test --- internal/consent/consent_test.go | 38 ++++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/internal/consent/consent_test.go b/internal/consent/consent_test.go index 434b23c..2d9ca34 100644 --- a/internal/consent/consent_test.go +++ b/internal/consent/consent_test.go @@ -49,10 +49,10 @@ func TestGetConsentStates(t *testing.T) { for name, tc := range tests { t.Run(name, func(t *testing.T) { t.Parallel() - cdir, err := setupTmpConsentFiles(t, tc.globalFile) + cDir, err := setupTmpConsentFiles(t, tc.globalFile) require.NoError(t, err, "failed to setup temporary consent files") - defer cdir.cleanup(t) - cm := consent.New(cdir.dir) + defer cDir.cleanup(t) + cm := consent.New(cDir.dir) got, err := cm.GetConsentStates(tc.sources) if tc.wantErr { @@ -96,7 +96,7 @@ func TestSetConsentStates(t *testing.T) { "Overwrite File, Write Global True": {globalFile: "valid_true-consent.toml", writeState: true}, "Overwrite File, Write Global False": {globalFile: "valid_false-consent.toml", writeState: false}, "Overwrite File, Write Source True": {globalFile: "valid_true-consent.toml", writeSource: "valid_true", writeState: true}, - "Overwrite File, Write Source False": {globalFile: "valid_false-consent.toml", writeSource: "valid_false"}, + "Overwrite File, Write Source False": {globalFile: "valid_false-consent.toml", writeSource: "valid_false", writeState: false}, } type goldenFile struct { @@ -107,10 +107,10 @@ func TestSetConsentStates(t *testing.T) { for name, tc := range tests { t.Run(name, func(t *testing.T) { t.Parallel() - cdir, err := setupTmpConsentFiles(t, tc.globalFile) + cDir, err := setupTmpConsentFiles(t, tc.globalFile) require.NoError(t, err, "failed to setup temporary consent files") - defer cdir.cleanup(t) - cm := consent.New(cdir.dir) + defer cDir.cleanup(t) + cm := consent.New(cDir.dir) err = cm.SetConsentState(tc.writeSource, tc.writeState) if tc.wantErr { @@ -122,7 +122,7 @@ func TestSetConsentStates(t *testing.T) { states, err := cm.GetConsentStates(tc.sources) require.NoError(t, err, "got an unexpected error while getting consent states") - d, err := os.ReadDir(cdir.dir) + d, err := os.ReadDir(cDir.dir) require.NoError(t, err, "failed to read temporary directory") got := goldenFile{States: states, FileCount: uint(len(d))} @@ -133,9 +133,9 @@ func TestSetConsentStates(t *testing.T) { } // cleanup unlocks all the locks and removes the temporary directory including its contents. -func (cdir consentDir) cleanup(t *testing.T) { +func (cDir consentDir) cleanup(t *testing.T) { t.Helper() - assert.NoError(t, os.RemoveAll(cdir.dir), "failed to remove temporary directory") + assert.NoError(t, os.RemoveAll(cDir.dir), "failed to remove temporary directory") } func copyFile(src, dst string) error { @@ -174,25 +174,25 @@ func copyDir(srcDir, dstDir string) error { func setupTmpConsentFiles(t *testing.T, globalFile string) (*consentDir, error) { t.Helper() - cdir := consentDir{} + cDir := consentDir{} // Setup temporary directory var err error - cdir.dir, err = os.MkdirTemp("", "consent-files") + cDir.dir, err = os.MkdirTemp("", "consent-files") if err != nil { - return &cdir, fmt.Errorf("failed to create temporary directory: %v", err) + return &cDir, fmt.Errorf("failed to create temporary directory: %v", err) } - if err = copyDir(filepath.Join("testdata", "consent_files"), cdir.dir); err != nil { - return &cdir, fmt.Errorf("failed to copy testdata directory to temporary directory: %v", err) + if err = copyDir(filepath.Join("testdata", "consent_files"), cDir.dir); err != nil { + return &cDir, fmt.Errorf("failed to copy testdata directory to temporary directory: %v", err) } - // Setup globalfile if provided + // Setup globalFile if provided if globalFile != "" { - if err = copyFile(filepath.Join(cdir.dir, globalFile), filepath.Join(cdir.dir, "consent.toml")); err != nil { - return &cdir, fmt.Errorf("failed to copy requested global consent file: %v", err) + if err = copyFile(filepath.Join(cDir.dir, globalFile), filepath.Join(cDir.dir, "consent.toml")); err != nil { + return &cDir, fmt.Errorf("failed to copy requested global consent file: %v", err) } } - return &cdir, nil + return &cDir, nil } From db013c3ddb21cc355f3376aa852b56e512dc1b1e Mon Sep 17 00:00:00 2001 From: kat <28567881+hk21702@users.noreply.github.com> Date: Wed, 15 Jan 2025 16:06:42 -0500 Subject: [PATCH 04/26] Fix testdata whitespace --- internal/consent/testdata/consent_files/extra_key-consent.toml | 2 +- .../consent/testdata/consent_files/invalid_file-consent.toml | 2 +- .../consent/testdata/consent_files/invalid_value-consent.toml | 2 +- .../consent/testdata/consent_files/junkfolder/junk-consent.toml | 2 +- internal/consent/testdata/consent_files/random_file.toml | 2 +- .../consent/testdata/consent_files/valid_false-consent.toml | 2 +- internal/consent/testdata/consent_files/valid_true-consent.toml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/consent/testdata/consent_files/extra_key-consent.toml b/internal/consent/testdata/consent_files/extra_key-consent.toml index 3ffc31a..c1699a0 100644 --- a/internal/consent/testdata/consent_files/extra_key-consent.toml +++ b/internal/consent/testdata/consent_files/extra_key-consent.toml @@ -1,2 +1,2 @@ consent_state=false -something=false \ No newline at end of file +something=false diff --git a/internal/consent/testdata/consent_files/invalid_file-consent.toml b/internal/consent/testdata/consent_files/invalid_file-consent.toml index d955e6e..c92baec 100644 --- a/internal/consent/testdata/consent_files/invalid_file-consent.toml +++ b/internal/consent/testdata/consent_files/invalid_file-consent.toml @@ -1 +1 @@ -asdfsadvla;skdjl \ No newline at end of file +asdfsadvla;skdjl diff --git a/internal/consent/testdata/consent_files/invalid_value-consent.toml b/internal/consent/testdata/consent_files/invalid_value-consent.toml index f8f187c..8b72e11 100644 --- a/internal/consent/testdata/consent_files/invalid_value-consent.toml +++ b/internal/consent/testdata/consent_files/invalid_value-consent.toml @@ -1 +1 @@ -consent_state="String!" \ No newline at end of file +consent_state="String!" diff --git a/internal/consent/testdata/consent_files/junkfolder/junk-consent.toml b/internal/consent/testdata/consent_files/junkfolder/junk-consent.toml index 613e7d9..41b40e2 100644 --- a/internal/consent/testdata/consent_files/junkfolder/junk-consent.toml +++ b/internal/consent/testdata/consent_files/junkfolder/junk-consent.toml @@ -1 +1 @@ -consent_state=true \ No newline at end of file +consent_state=true diff --git a/internal/consent/testdata/consent_files/random_file.toml b/internal/consent/testdata/consent_files/random_file.toml index 8689401..8070820 100644 --- a/internal/consent/testdata/consent_files/random_file.toml +++ b/internal/consent/testdata/consent_files/random_file.toml @@ -1 +1 @@ -consent_state=false \ No newline at end of file +consent_state=false diff --git a/internal/consent/testdata/consent_files/valid_false-consent.toml b/internal/consent/testdata/consent_files/valid_false-consent.toml index 8689401..8070820 100644 --- a/internal/consent/testdata/consent_files/valid_false-consent.toml +++ b/internal/consent/testdata/consent_files/valid_false-consent.toml @@ -1 +1 @@ -consent_state=false \ No newline at end of file +consent_state=false diff --git a/internal/consent/testdata/consent_files/valid_true-consent.toml b/internal/consent/testdata/consent_files/valid_true-consent.toml index 613e7d9..41b40e2 100644 --- a/internal/consent/testdata/consent_files/valid_true-consent.toml +++ b/internal/consent/testdata/consent_files/valid_true-consent.toml @@ -1 +1 @@ -consent_state=true \ No newline at end of file +consent_state=true From 8d76e9883439756d1ce5059a4ea041fb1faebe5e Mon Sep 17 00:00:00 2001 From: kat <28567881+hk21702@users.noreply.github.com> Date: Wed, 15 Jan 2025 19:09:30 -0500 Subject: [PATCH 05/26] Enhance consent component with improved logging and error messages --- internal/consent/consent.go | 19 +++++++++++-------- internal/consent/consent_test.go | 6 +++--- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/internal/consent/consent.go b/internal/consent/consent.go index fcf129e..696e03d 100644 --- a/internal/consent/consent.go +++ b/internal/consent/consent.go @@ -126,27 +126,28 @@ func getMatchingConsentFiles(sources []string, folderPath string) (sourceFiles m continue } + // Global file + if entry.Name() == constants.BaseConsentFileName { + globalFile = filepath.Join(folderPath, entry.Name()) + slog.Debug("Found global consent file", "file", globalFile) + continue + } + if len(sources) == 0 { - // Global file - if entry.Name() == constants.BaseConsentFileName { - globalFile = filepath.Join(folderPath, entry.Name()) - continue - } // Source file if !strings.HasSuffix(entry.Name(), constants.ConsentSourceBaseSeparator+constants.BaseConsentFileName) { continue } source := strings.TrimSuffix(entry.Name(), constants.ConsentSourceBaseSeparator+constants.BaseConsentFileName) sourceFiles[source] = filepath.Join(folderPath, entry.Name()) + slog.Debug("Found source consent file", "file", sourceFiles[source]) continue } for _, source := range sources { if entry.Name() == source+constants.ConsentSourceBaseSeparator+constants.BaseConsentFileName { sourceFiles[source] = filepath.Join(folderPath, entry.Name()) - break - } else if entry.Name() == constants.BaseConsentFileName { - globalFile = filepath.Join(folderPath, entry.Name()) + slog.Debug("Found matching source consent file", "file", sourceFiles[source]) break } } @@ -160,6 +161,7 @@ func readConsentFile(filePath string) (*consentFile, error) { return &consent, nil } _, err := toml.DecodeFile(filePath, &consent) + slog.Debug("Read consent file", "file", filePath, "consent", consent.ConsentState) return &consent, err } @@ -188,6 +190,7 @@ func writeConsentFile(filePath string, consent *consentFile) (err error) { if err := os.Rename(tempFile.Name(), filePath); err != nil { return fmt.Errorf("could not rename temporary file: %w", err) } + slog.Debug("Wrote consent file", "file", filePath, "consent", consent.ConsentState) return nil } diff --git a/internal/consent/consent_test.go b/internal/consent/consent_test.go index 2d9ca34..208d707 100644 --- a/internal/consent/consent_test.go +++ b/internal/consent/consent_test.go @@ -50,7 +50,7 @@ func TestGetConsentStates(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() cDir, err := setupTmpConsentFiles(t, tc.globalFile) - require.NoError(t, err, "failed to setup temporary consent files") + require.NoError(t, err, "Setup: failed to setup temporary consent files") defer cDir.cleanup(t) cm := consent.New(cDir.dir) @@ -108,7 +108,7 @@ func TestSetConsentStates(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() cDir, err := setupTmpConsentFiles(t, tc.globalFile) - require.NoError(t, err, "failed to setup temporary consent files") + require.NoError(t, err, "Setup: failed to setup temporary consent files") defer cDir.cleanup(t) cm := consent.New(cDir.dir) @@ -135,7 +135,7 @@ func TestSetConsentStates(t *testing.T) { // cleanup unlocks all the locks and removes the temporary directory including its contents. func (cDir consentDir) cleanup(t *testing.T) { t.Helper() - assert.NoError(t, os.RemoveAll(cDir.dir), "failed to remove temporary directory") + assert.NoError(t, os.RemoveAll(cDir.dir), "Cleanup: failed to remove temporary directory") } func copyFile(src, dst string) error { From 4ed6a889060020b7f4ec3896f55d4147531bfaae Mon Sep 17 00:00:00 2001 From: kat <28567881+hk21702@users.noreply.github.com> Date: Thu, 16 Jan 2025 12:11:50 -0500 Subject: [PATCH 06/26] Initial report utils --- internal/constants/constants.go | 3 + internal/reportutils/reportutils.go | 64 ++++++++++++++++++ internal/reportutils/reportutils_test.go | 85 ++++++++++++++++++++++++ internal/uploader/export_test.go | 2 + 4 files changed, 154 insertions(+) create mode 100644 internal/reportutils/reportutils.go create mode 100644 internal/reportutils/reportutils_test.go create mode 100644 internal/uploader/export_test.go diff --git a/internal/constants/constants.go b/internal/constants/constants.go index 40bc64b..dbebb2d 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -23,6 +23,9 @@ const ( // ConsentSourceBaseSeparator is the default separator between the source and the base name of the consent state files. ConsentSourceBaseSeparator = "-" + + // ReportExtension is the default extension for the report files. + ReportExtension = ".json" ) type options struct { diff --git a/internal/reportutils/reportutils.go b/internal/reportutils/reportutils.go new file mode 100644 index 0000000..9e77cf4 --- /dev/null +++ b/internal/reportutils/reportutils.go @@ -0,0 +1,64 @@ +// Package reportutils provides utility functions for handling reports. +package reportutils + +import ( + "log/slog" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/ubuntu/ubuntu-insights/internal/constants" +) + +// GetPeriodStart returns the start of the period window for a given period in seconds. +func GetPeriodStart(period uint) uint64 { + utcTime := uint64(time.Now().UTC().Unix()) + return utcTime - (utcTime % uint64(period)) +} + +// GetReportTime returns a unint64 representation of the report time from the report path. +func GetReportTime(reportPath string) (uint64, error) { + fileName := filepath.Base(reportPath) + return strconv.ParseUint(strings.TrimSuffix(fileName, filepath.Ext(fileName)), 10, 64) +} + +// GetReportPath returns the path for the most recent report within a period window, returning an empty string if no report is found. +// Not inclusive of the period end (periodStart + period). +func GetReportPath(reportsDir string, time uint64, period uint) (string, error) { + periodStart := time - (time % uint64(period)) + periodEnd := periodStart + uint64(period) + + // Reports names are utc timestamps. Get the most recent report within the period window. + var mostRecentReportPath string + files, err := os.ReadDir(reportsDir) + if err != nil { + slog.Error("Failed to read directory", "directory", reportsDir, "error", err) + return "", err + } + + for _, reportPath := range files { + if filepath.Ext(reportPath.Name()) != constants.ReportExtension { + slog.Info("Skipping non-report file, invalid extension", "file", reportPath.Name()) + continue + } + + reportTime, err := GetReportTime(reportPath.Name()) + if err != nil { + slog.Info("Skipping non-report file, invalid file name", "file", reportPath.Name()) + continue + } + + if reportTime < periodStart { + continue + } + if reportTime >= periodEnd { + break + } + + mostRecentReportPath = filepath.Join(reportsDir, reportPath.Name()) + } + + return mostRecentReportPath, nil +} diff --git a/internal/reportutils/reportutils_test.go b/internal/reportutils/reportutils_test.go new file mode 100644 index 0000000..7a34820 --- /dev/null +++ b/internal/reportutils/reportutils_test.go @@ -0,0 +1,85 @@ +package reportutils_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "github.com/ubuntu/ubuntu-insights/internal/reportutils" + "github.com/ubuntu/ubuntu-insights/internal/testutils" +) + +func TestGetReportPath(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + files []string + subDir string + subDirFiles []string + time uint64 + period uint + + wantErr bool + }{ + "Empty Directory": {time: 1, period: 500}, + "Files in subDir": {subDir: "subdir", subDirFiles: []string{"1.json", "2.json"}, time: 1, period: 500}, + "Empty subDir": {subDir: "subdir", time: 1, period: 500}, + "Invalid File Extension": {files: []string{"1.txt", "2.txt"}, time: 1, period: 500}, + "Invalid File Names": {files: []string{"-1.json", "-2.json", "-3.json", "test.json", "one.json"}, time: 1, period: 500}, + + "Specific Time: Single Valid Report": {files: []string{"1.json", "2.json"}, time: 2, period: 1}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + dir, err := setupTmpDir(t, tc.files, tc.subDir, tc.subDirFiles) + require.NoError(t, err, "Setup: failed to setup temporary directory") + defer os.RemoveAll(dir) + + got, err := reportutils.GetReportPath(dir, tc.time, tc.period) + if tc.wantErr { + require.Error(t, err, "expected an error but got none") + return + } + require.NoError(t, err, "got an unexpected error") + + want := testutils.LoadWithUpdateFromGolden(t, got) + require.Equal(t, want, got, "GetReportPath should return the most recent report within the period window") + }) + } +} + +func setupTmpDir(t *testing.T, files []string, subDir string, subDirFiles []string) (string, error) { + t.Helper() + + dir, err := os.MkdirTemp("", "reportutils-test") + if err != nil { + return "", err + } + + for _, file := range files { + path := filepath.Join(dir, file) + if err := os.WriteFile(path, []byte{}, 0600); err != nil { + return "", err + } + } + + if subDir != "" { + subDirPath := filepath.Join(dir, subDir) + if err := os.Mkdir(subDirPath, 0700); err != nil { + return "", err + } + + for _, file := range subDirFiles { + path := filepath.Join(subDirPath, file) + if err := os.WriteFile(path, []byte{}, 0600); err != nil { + return "", err + } + } + } + + return dir, nil +} diff --git a/internal/uploader/export_test.go b/internal/uploader/export_test.go new file mode 100644 index 0000000..7619a9b --- /dev/null +++ b/internal/uploader/export_test.go @@ -0,0 +1,2 @@ +package uploader + From ddba13a025e3a88776fc75d77eecbb3103cd070d Mon Sep 17 00:00:00 2001 From: kat <28567881+hk21702@users.noreply.github.com> Date: Thu, 16 Jan 2025 13:10:35 -0500 Subject: [PATCH 07/26] Refactor consent component --- go.mod | 3 ++ go.sum | 13 +++++ internal/consent/consent.go | 85 +++++++++++++++++--------------- internal/consent/consent_test.go | 2 +- internal/constants/constants.go | 7 ++- 5 files changed, 68 insertions(+), 42 deletions(-) diff --git a/go.mod b/go.mod index 954ae7b..fba82ff 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.10.0 + github.com/ubuntu/decorate v0.0.0-20240820145549-b76bb81d1209 gopkg.in/yaml.v3 v3.0.1 ) @@ -15,5 +16,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + golang.org/x/sys v0.3.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index 11637b2..f226956 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0 github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -14,19 +16,30 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/ubuntu/decorate v0.0.0-20240820145549-b76bb81d1209 h1:paNkjGGwB/Ypory/EPTwVR5uX94TDgrH4PGSaCNAvhE= +github.com/ubuntu/decorate v0.0.0-20240820145549-b76bb81d1209/go.mod h1:PUpwIgUuCQyuCz/gwiq6WYbo7IvtXXd8JqL01ez+jZE= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/consent/consent.go b/internal/consent/consent.go index 696e03d..2f5dba0 100644 --- a/internal/consent/consent.go +++ b/internal/consent/consent.go @@ -10,12 +10,13 @@ import ( "strings" "github.com/BurntSushi/toml" + "github.com/ubuntu/decorate" "github.com/ubuntu/ubuntu-insights/internal/constants" ) // Manager is a struct that manages consent files. type Manager struct { - folderPath string + path string } // States is a struct that represents the consent states for a list of sources and the global consent state. @@ -36,20 +37,22 @@ type consentFile struct { } // New returns a new ConsentManager. -func New(folderPath string) *Manager { - return &Manager{folderPath: folderPath} +// path is the folder the consents are store into. +func New(path string) *Manager { + return &Manager{path: path} } // GetConsentStates gets the consent state for the given sources and the global consent state. // If any of the sources do not have a consent file, it will be considered as a false state. // If a specified source does not have a consent file, it will not be included in the returned ConsentStates struct. -func (cm *Manager) GetConsentStates(sources []string) (*States, error) { - consentStates := States{SourceStates: make(map[string]consentStateResult)} +// TODO: Simplify to single source, update spec +func (cm Manager) GetConsentStates(sources []string) (consentStates States, err error) { + consentStates = States{SourceStates: make(map[string]consentStateResult)} - sourceFiles, globalFile, err := getMatchingConsentFiles(sources, cm.folderPath) + sourceFiles, globalFile, err := getMatchingConsentFiles(sources, cm.path) if err != nil { slog.Error("Error getting consent files", "error", err) - return nil, err + return States{}, err } results := make(chan consentStateResult, len(sourceFiles)) @@ -91,25 +94,28 @@ func (cm *Manager) GetConsentStates(sources []string) (*States, error) { consentStates.SourceStates[res.Source] = res } - return &consentStates, nil + return consentStates, nil } -// SetConsentState sets the consent state for the given source. +var consentSourceFilePattern = `%s` + constants.ConsentSourceBaseSeparator + constants.GlobalFileName + +// SetConsentState updates the consent state for the given source. // If the source is an empty string, then the global consent state will be set. // If the target consent file does not exist, it will be created. -func (cm *Manager) SetConsentState(source string, state bool) error { - var filePath string - if source == "" { - filePath = filepath.Join(cm.folderPath, constants.BaseConsentFileName) - } else { - filePath = filepath.Join(cm.folderPath, source+constants.ConsentSourceBaseSeparator+constants.BaseConsentFileName) +func (cm *Manager) SetConsentState(source string, state bool) (err error) { + defer decorate.OnError(&err, "could not set consent state:") + + p := filepath.Join(cm.path, constants.GlobalFileName) + if source != "" { + p = filepath.Join(cm.path, fmt.Sprintf(consentSourceFilePattern, source)) } consent := consentFile{ConsentState: state} - return writeConsentFile(filePath, &consent) + return consent.write(p) } -// getMatchingConsentFiles returns a map of all paths to consent files matching the given sources and a path to the global consent file. +// getMatchingConsentFiles returns a map of all paths to consent files matching the given sources and a path +// to the global consent file. // If sources is empty, all consent files in the folder will be returned. // If a source does not have a consent file, it will be represented as an empty string // Does not traverse subdirectories. @@ -127,7 +133,7 @@ func getMatchingConsentFiles(sources []string, folderPath string) (sourceFiles m } // Global file - if entry.Name() == constants.BaseConsentFileName { + if entry.Name() == constants.GlobalFileName { globalFile = filepath.Join(folderPath, entry.Name()) slog.Debug("Found global consent file", "file", globalFile) continue @@ -135,17 +141,17 @@ func getMatchingConsentFiles(sources []string, folderPath string) (sourceFiles m if len(sources) == 0 { // Source file - if !strings.HasSuffix(entry.Name(), constants.ConsentSourceBaseSeparator+constants.BaseConsentFileName) { + if !strings.HasSuffix(entry.Name(), constants.ConsentSourceBaseSeparator+constants.GlobalFileName) { continue } - source := strings.TrimSuffix(entry.Name(), constants.ConsentSourceBaseSeparator+constants.BaseConsentFileName) + source := strings.TrimSuffix(entry.Name(), constants.ConsentSourceBaseSeparator+constants.GlobalFileName) sourceFiles[source] = filepath.Join(folderPath, entry.Name()) slog.Debug("Found source consent file", "file", sourceFiles[source]) continue } for _, source := range sources { - if entry.Name() == source+constants.ConsentSourceBaseSeparator+constants.BaseConsentFileName { + if entry.Name() == fmt.Sprintf(consentSourceFilePattern, source) { sourceFiles[source] = filepath.Join(folderPath, entry.Name()) slog.Debug("Found matching source consent file", "file", sourceFiles[source]) break @@ -155,42 +161,43 @@ func getMatchingConsentFiles(sources []string, folderPath string) (sourceFiles m return sourceFiles, globalFile, err } -func readConsentFile(filePath string) (*consentFile, error) { +func readConsentFile(path string) (*consentFile, error) { var consent consentFile - if _, err := os.Stat(filePath); os.IsNotExist(err) { + if _, err := os.Stat(path); os.IsNotExist(err) { return &consent, nil } - _, err := toml.DecodeFile(filePath, &consent) - slog.Debug("Read consent file", "file", filePath, "consent", consent.ConsentState) + _, err := toml.DecodeFile(path, &consent) + slog.Debug("Read consent file", "file", path, "consent", consent.ConsentState) return &consent, err } // writeConsentFile writes the given consent file to the given path atomically, replacing it if it already exists. -// Not atomic in Windows. -func writeConsentFile(filePath string, consent *consentFile) (err error) { - dir := filepath.Dir(filePath) - tempFile, err := os.CreateTemp(dir, "consent-*.tmp") +// Not atomic on Windows. +func (cf consentFile) write(path string) (err error) { + tmp, err := os.CreateTemp(filepath.Dir(path), "consent-*.tmp") if err != nil { - return fmt.Errorf("could not create temporary file: %w", err) + return fmt.Errorf("could not create temporary file: %v", err) } defer func() { - tempFile.Close() - os.Remove(tempFile.Name()) + _ = tmp.Close() + if err := os.Remove(tmp.Name()); err != nil { + slog.Warn("Failed to remove temporary file when writing consent file", "file", tmp.Name(), "error", err) + } }() - if err := toml.NewEncoder(tempFile).Encode(consent); err != nil { - return fmt.Errorf("could not encode consent file: %w", err) + if err := toml.NewEncoder(tmp).Encode(cf); err != nil { + return fmt.Errorf("could not encode consent file: %v", err) } - if err := tempFile.Close(); err != nil { - return fmt.Errorf("could not close temporary file: %w", err) + if err := tmp.Close(); err != nil { + return fmt.Errorf("could not close temporary file: %v", err) } - if err := os.Rename(tempFile.Name(), filePath); err != nil { - return fmt.Errorf("could not rename temporary file: %w", err) + if err := os.Rename(tmp.Name(), path); err != nil { + return fmt.Errorf("could not rename temporary file: %v", err) } - slog.Debug("Wrote consent file", "file", filePath, "consent", consent.ConsentState) + slog.Debug("Wrote consent file", "file", path, "consent", cf.ConsentState) return nil } diff --git a/internal/consent/consent_test.go b/internal/consent/consent_test.go index 208d707..c79c8a3 100644 --- a/internal/consent/consent_test.go +++ b/internal/consent/consent_test.go @@ -100,7 +100,7 @@ func TestSetConsentStates(t *testing.T) { } type goldenFile struct { - States *consent.States + States consent.States FileCount uint } diff --git a/internal/constants/constants.go b/internal/constants/constants.go index c563390..0ced45b 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -18,11 +18,14 @@ const ( // DefaultLogLevel is the default log level selected without any verbosity flags. DefaultLogLevel = slog.LevelInfo - // DefaultConfigFileName is the default base name of the consent state files - BaseConsentFileName = "consent.toml" + // GlobalFileName is the default base name of the consent state files. + GlobalFileName = "consent.toml" // ConsentSourceBaseSeparator is the default separator between the source and the base name of the consent state files ConsentSourceBaseSeparator = "-" + + // Report extension is the default extension for the report files. + ReportExtension = ".json" ) type options struct { From 928f55ae6244e3f48b661bf6dcccc86a55b9de24 Mon Sep 17 00:00:00 2001 From: kat <28567881+hk21702@users.noreply.github.com> Date: Thu, 16 Jan 2025 17:02:14 -0500 Subject: [PATCH 08/26] Refactor consent manager to simplify functions --- internal/consent/consent.go | 134 +++++------------- internal/consent/consent_test.go | 37 +++-- internal/consent/export_test.go | 26 ++++ .../golden/valid_false_global_file | 1 + .../golden/valid_true_global_file | 1 + ...valid_true_global_file,_valid_false_source | 1 + .../valid_true_global_file,_valid_true_source | 1 + .../golden/invalid_file_global_file | 29 ---- .../golden/invalid_value_global_file | 29 ---- .../golden/no_global_file | 29 ---- .../golden/valid_false_global_file | 29 ---- .../golden/valid_true_global_file | 29 ---- ...l_file,_2_multiple_sources_(vtrue,_vfalse) | 13 -- ...e,_3_multiple_sources_(vtrue,_vfalse,_naf) | 13 -- ...alid_true_global_file,_invalid_file_source | 9 -- ...lid_true_global_file,_invalid_value_source | 9 -- .../valid_true_global_file,_no_file_source | 5 - ...valid_true_global_file,_valid_false_source | 9 -- .../valid_true_global_file,_valid_true_source | 9 -- .../golden/new_file,_write_global_false | 33 +---- .../golden/new_file,_write_global_true | 33 +---- .../golden/new_file,_write_source_false | 38 +---- .../golden/new_file,_write_source_true | 38 +---- .../overwrite_file,_write_diff_global_false | 33 +---- .../overwrite_file,_write_diff_global_true | 33 +---- .../overwrite_file,_write_diff_source_false | 33 +---- .../overwrite_file,_write_diff_source_true | 33 +---- .../golden/overwrite_file,_write_global_false | 33 +---- .../golden/overwrite_file,_write_global_true | 33 +---- .../golden/overwrite_file,_write_source_false | 33 +---- .../golden/overwrite_file,_write_source_true | 33 +---- internal/constants/constants.go | 2 +- 32 files changed, 130 insertions(+), 691 deletions(-) create mode 100644 internal/consent/export_test.go create mode 100644 internal/consent/testdata/TestGetConsentState/golden/valid_false_global_file create mode 100644 internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file create mode 100644 internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_false_source create mode 100644 internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_true_source delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/invalid_file_global_file delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/invalid_value_global_file delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/no_global_file delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_false_global_file delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_2_multiple_sources_(vtrue,_vfalse) delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_3_multiple_sources_(vtrue,_vfalse,_naf) delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_invalid_file_source delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_invalid_value_source delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_file_source delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_valid_false_source delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_valid_true_source diff --git a/internal/consent/consent.go b/internal/consent/consent.go index 2f5dba0..14f7a7a 100644 --- a/internal/consent/consent.go +++ b/internal/consent/consent.go @@ -19,18 +19,6 @@ type Manager struct { path string } -// States is a struct that represents the consent states for a list of sources and the global consent state. -type States struct { - SourceStates map[string]consentStateResult - GlobalState consentStateResult -} - -type consentStateResult struct { - Source string // the source for the consent state - State bool // the consent state for the source - ReadErr bool // true if there was an error reading the consent file -} - // consentFile is a struct that represents a consent file. type consentFile struct { ConsentState bool `toml:"consent_state"` @@ -42,59 +30,18 @@ func New(path string) *Manager { return &Manager{path: path} } -// GetConsentStates gets the consent state for the given sources and the global consent state. -// If any of the sources do not have a consent file, it will be considered as a false state. -// If a specified source does not have a consent file, it will not be included in the returned ConsentStates struct. -// TODO: Simplify to single source, update spec -func (cm Manager) GetConsentStates(sources []string) (consentStates States, err error) { - consentStates = States{SourceStates: make(map[string]consentStateResult)} - - sourceFiles, globalFile, err := getMatchingConsentFiles(sources, cm.path) +// GetConsentState gets the consent state for the given source. +// If the source do not have a consent file, it will be considered as a false state. +// If the source is an empty string, then the global consent state will be returned. +// If the target consent file does not exist, it will not be created. +func (cm Manager) GetConsentState(source string) (bool, error) { + sourceConsent, err := readConsentFile(cm.getConsentFile(source)) if err != nil { - slog.Error("Error getting consent files", "error", err) - return States{}, err - } - - results := make(chan consentStateResult, len(sourceFiles)) - defer close(results) - - // Global consent file - if globalFile != "" { - globalResult := make(chan consentStateResult, 1) - defer close(globalResult) - - go func() { - globalConsent, err := readConsentFile(globalFile) - if err != nil { - slog.Error("Error reading global consent file", "file", globalFile, "error", err) - globalResult <- consentStateResult{Source: "global", State: false, ReadErr: true} - return - } - globalResult <- consentStateResult{Source: "global", State: globalConsent.ConsentState, ReadErr: false} - }() - - consentStates.GlobalState = <-globalResult - } - - // Goroutine to read the consent files for each source, excluding the global consent file. - for source, filePath := range sourceFiles { - go func(source, filePath string) { - consent, err := readConsentFile(filePath) - if err != nil { - slog.Error("Error reading consent file", "source", source, "error", err) - results <- consentStateResult{Source: source, State: false, ReadErr: true} - return - } - results <- consentStateResult{Source: source, State: consent.ConsentState, ReadErr: false} - }(source, filePath) - } - - for range sourceFiles { - res := <-results - consentStates.SourceStates[res.Source] = res + slog.Error("Error reading source consent file", "source", source, "error", err) + return false, err } - return consentStates, nil + return sourceConsent.ConsentState, nil } var consentSourceFilePattern = `%s` + constants.ConsentSourceBaseSeparator + constants.GlobalFileName @@ -102,29 +49,32 @@ var consentSourceFilePattern = `%s` + constants.ConsentSourceBaseSeparator + con // SetConsentState updates the consent state for the given source. // If the source is an empty string, then the global consent state will be set. // If the target consent file does not exist, it will be created. -func (cm *Manager) SetConsentState(source string, state bool) (err error) { +func (cm Manager) SetConsentState(source string, state bool) (err error) { defer decorate.OnError(&err, "could not set consent state:") + consent := consentFile{ConsentState: state} + return consent.write(cm.getConsentFile(source)) +} + +// getConsentFile returns the expected path to the consent file for the given source. +// If source is blank, it returns the path to the global consent file. +// It does not check if the file exists, or if it is valid. +func (cm Manager) getConsentFile(source string) string { p := filepath.Join(cm.path, constants.GlobalFileName) if source != "" { p = filepath.Join(cm.path, fmt.Sprintf(consentSourceFilePattern, source)) } - consent := consentFile{ConsentState: state} - return consent.write(p) + return p } -// getMatchingConsentFiles returns a map of all paths to consent files matching the given sources and a path -// to the global consent file. -// If sources is empty, all consent files in the folder will be returned. -// If a source does not have a consent file, it will be represented as an empty string -// Does not traverse subdirectories. -func getMatchingConsentFiles(sources []string, folderPath string) (sourceFiles map[string]string, globalFile string, err error) { - sourceFiles = make(map[string]string) +// getSourceConsentFiles returns a map of all paths to validly named consent files in the folder, other than the global file. +func (cm Manager) getConsentFiles() (map[string]string, error) { + sourceFiles := make(map[string]string) - entries, err := os.ReadDir(folderPath) + entries, err := os.ReadDir(cm.path) if err != nil { - return sourceFiles, globalFile, err + return sourceFiles, err } for _, entry := range entries { @@ -132,40 +82,20 @@ func getMatchingConsentFiles(sources []string, folderPath string) (sourceFiles m continue } - // Global file - if entry.Name() == constants.GlobalFileName { - globalFile = filepath.Join(folderPath, entry.Name()) - slog.Debug("Found global consent file", "file", globalFile) + // Source file + if !strings.HasSuffix(entry.Name(), constants.ConsentSourceBaseSeparator+constants.GlobalFileName) { continue } - - if len(sources) == 0 { - // Source file - if !strings.HasSuffix(entry.Name(), constants.ConsentSourceBaseSeparator+constants.GlobalFileName) { - continue - } - source := strings.TrimSuffix(entry.Name(), constants.ConsentSourceBaseSeparator+constants.GlobalFileName) - sourceFiles[source] = filepath.Join(folderPath, entry.Name()) - slog.Debug("Found source consent file", "file", sourceFiles[source]) - continue - } - - for _, source := range sources { - if entry.Name() == fmt.Sprintf(consentSourceFilePattern, source) { - sourceFiles[source] = filepath.Join(folderPath, entry.Name()) - slog.Debug("Found matching source consent file", "file", sourceFiles[source]) - break - } - } + source := strings.TrimSuffix(entry.Name(), constants.ConsentSourceBaseSeparator+constants.GlobalFileName) + sourceFiles[source] = filepath.Join(cm.path, entry.Name()) + slog.Debug("Found source consent file", "file", sourceFiles[source]) } - return sourceFiles, globalFile, err + + return sourceFiles, nil } func readConsentFile(path string) (*consentFile, error) { var consent consentFile - if _, err := os.Stat(path); os.IsNotExist(err) { - return &consent, nil - } _, err := toml.DecodeFile(path, &consent) slog.Debug("Read consent file", "file", path, "consent", consent.ConsentState) @@ -181,7 +111,7 @@ func (cf consentFile) write(path string) (err error) { } defer func() { _ = tmp.Close() - if err := os.Remove(tmp.Name()); err != nil { + if err := os.Remove(tmp.Name()); err != nil && !os.IsNotExist(err) { slog.Warn("Failed to remove temporary file when writing consent file", "file", tmp.Name(), "error", err) } }() diff --git a/internal/consent/consent_test.go b/internal/consent/consent_test.go index c79c8a3..471fbb9 100644 --- a/internal/consent/consent_test.go +++ b/internal/consent/consent_test.go @@ -13,37 +13,35 @@ import ( "github.com/ubuntu/ubuntu-insights/internal/testutils" ) -// consentDir is a struct that holds a test's temporary directory and its locks. +// consentDir is a struct that holds a test's temporary directory. // It should be cleaned up after the test is done. type consentDir struct { dir string } -func TestGetConsentStates(t *testing.T) { +func TestGetConsentState(t *testing.T) { t.Parallel() tests := map[string]struct { - sources []string + source string globalFile string wantErr bool }{ - "No Global File": {}, + "No Global File": {wantErr: true}, // Global File Tests "Valid True Global File": {globalFile: "valid_true-consent.toml"}, "Valid False Global File": {globalFile: "valid_false-consent.toml"}, - "Invalid Value Global File": {globalFile: "invalid_value-consent.toml"}, - "Invalid File Global File": {globalFile: "invalid_file-consent.toml"}, + "Invalid Value Global File": {globalFile: "invalid_value-consent.toml", wantErr: true}, + "Invalid File Global File": {globalFile: "invalid_file-consent.toml", wantErr: true}, // Source Specific Tests - "Valid True Global File, Valid True Source": {globalFile: "valid_true-consent.toml", sources: []string{"valid_true"}}, - "Valid True Global File, Valid False Source": {globalFile: "valid_true-consent.toml", sources: []string{"valid_false"}}, - "Valid True Global File, Invalid Value Source": {globalFile: "valid_true-consent.toml", sources: []string{"invalid_value"}}, - "Valid True Global File, Invalid File Source": {globalFile: "valid_true-consent.toml", sources: []string{"invalid_file"}}, - "Valid True Global File, No File Source": {globalFile: "valid_true-consent.toml", sources: []string{"not_a_file"}}, - "Valid True Global File, 2 Multiple Sources (VTrue, VFalse)": {globalFile: "valid_true-consent.toml", sources: []string{"valid_true", "valid_false"}}, - "Valid True Global File, 3 Multiple Sources (VTrue, VFalse, NAF)": {globalFile: "valid_true-consent.toml", sources: []string{"valid_true", "valid_false", "not_a_file"}}, + "Valid True Global File, Valid True Source": {globalFile: "valid_true-consent.toml", source: "valid_true"}, + "Valid True Global File, Valid False Source": {globalFile: "valid_true-consent.toml", source: "valid_false"}, + "Valid True Global File, Invalid Value Source": {globalFile: "valid_true-consent.toml", source: "invalid_value", wantErr: true}, + "Valid True Global File, Invalid File Source": {globalFile: "valid_true-consent.toml", source: "invalid_file", wantErr: true}, + "Valid True Global File, No File Source": {globalFile: "valid_true-consent.toml", source: "not_a_file", wantErr: true}, } for name, tc := range tests { @@ -54,7 +52,7 @@ func TestGetConsentStates(t *testing.T) { defer cDir.cleanup(t) cm := consent.New(cDir.dir) - got, err := cm.GetConsentStates(tc.sources) + got, err := cm.GetConsentState(tc.source) if tc.wantErr { require.Error(t, err, "expected an error but got none") return @@ -62,7 +60,7 @@ func TestGetConsentStates(t *testing.T) { require.NoError(t, err, "got an unexpected error") want := testutils.LoadWithUpdateFromGoldenYAML(t, got) - require.Equal(t, want, got, "GetConsentStates should return expected consent states") + require.Equal(t, want, got, "GetConsentState should return expected consent state") }) } } @@ -71,7 +69,6 @@ func TestSetConsentStates(t *testing.T) { t.Parallel() tests := map[string]struct { - sources []string consentStates map[string]bool globalFile string @@ -100,8 +97,8 @@ func TestSetConsentStates(t *testing.T) { } type goldenFile struct { - States consent.States - FileCount uint + States map[string]bool + FileCount int } for name, tc := range tests { @@ -119,12 +116,12 @@ func TestSetConsentStates(t *testing.T) { } require.NoError(t, err, "got an unexpected error") - states, err := cm.GetConsentStates(tc.sources) + states, err := cm.GetAllSourceConsentStates(true) require.NoError(t, err, "got an unexpected error while getting consent states") d, err := os.ReadDir(cDir.dir) require.NoError(t, err, "failed to read temporary directory") - got := goldenFile{States: states, FileCount: uint(len(d))} + got := goldenFile{States: states, FileCount: len(d)} want := testutils.LoadWithUpdateFromGoldenYAML(t, got) require.Equal(t, want, got, "GetConsentStates should return expected consent states") diff --git a/internal/consent/export_test.go b/internal/consent/export_test.go new file mode 100644 index 0000000..f887256 --- /dev/null +++ b/internal/consent/export_test.go @@ -0,0 +1,26 @@ +package consent + +// GetAllSourceConsentStates gets the consent states for all sources. +// It does not get the global consent state. +// If continueOnErr is true, it will continue to the next source if an error occurs. +func (cm Manager) GetAllSourceConsentStates(continueOnErr bool) (map[string]bool, error) { + p, err := cm.getConsentFiles() + if err != nil { + return nil, err + } + + consentStates := make(map[string]bool) + for source, path := range p { + consent, err := readConsentFile(path) + if err != nil && !continueOnErr { + return nil, err + } + if err != nil { + continue + } + + consentStates[source] = consent.ConsentState + } + + return consentStates, nil +} diff --git a/internal/consent/testdata/TestGetConsentState/golden/valid_false_global_file b/internal/consent/testdata/TestGetConsentState/golden/valid_false_global_file new file mode 100644 index 0000000..c508d53 --- /dev/null +++ b/internal/consent/testdata/TestGetConsentState/golden/valid_false_global_file @@ -0,0 +1 @@ +false diff --git a/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file b/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file new file mode 100644 index 0000000..27ba77d --- /dev/null +++ b/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file @@ -0,0 +1 @@ +true diff --git a/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_false_source b/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_false_source new file mode 100644 index 0000000..c508d53 --- /dev/null +++ b/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_false_source @@ -0,0 +1 @@ +false diff --git a/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_true_source b/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_true_source new file mode 100644 index 0000000..27ba77d --- /dev/null +++ b/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_true_source @@ -0,0 +1 @@ +true diff --git a/internal/consent/testdata/TestGetConsentStates/golden/invalid_file_global_file b/internal/consent/testdata/TestGetConsentStates/golden/invalid_file_global_file deleted file mode 100644 index 2ae9869..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/invalid_file_global_file +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: false - readerr: true diff --git a/internal/consent/testdata/TestGetConsentStates/golden/invalid_value_global_file b/internal/consent/testdata/TestGetConsentStates/golden/invalid_value_global_file deleted file mode 100644 index 2ae9869..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/invalid_value_global_file +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: false - readerr: true diff --git a/internal/consent/testdata/TestGetConsentStates/golden/no_global_file b/internal/consent/testdata/TestGetConsentStates/golden/no_global_file deleted file mode 100644 index 3285f07..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/no_global_file +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: "" - state: false - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_false_global_file b/internal/consent/testdata/TestGetConsentStates/golden/valid_false_global_file deleted file mode 100644 index 8a2bbfd..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_false_global_file +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: false - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file deleted file mode 100644 index 7ff6725..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_2_multiple_sources_(vtrue,_vfalse) b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_2_multiple_sources_(vtrue,_vfalse) deleted file mode 100644 index bc8c6a4..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_2_multiple_sources_(vtrue,_vfalse) +++ /dev/null @@ -1,13 +0,0 @@ -sourcestates: - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_3_multiple_sources_(vtrue,_vfalse,_naf) b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_3_multiple_sources_(vtrue,_vfalse,_naf) deleted file mode 100644 index bc8c6a4..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_3_multiple_sources_(vtrue,_vfalse,_naf) +++ /dev/null @@ -1,13 +0,0 @@ -sourcestates: - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_invalid_file_source b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_invalid_file_source deleted file mode 100644 index 1b41f2d..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_invalid_file_source +++ /dev/null @@ -1,9 +0,0 @@ -sourcestates: - invalid_file: - source: invalid_file - state: false - readerr: true -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_invalid_value_source b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_invalid_value_source deleted file mode 100644 index 879de3f..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_invalid_value_source +++ /dev/null @@ -1,9 +0,0 @@ -sourcestates: - invalid_value: - source: invalid_value - state: false - readerr: true -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_file_source b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_file_source deleted file mode 100644 index 462e7f4..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_file_source +++ /dev/null @@ -1,5 +0,0 @@ -sourcestates: {} -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_valid_false_source b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_valid_false_source deleted file mode 100644 index 63ed6e9..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_valid_false_source +++ /dev/null @@ -1,9 +0,0 @@ -sourcestates: - valid_false: - source: valid_false - state: false - readerr: false -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_valid_true_source b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_valid_true_source deleted file mode 100644 index 4add7f5..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_valid_true_source +++ /dev/null @@ -1,9 +0,0 @@ -sourcestates: - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_false b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_false index 28a95b6..38c7a46 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_false +++ b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_false @@ -1,31 +1,6 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false - globalstate: - source: global - state: false - readerr: false + empty: false + extra_key: false + valid_false: false + valid_true: true filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_true b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_true index bbe6efd..38c7a46 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_true +++ b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_true @@ -1,31 +1,6 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false - globalstate: - source: global - state: true - readerr: false + empty: false + extra_key: false + valid_false: false + valid_true: true filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_false b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_false index d50e411..e6f5011 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_false +++ b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_false @@ -1,35 +1,7 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - new_false: - source: new_false - state: false - readerr: false - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false - globalstate: - source: "" - state: false - readerr: false + empty: false + extra_key: false + new_false: false + valid_false: false + valid_true: true filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_true b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_true index 0491904..b5c530c 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_true +++ b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_true @@ -1,35 +1,7 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - new_true: - source: new_true - state: true - readerr: false - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false - globalstate: - source: "" - state: false - readerr: false + empty: false + extra_key: false + new_true: true + valid_false: false + valid_true: true filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_false b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_false index 28a95b6..38c7a46 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_false +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_false @@ -1,31 +1,6 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false - globalstate: - source: global - state: false - readerr: false + empty: false + extra_key: false + valid_false: false + valid_true: true filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_true b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_true index bbe6efd..38c7a46 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_true +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_true @@ -1,31 +1,6 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false - globalstate: - source: global - state: true - readerr: false + empty: false + extra_key: false + valid_false: false + valid_true: true filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_false b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_false index e97680a..20cfa22 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_false +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_false @@ -1,31 +1,6 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: false - readerr: false - globalstate: - source: global - state: true - readerr: false + empty: false + extra_key: false + valid_false: false + valid_true: false filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_true b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_true index d035451..723440d 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_true +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_true @@ -1,31 +1,6 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: true - readerr: false - valid_true: - source: valid_true - state: true - readerr: false - globalstate: - source: global - state: true - readerr: false + empty: false + extra_key: false + valid_false: true + valid_true: true filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_false b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_false index 28a95b6..38c7a46 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_false +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_false @@ -1,31 +1,6 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false - globalstate: - source: global - state: false - readerr: false + empty: false + extra_key: false + valid_false: false + valid_true: true filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_true b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_true index bbe6efd..38c7a46 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_true +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_true @@ -1,31 +1,6 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false - globalstate: - source: global - state: true - readerr: false + empty: false + extra_key: false + valid_false: false + valid_true: true filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_false b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_false index 28a95b6..38c7a46 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_false +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_false @@ -1,31 +1,6 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false - globalstate: - source: global - state: false - readerr: false + empty: false + extra_key: false + valid_false: false + valid_true: true filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_true b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_true index bbe6efd..38c7a46 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_true +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_true @@ -1,31 +1,6 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false - globalstate: - source: global - state: true - readerr: false + empty: false + extra_key: false + valid_false: false + valid_true: true filecount: 9 diff --git a/internal/constants/constants.go b/internal/constants/constants.go index 0ced45b..9a400f1 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -24,7 +24,7 @@ const ( // ConsentSourceBaseSeparator is the default separator between the source and the base name of the consent state files ConsentSourceBaseSeparator = "-" - // Report extension is the default extension for the report files. + // ReportExtension is the default extension for the report files. ReportExtension = ".json" ) From 4da7645732667d0c684d6c029f3681bbc2054a54 Mon Sep 17 00:00:00 2001 From: Trevor Shoe Date: Thu, 16 Jan 2025 13:17:14 -0500 Subject: [PATCH 09/26] Add mac and windows sysinfo bones --- internal/collector/sysinfo/sysinfo_darwin.go | 9 +++++++++ internal/collector/sysinfo/sysinfo_windows.go | 9 +++++++++ 2 files changed, 18 insertions(+) create mode 100644 internal/collector/sysinfo/sysinfo_darwin.go create mode 100644 internal/collector/sysinfo/sysinfo_windows.go diff --git a/internal/collector/sysinfo/sysinfo_darwin.go b/internal/collector/sysinfo/sysinfo_darwin.go new file mode 100644 index 0000000..08b8dca --- /dev/null +++ b/internal/collector/sysinfo/sysinfo_darwin.go @@ -0,0 +1,9 @@ +package sysinfo + +func (s Manager) collectHardware() (HwInfo, error) { + return HwInfo{}, nil +} + +func (s Manager) collectSoftware() (SwInfo, error) { + return SwInfo{}, nil +} diff --git a/internal/collector/sysinfo/sysinfo_windows.go b/internal/collector/sysinfo/sysinfo_windows.go new file mode 100644 index 0000000..08b8dca --- /dev/null +++ b/internal/collector/sysinfo/sysinfo_windows.go @@ -0,0 +1,9 @@ +package sysinfo + +func (s Manager) collectHardware() (HwInfo, error) { + return HwInfo{}, nil +} + +func (s Manager) collectSoftware() (SwInfo, error) { + return SwInfo{}, nil +} From 288c8eeb1439c79bc46c018230ab15c4fdfe0b87 Mon Sep 17 00:00:00 2001 From: Trevor Shoe Date: Thu, 16 Jan 2025 13:31:55 -0500 Subject: [PATCH 10/26] Apply linting --- internal/collector/sysinfo/sysinfo.go | 2 +- internal/collector/sysinfo/sysinfo_linux.go | 2 -- internal/collector/sysinfo/sysinfo_linux_test.go | 3 +-- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/internal/collector/sysinfo/sysinfo.go b/internal/collector/sysinfo/sysinfo.go index d57a211..82e1369 100644 --- a/internal/collector/sysinfo/sysinfo.go +++ b/internal/collector/sysinfo/sysinfo.go @@ -1,4 +1,4 @@ -// package sysinfo allows collecting "common" system information for all insight reports. +// Package sysinfo allows collecting "common" system information for all insight reports. package sysinfo type options struct { diff --git a/internal/collector/sysinfo/sysinfo_linux.go b/internal/collector/sysinfo/sysinfo_linux.go index 0562562..915425e 100644 --- a/internal/collector/sysinfo/sysinfo_linux.go +++ b/internal/collector/sysinfo/sysinfo_linux.go @@ -6,7 +6,6 @@ import ( ) func (s Manager) collectHardware() (hwInfo HwInfo, err error) { - // System vendor d, err := os.ReadFile(filepath.Join(s.root, "sys/class/dmi/id/sys_vendor")) if err != nil { @@ -20,6 +19,5 @@ func (s Manager) collectHardware() (hwInfo HwInfo, err error) { } func (s Manager) collectSoftware() (swInfo SwInfo, err error) { - return swInfo, nil } diff --git a/internal/collector/sysinfo/sysinfo_linux_test.go b/internal/collector/sysinfo/sysinfo_linux_test.go index c742f6d..5a3c8ce 100644 --- a/internal/collector/sysinfo/sysinfo_linux_test.go +++ b/internal/collector/sysinfo/sysinfo_linux_test.go @@ -16,7 +16,7 @@ func TestNew(t *testing.T) { }{ "Instantiate a sys info manager": {}, } - for name, _ := range tests { + for name := range tests { t.Run(name, func(t *testing.T) { t.Parallel() @@ -56,5 +56,4 @@ func TestCollect(t *testing.T) { require.Equal(t, want, got, "Collect should return expected sys information") }) } - } From bfa40998d496d9fe2269faf925c1aeafb6fa6737 Mon Sep 17 00:00:00 2001 From: kat <28567881+hk21702@users.noreply.github.com> Date: Fri, 17 Jan 2025 10:56:31 -0500 Subject: [PATCH 11/26] Refactor and implement tests for reportutils --- internal/constants/constants.go | 4 +- internal/reportutils/reportutils.go | 49 +++++++---- internal/reportutils/reportutils_test.go | 85 +++++++++++++++++-- .../TestGetReportPath/golden/empty_directory | 0 .../TestGetReportPath/golden/empty_subdir | 0 .../TestGetReportPath/golden/files_in_subdir | 0 .../golden/invalid_file_extension | 0 .../golden/invalid_file_names | 0 .../golden/negative_timestamp | 1 + .../golden/not_inclusive_period | 1 + .../golden/specific_time_single_valid_report | 1 + .../TestGetReportTime/golden/alt_extension | 1 + .../golden/valid_report_time | 1 + .../golden/valid_report_time_with_path | 1 + 14 files changed, 117 insertions(+), 27 deletions(-) create mode 100644 internal/reportutils/testdata/TestGetReportPath/golden/empty_directory create mode 100644 internal/reportutils/testdata/TestGetReportPath/golden/empty_subdir create mode 100644 internal/reportutils/testdata/TestGetReportPath/golden/files_in_subdir create mode 100644 internal/reportutils/testdata/TestGetReportPath/golden/invalid_file_extension create mode 100644 internal/reportutils/testdata/TestGetReportPath/golden/invalid_file_names create mode 100644 internal/reportutils/testdata/TestGetReportPath/golden/negative_timestamp create mode 100644 internal/reportutils/testdata/TestGetReportPath/golden/not_inclusive_period create mode 100644 internal/reportutils/testdata/TestGetReportPath/golden/specific_time_single_valid_report create mode 100644 internal/reportutils/testdata/TestGetReportTime/golden/alt_extension create mode 100644 internal/reportutils/testdata/TestGetReportTime/golden/valid_report_time create mode 100644 internal/reportutils/testdata/TestGetReportTime/golden/valid_report_time_with_path diff --git a/internal/constants/constants.go b/internal/constants/constants.go index 3210a80..448720d 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -24,8 +24,8 @@ const ( // ConsentSourceBaseSeparator is the default separator between the source and the base name of the consent state files. ConsentSourceBaseSeparator = "-" - // ReportExtension is the default extension for the report files. - ReportExtension = ".json" + // ReportExt is the default extension for the report files. + ReportExt = ".json" ) type options struct { diff --git a/internal/reportutils/reportutils.go b/internal/reportutils/reportutils.go index 9e77cf4..615ceb6 100644 --- a/internal/reportutils/reportutils.go +++ b/internal/reportutils/reportutils.go @@ -2,6 +2,7 @@ package reportutils import ( + "errors" "log/slog" "os" "path/filepath" @@ -12,41 +13,53 @@ import ( "github.com/ubuntu/ubuntu-insights/internal/constants" ) +// ErrInvalidPeriod is returned when a function requiring a period, received an invalid, period that isn't a non-negative integer. +var ErrInvalidPeriod = errors.New("invalid period, period should be a positive integer") + // GetPeriodStart returns the start of the period window for a given period in seconds. -func GetPeriodStart(period uint) uint64 { - utcTime := uint64(time.Now().UTC().Unix()) - return utcTime - (utcTime % uint64(period)) +func GetPeriodStart(period int) (int64, error) { + if period <= 0 { + return 0, ErrInvalidPeriod + } + utcTime := time.Now().UTC().Unix() + return utcTime - (utcTime % int64(period)), nil } -// GetReportTime returns a unint64 representation of the report time from the report path. -func GetReportTime(reportPath string) (uint64, error) { - fileName := filepath.Base(reportPath) - return strconv.ParseUint(strings.TrimSuffix(fileName, filepath.Ext(fileName)), 10, 64) +// GetReportTime returns a int64 representation of the report time from the report path. +func GetReportTime(path string) (int64, error) { + fileName := filepath.Base(path) + return strconv.ParseInt(strings.TrimSuffix(fileName, filepath.Ext(fileName)), 10, 64) } // GetReportPath returns the path for the most recent report within a period window, returning an empty string if no report is found. // Not inclusive of the period end (periodStart + period). -func GetReportPath(reportsDir string, time uint64, period uint) (string, error) { - periodStart := time - (time % uint64(period)) - periodEnd := periodStart + uint64(period) +// +// For example, given reports 1 and 7, with time 2 and period 7, the function will return the path for report 1. +func GetReportPath(dir string, time int64, period int) (string, error) { + if period <= 0 { + return "", ErrInvalidPeriod + } + + periodStart := time - (time % int64(period)) + periodEnd := periodStart + int64(period) // Reports names are utc timestamps. Get the most recent report within the period window. var mostRecentReportPath string - files, err := os.ReadDir(reportsDir) + files, err := os.ReadDir(dir) if err != nil { - slog.Error("Failed to read directory", "directory", reportsDir, "error", err) + slog.Error("Failed to read directory", "directory", dir, "error", err) return "", err } - for _, reportPath := range files { - if filepath.Ext(reportPath.Name()) != constants.ReportExtension { - slog.Info("Skipping non-report file, invalid extension", "file", reportPath.Name()) + for _, file := range files { + if filepath.Ext(file.Name()) != constants.ReportExt { + slog.Info("Skipping non-report file, invalid extension", "file", file.Name()) continue } - reportTime, err := GetReportTime(reportPath.Name()) + reportTime, err := GetReportTime(file.Name()) if err != nil { - slog.Info("Skipping non-report file, invalid file name", "file", reportPath.Name()) + slog.Info("Skipping non-report file, invalid file name", "file", file.Name()) continue } @@ -57,7 +70,7 @@ func GetReportPath(reportsDir string, time uint64, period uint) (string, error) break } - mostRecentReportPath = filepath.Join(reportsDir, reportPath.Name()) + mostRecentReportPath = filepath.Join(dir, file.Name()) } return mostRecentReportPath, nil diff --git a/internal/reportutils/reportutils_test.go b/internal/reportutils/reportutils_test.go index 7a34820..3216cbb 100644 --- a/internal/reportutils/reportutils_test.go +++ b/internal/reportutils/reportutils_test.go @@ -10,6 +10,67 @@ import ( "github.com/ubuntu/ubuntu-insights/internal/testutils" ) +func TestGetPeriodStart(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + period int + + wantErr error + }{ + "Valid Period": {period: 500}, + + "Invalid Negative Period": {period: -500, wantErr: reportutils.ErrInvalidPeriod}, + "Invalid Zero Period": {period: 0, wantErr: reportutils.ErrInvalidPeriod}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + got, err := reportutils.GetPeriodStart(tc.period) + if tc.wantErr != nil { + require.ErrorIs(t, err, tc.wantErr) + return + } + require.NoError(t, err, "got an unexpected error") + + require.IsType(t, int64(0), got) + }) + } +} + +func TestGetReportTime(t *testing.T) { + tests := map[string]struct { + path string + + wantErr bool + }{ + "Valid Report Time": {path: "1627847285.json", wantErr: false}, + "Valid Report Time with Path": {path: "/some/dir/1627847285.json", wantErr: false}, + "Alt Extension": {path: "1627847285.txt", wantErr: false}, + + "Empty File Name": {path: ".json", wantErr: true}, + "Invalid Report Time": {path: "invalid.json", wantErr: true}, + "Invalid Report Mixed": {path: "i-1.json", wantErr: true}, + "Invalid Report Time with Path": {path: "/123/123/invalid.json", wantErr: true}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + got, err := reportutils.GetReportTime(tc.path) + if tc.wantErr { + require.Error(t, err, "expected an error but got none") + return + } + require.NoError(t, err, "got an unexpected error") + + want := testutils.LoadWithUpdateFromGoldenYAML(t, got) + require.Equal(t, want, got, "GetReportTime should return the report time from the report path") + }) + } +} + func TestGetReportPath(t *testing.T) { t.Parallel() @@ -17,18 +78,23 @@ func TestGetReportPath(t *testing.T) { files []string subDir string subDirFiles []string - time uint64 - period uint + time int64 + period int - wantErr bool + wantErr error }{ "Empty Directory": {time: 1, period: 500}, "Files in subDir": {subDir: "subdir", subDirFiles: []string{"1.json", "2.json"}, time: 1, period: 500}, "Empty subDir": {subDir: "subdir", time: 1, period: 500}, "Invalid File Extension": {files: []string{"1.txt", "2.txt"}, time: 1, period: 500}, - "Invalid File Names": {files: []string{"-1.json", "-2.json", "-3.json", "test.json", "one.json"}, time: 1, period: 500}, + "Invalid File Names": {files: []string{"i-1.json", "i-2.json", "i-3.json", "test.json", "one.json"}, time: -100, period: 500}, - "Specific Time: Single Valid Report": {files: []string{"1.json", "2.json"}, time: 2, period: 1}, + "Specific Time Single Valid Report": {files: []string{"1.json", "2.json"}, time: 2, period: 1}, + "Negative Timestamp": {files: []string{"-100.json", "-101.json"}, time: -150, period: 100}, + "Not Inclusive Period": {files: []string{"1.json", "7.json"}, time: 2, period: 7}, + + "Invalid Negative Period": {files: []string{"1.json", "7.json"}, time: 2, period: -7, wantErr: reportutils.ErrInvalidPeriod}, + "Invalid Zero Period": {files: []string{"1.json", "7.json"}, time: 2, period: 0, wantErr: reportutils.ErrInvalidPeriod}, } for name, tc := range tests { @@ -40,12 +106,17 @@ func TestGetReportPath(t *testing.T) { defer os.RemoveAll(dir) got, err := reportutils.GetReportPath(dir, tc.time, tc.period) - if tc.wantErr { - require.Error(t, err, "expected an error but got none") + if tc.wantErr != nil { + require.ErrorIs(t, err, tc.wantErr) return } require.NoError(t, err, "got an unexpected error") + if got != "" { + got, err = filepath.Rel(dir, got) + require.NoError(t, err, "failed to get relative path") + } + want := testutils.LoadWithUpdateFromGolden(t, got) require.Equal(t, want, got, "GetReportPath should return the most recent report within the period window") }) diff --git a/internal/reportutils/testdata/TestGetReportPath/golden/empty_directory b/internal/reportutils/testdata/TestGetReportPath/golden/empty_directory new file mode 100644 index 0000000..e69de29 diff --git a/internal/reportutils/testdata/TestGetReportPath/golden/empty_subdir b/internal/reportutils/testdata/TestGetReportPath/golden/empty_subdir new file mode 100644 index 0000000..e69de29 diff --git a/internal/reportutils/testdata/TestGetReportPath/golden/files_in_subdir b/internal/reportutils/testdata/TestGetReportPath/golden/files_in_subdir new file mode 100644 index 0000000..e69de29 diff --git a/internal/reportutils/testdata/TestGetReportPath/golden/invalid_file_extension b/internal/reportutils/testdata/TestGetReportPath/golden/invalid_file_extension new file mode 100644 index 0000000..e69de29 diff --git a/internal/reportutils/testdata/TestGetReportPath/golden/invalid_file_names b/internal/reportutils/testdata/TestGetReportPath/golden/invalid_file_names new file mode 100644 index 0000000..e69de29 diff --git a/internal/reportutils/testdata/TestGetReportPath/golden/negative_timestamp b/internal/reportutils/testdata/TestGetReportPath/golden/negative_timestamp new file mode 100644 index 0000000..411e61e --- /dev/null +++ b/internal/reportutils/testdata/TestGetReportPath/golden/negative_timestamp @@ -0,0 +1 @@ +-100.json \ No newline at end of file diff --git a/internal/reportutils/testdata/TestGetReportPath/golden/not_inclusive_period b/internal/reportutils/testdata/TestGetReportPath/golden/not_inclusive_period new file mode 100644 index 0000000..a3d3d4c --- /dev/null +++ b/internal/reportutils/testdata/TestGetReportPath/golden/not_inclusive_period @@ -0,0 +1 @@ +1.json \ No newline at end of file diff --git a/internal/reportutils/testdata/TestGetReportPath/golden/specific_time_single_valid_report b/internal/reportutils/testdata/TestGetReportPath/golden/specific_time_single_valid_report new file mode 100644 index 0000000..e1f33a0 --- /dev/null +++ b/internal/reportutils/testdata/TestGetReportPath/golden/specific_time_single_valid_report @@ -0,0 +1 @@ +2.json \ No newline at end of file diff --git a/internal/reportutils/testdata/TestGetReportTime/golden/alt_extension b/internal/reportutils/testdata/TestGetReportTime/golden/alt_extension new file mode 100644 index 0000000..d29665a --- /dev/null +++ b/internal/reportutils/testdata/TestGetReportTime/golden/alt_extension @@ -0,0 +1 @@ +1627847285 diff --git a/internal/reportutils/testdata/TestGetReportTime/golden/valid_report_time b/internal/reportutils/testdata/TestGetReportTime/golden/valid_report_time new file mode 100644 index 0000000..d29665a --- /dev/null +++ b/internal/reportutils/testdata/TestGetReportTime/golden/valid_report_time @@ -0,0 +1 @@ +1627847285 diff --git a/internal/reportutils/testdata/TestGetReportTime/golden/valid_report_time_with_path b/internal/reportutils/testdata/TestGetReportTime/golden/valid_report_time_with_path new file mode 100644 index 0000000..d29665a --- /dev/null +++ b/internal/reportutils/testdata/TestGetReportTime/golden/valid_report_time_with_path @@ -0,0 +1 @@ +1627847285 From 46e0dd6e2e1d5af8b9ae82a9c1f8e3456d77de10 Mon Sep 17 00:00:00 2001 From: kat <28567881+hk21702@users.noreply.github.com> Date: Thu, 16 Jan 2025 17:02:14 -0500 Subject: [PATCH 12/26] Refactor consent manager to simplify functions --- internal/consent/consent.go | 132 ++++-------------- internal/consent/consent_test.go | 37 +++-- internal/consent/export_test.go | 26 ++++ .../golden/valid_false_global_file | 1 + .../golden/valid_true_global_file | 1 + ...valid_true_global_file,_valid_false_source | 1 + .../valid_true_global_file,_valid_true_source | 1 + .../golden/invalid_file_global_file | 29 ---- .../golden/invalid_value_global_file | 29 ---- .../golden/no_global_file | 29 ---- .../golden/valid_false_global_file | 29 ---- .../golden/valid_true_global_file | 29 ---- ...l_file,_2_multiple_sources_(vtrue,_vfalse) | 13 -- ...e,_3_multiple_sources_(vtrue,_vfalse,_naf) | 13 -- ...alid_true_global_file,_invalid_file_source | 9 -- ...lid_true_global_file,_invalid_value_source | 9 -- .../valid_true_global_file,_no_file_source | 5 - ...valid_true_global_file,_valid_false_source | 9 -- .../valid_true_global_file,_valid_true_source | 9 -- .../golden/new_file,_write_global_false | 33 +---- .../golden/new_file,_write_global_true | 33 +---- .../golden/new_file,_write_source_false | 38 +---- .../golden/new_file,_write_source_true | 38 +---- .../overwrite_file,_write_diff_global_false | 33 +---- .../overwrite_file,_write_diff_global_true | 33 +---- .../overwrite_file,_write_diff_source_false | 33 +---- .../overwrite_file,_write_diff_source_true | 33 +---- .../golden/overwrite_file,_write_global_false | 33 +---- .../golden/overwrite_file,_write_global_true | 33 +---- .../golden/overwrite_file,_write_source_false | 33 +---- .../golden/overwrite_file,_write_source_true | 33 +---- internal/constants/constants.go | 2 +- 32 files changed, 129 insertions(+), 690 deletions(-) create mode 100644 internal/consent/export_test.go create mode 100644 internal/consent/testdata/TestGetConsentState/golden/valid_false_global_file create mode 100644 internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file create mode 100644 internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_false_source create mode 100644 internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_true_source delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/invalid_file_global_file delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/invalid_value_global_file delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/no_global_file delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_false_global_file delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_2_multiple_sources_(vtrue,_vfalse) delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_3_multiple_sources_(vtrue,_vfalse,_naf) delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_invalid_file_source delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_invalid_value_source delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_file_source delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_valid_false_source delete mode 100644 internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_valid_true_source diff --git a/internal/consent/consent.go b/internal/consent/consent.go index a6cc919..d13a145 100644 --- a/internal/consent/consent.go +++ b/internal/consent/consent.go @@ -19,18 +19,6 @@ type Manager struct { path string } -// States is a struct that represents the consent states for a list of sources and the global consent state. -type States struct { - SourceStates map[string]consentStateResult - GlobalState consentStateResult -} - -type consentStateResult struct { - Source string // the source for the consent state - State bool // the consent state for the source - ReadErr bool // true if there was an error reading the consent file -} - // consentFile is a struct that represents a consent file. type consentFile struct { ConsentState bool `toml:"consent_state"` @@ -42,59 +30,18 @@ func New(path string) *Manager { return &Manager{path: path} } -// GetConsentStates gets the consent state for the given sources and the global consent state. -// If any of the sources do not have a consent file, it will be considered as a false state. -// If a specified source does not have a consent file, it will not be included in the returned ConsentStates struct. -// TODO: Simplify to single source, update spec -func (cm Manager) GetConsentStates(sources []string) (consentStates States, err error) { - consentStates = States{SourceStates: make(map[string]consentStateResult)} - - sourceFiles, globalFile, err := getMatchingConsentFiles(sources, cm.path) +// GetConsentState gets the consent state for the given source. +// If the source do not have a consent file, it will be considered as a false state. +// If the source is an empty string, then the global consent state will be returned. +// If the target consent file does not exist, it will not be created. +func (cm Manager) GetConsentState(source string) (bool, error) { + sourceConsent, err := readConsentFile(cm.getConsentFile(source)) if err != nil { - slog.Error("Error getting consent files", "error", err) - return States{}, err - } - - results := make(chan consentStateResult, len(sourceFiles)) - defer close(results) - - // Global consent file - if globalFile != "" { - globalResult := make(chan consentStateResult, 1) - defer close(globalResult) - - go func() { - globalConsent, err := readConsentFile(globalFile) - if err != nil { - slog.Error("Error reading global consent file", "file", globalFile, "error", err) - globalResult <- consentStateResult{Source: "global", State: false, ReadErr: true} - return - } - globalResult <- consentStateResult{Source: "global", State: globalConsent.ConsentState, ReadErr: false} - }() - - consentStates.GlobalState = <-globalResult - } - - // Goroutine to read the consent files for each source, excluding the global consent file. - for source, filePath := range sourceFiles { - go func(source, filePath string) { - consent, err := readConsentFile(filePath) - if err != nil { - slog.Error("Error reading consent file", "source", source, "error", err) - results <- consentStateResult{Source: source, State: false, ReadErr: true} - return - } - results <- consentStateResult{Source: source, State: consent.ConsentState, ReadErr: false} - }(source, filePath) - } - - for range sourceFiles { - res := <-results - consentStates.SourceStates[res.Source] = res + slog.Error("Error reading source consent file", "source", source, "error", err) + return false, err } - return consentStates, nil + return sourceConsent.ConsentState, nil } var consentSourceFilePattern = `%s` + constants.ConsentSourceBaseSeparator + constants.GlobalFileName @@ -105,26 +52,29 @@ var consentSourceFilePattern = `%s` + constants.ConsentSourceBaseSeparator + con func (cm Manager) SetConsentState(source string, state bool) (err error) { defer decorate.OnError(&err, "could not set consent state") + consent := consentFile{ConsentState: state} + return consent.write(cm.getConsentFile(source)) +} + +// getConsentFile returns the expected path to the consent file for the given source. +// If source is blank, it returns the path to the global consent file. +// It does not check if the file exists, or if it is valid. +func (cm Manager) getConsentFile(source string) string { p := filepath.Join(cm.path, constants.GlobalFileName) if source != "" { p = filepath.Join(cm.path, fmt.Sprintf(consentSourceFilePattern, source)) } - consent := consentFile{ConsentState: state} - return consent.write(p) + return p } -// getMatchingConsentFiles returns a map of all paths to consent files matching the given sources and a path -// to the global consent file. -// If sources is empty, all consent files in the folder will be returned. -// If a source does not have a consent file, it will be represented as an empty string -// Does not traverse subdirectories. -func getMatchingConsentFiles(sources []string, folderPath string) (sourceFiles map[string]string, globalFile string, err error) { - sourceFiles = make(map[string]string) +// getSourceConsentFiles returns a map of all paths to validly named consent files in the folder, other than the global file. +func (cm Manager) getConsentFiles() (map[string]string, error) { + sourceFiles := make(map[string]string) - entries, err := os.ReadDir(folderPath) + entries, err := os.ReadDir(cm.path) if err != nil { - return sourceFiles, globalFile, err + return sourceFiles, err } for _, entry := range entries { @@ -132,40 +82,20 @@ func getMatchingConsentFiles(sources []string, folderPath string) (sourceFiles m continue } - // Global file - if entry.Name() == constants.GlobalFileName { - globalFile = filepath.Join(folderPath, entry.Name()) - slog.Debug("Found global consent file", "file", globalFile) + // Source file + if !strings.HasSuffix(entry.Name(), constants.ConsentSourceBaseSeparator+constants.GlobalFileName) { continue } - - if len(sources) == 0 { - // Source file - if !strings.HasSuffix(entry.Name(), constants.ConsentSourceBaseSeparator+constants.GlobalFileName) { - continue - } - source := strings.TrimSuffix(entry.Name(), constants.ConsentSourceBaseSeparator+constants.GlobalFileName) - sourceFiles[source] = filepath.Join(folderPath, entry.Name()) - slog.Debug("Found source consent file", "file", sourceFiles[source]) - continue - } - - for _, source := range sources { - if entry.Name() == fmt.Sprintf(consentSourceFilePattern, source) { - sourceFiles[source] = filepath.Join(folderPath, entry.Name()) - slog.Debug("Found matching source consent file", "file", sourceFiles[source]) - break - } - } + source := strings.TrimSuffix(entry.Name(), constants.ConsentSourceBaseSeparator+constants.GlobalFileName) + sourceFiles[source] = filepath.Join(cm.path, entry.Name()) + slog.Debug("Found source consent file", "file", sourceFiles[source]) } - return sourceFiles, globalFile, err + + return sourceFiles, nil } func readConsentFile(path string) (consentFile, error) { var consent consentFile - if _, err := os.Stat(path); os.IsNotExist(err) { - return &consent, nil - } _, err := toml.DecodeFile(path, &consent) slog.Debug("Read consent file", "file", path, "consent", consent.ConsentState) @@ -181,7 +111,7 @@ func (cf consentFile) write(path string) (err error) { } defer func() { _ = tmp.Close() - if err := os.Remove(tmp.Name()); err != nil { + if err := os.Remove(tmp.Name()); err != nil && !os.IsNotExist(err) { slog.Warn("Failed to remove temporary file when writing consent file", "file", tmp.Name(), "error", err) } }() diff --git a/internal/consent/consent_test.go b/internal/consent/consent_test.go index 6e858f8..a4c43d1 100644 --- a/internal/consent/consent_test.go +++ b/internal/consent/consent_test.go @@ -13,37 +13,35 @@ import ( "github.com/ubuntu/ubuntu-insights/internal/testutils" ) -// consentDir is a struct that holds a test's temporary directory and its locks. +// consentDir is a struct that holds a test's temporary directory. // It should be cleaned up after the test is done. type consentDir struct { dir string } -func TestGetConsentStates(t *testing.T) { +func TestGetConsentState(t *testing.T) { t.Parallel() tests := map[string]struct { - sources []string + source string globalFile string wantErr bool }{ - "No Global File": {}, + "No Global File": {wantErr: true}, // Global File Tests "Valid True Global File": {globalFile: "valid_true-consent.toml"}, "Valid False Global File": {globalFile: "valid_false-consent.toml"}, - "Invalid Value Global File": {globalFile: "invalid_value-consent.toml"}, - "Invalid File Global File": {globalFile: "invalid_file-consent.toml"}, + "Invalid Value Global File": {globalFile: "invalid_value-consent.toml", wantErr: true}, + "Invalid File Global File": {globalFile: "invalid_file-consent.toml", wantErr: true}, // Source Specific Tests - "Valid True Global File, Valid True Source": {globalFile: "valid_true-consent.toml", sources: []string{"valid_true"}}, - "Valid True Global File, Valid False Source": {globalFile: "valid_true-consent.toml", sources: []string{"valid_false"}}, - "Valid True Global File, Invalid Value Source": {globalFile: "valid_true-consent.toml", sources: []string{"invalid_value"}}, - "Valid True Global File, Invalid File Source": {globalFile: "valid_true-consent.toml", sources: []string{"invalid_file"}}, - "Valid True Global File, No File Source": {globalFile: "valid_true-consent.toml", sources: []string{"not_a_file"}}, - "Valid True Global File, 2 Multiple Sources (VTrue, VFalse)": {globalFile: "valid_true-consent.toml", sources: []string{"valid_true", "valid_false"}}, - "Valid True Global File, 3 Multiple Sources (VTrue, VFalse, NAF)": {globalFile: "valid_true-consent.toml", sources: []string{"valid_true", "valid_false", "not_a_file"}}, + "Valid True Global File, Valid True Source": {globalFile: "valid_true-consent.toml", source: "valid_true"}, + "Valid True Global File, Valid False Source": {globalFile: "valid_true-consent.toml", source: "valid_false"}, + "Valid True Global File, Invalid Value Source": {globalFile: "valid_true-consent.toml", source: "invalid_value", wantErr: true}, + "Valid True Global File, Invalid File Source": {globalFile: "valid_true-consent.toml", source: "invalid_file", wantErr: true}, + "Valid True Global File, No File Source": {globalFile: "valid_true-consent.toml", source: "not_a_file", wantErr: true}, } for name, tc := range tests { @@ -54,7 +52,7 @@ func TestGetConsentStates(t *testing.T) { defer cDir.cleanup(t) cm := consent.New(cDir.dir) - got, err := cm.GetConsentStates(tc.sources) + got, err := cm.GetConsentState(tc.source) if tc.wantErr { require.Error(t, err, "expected an error but got none") return @@ -62,7 +60,7 @@ func TestGetConsentStates(t *testing.T) { require.NoError(t, err, "got an unexpected error") want := testutils.LoadWithUpdateFromGoldenYAML(t, got) - require.Equal(t, want, got, "GetConsentStates should return expected consent states") + require.Equal(t, want, got, "GetConsentState should return expected consent state") }) } } @@ -71,7 +69,6 @@ func TestSetConsentStates(t *testing.T) { t.Parallel() tests := map[string]struct { - sources []string consentStates map[string]bool globalFile string @@ -100,8 +97,8 @@ func TestSetConsentStates(t *testing.T) { } type goldenFile struct { - States consent.States - FileCount uint + States map[string]bool + FileCount int } for name, tc := range tests { @@ -119,12 +116,12 @@ func TestSetConsentStates(t *testing.T) { } require.NoError(t, err, "got an unexpected error") - states, err := cm.GetConsentStates(tc.sources) + states, err := cm.GetAllSourceConsentStates(true) require.NoError(t, err, "got an unexpected error while getting consent states") d, err := os.ReadDir(cDir.dir) require.NoError(t, err, "failed to read temporary directory") - got := goldenFile{States: states, FileCount: uint(len(d))} + got := goldenFile{States: states, FileCount: len(d)} want := testutils.LoadWithUpdateFromGoldenYAML(t, got) require.Equal(t, want, got, "GetConsentStates should return expected consent states") diff --git a/internal/consent/export_test.go b/internal/consent/export_test.go new file mode 100644 index 0000000..f887256 --- /dev/null +++ b/internal/consent/export_test.go @@ -0,0 +1,26 @@ +package consent + +// GetAllSourceConsentStates gets the consent states for all sources. +// It does not get the global consent state. +// If continueOnErr is true, it will continue to the next source if an error occurs. +func (cm Manager) GetAllSourceConsentStates(continueOnErr bool) (map[string]bool, error) { + p, err := cm.getConsentFiles() + if err != nil { + return nil, err + } + + consentStates := make(map[string]bool) + for source, path := range p { + consent, err := readConsentFile(path) + if err != nil && !continueOnErr { + return nil, err + } + if err != nil { + continue + } + + consentStates[source] = consent.ConsentState + } + + return consentStates, nil +} diff --git a/internal/consent/testdata/TestGetConsentState/golden/valid_false_global_file b/internal/consent/testdata/TestGetConsentState/golden/valid_false_global_file new file mode 100644 index 0000000..c508d53 --- /dev/null +++ b/internal/consent/testdata/TestGetConsentState/golden/valid_false_global_file @@ -0,0 +1 @@ +false diff --git a/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file b/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file new file mode 100644 index 0000000..27ba77d --- /dev/null +++ b/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file @@ -0,0 +1 @@ +true diff --git a/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_false_source b/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_false_source new file mode 100644 index 0000000..c508d53 --- /dev/null +++ b/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_false_source @@ -0,0 +1 @@ +false diff --git a/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_true_source b/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_true_source new file mode 100644 index 0000000..27ba77d --- /dev/null +++ b/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_true_source @@ -0,0 +1 @@ +true diff --git a/internal/consent/testdata/TestGetConsentStates/golden/invalid_file_global_file b/internal/consent/testdata/TestGetConsentStates/golden/invalid_file_global_file deleted file mode 100644 index 2ae9869..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/invalid_file_global_file +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: false - readerr: true diff --git a/internal/consent/testdata/TestGetConsentStates/golden/invalid_value_global_file b/internal/consent/testdata/TestGetConsentStates/golden/invalid_value_global_file deleted file mode 100644 index 2ae9869..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/invalid_value_global_file +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: false - readerr: true diff --git a/internal/consent/testdata/TestGetConsentStates/golden/no_global_file b/internal/consent/testdata/TestGetConsentStates/golden/no_global_file deleted file mode 100644 index 3285f07..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/no_global_file +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: "" - state: false - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_false_global_file b/internal/consent/testdata/TestGetConsentStates/golden/valid_false_global_file deleted file mode 100644 index 8a2bbfd..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_false_global_file +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: false - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file deleted file mode 100644 index 7ff6725..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file +++ /dev/null @@ -1,29 +0,0 @@ -sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_2_multiple_sources_(vtrue,_vfalse) b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_2_multiple_sources_(vtrue,_vfalse) deleted file mode 100644 index bc8c6a4..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_2_multiple_sources_(vtrue,_vfalse) +++ /dev/null @@ -1,13 +0,0 @@ -sourcestates: - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_3_multiple_sources_(vtrue,_vfalse,_naf) b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_3_multiple_sources_(vtrue,_vfalse,_naf) deleted file mode 100644 index bc8c6a4..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_3_multiple_sources_(vtrue,_vfalse,_naf) +++ /dev/null @@ -1,13 +0,0 @@ -sourcestates: - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_invalid_file_source b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_invalid_file_source deleted file mode 100644 index 1b41f2d..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_invalid_file_source +++ /dev/null @@ -1,9 +0,0 @@ -sourcestates: - invalid_file: - source: invalid_file - state: false - readerr: true -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_invalid_value_source b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_invalid_value_source deleted file mode 100644 index 879de3f..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_invalid_value_source +++ /dev/null @@ -1,9 +0,0 @@ -sourcestates: - invalid_value: - source: invalid_value - state: false - readerr: true -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_file_source b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_file_source deleted file mode 100644 index 462e7f4..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_no_file_source +++ /dev/null @@ -1,5 +0,0 @@ -sourcestates: {} -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_valid_false_source b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_valid_false_source deleted file mode 100644 index 63ed6e9..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_valid_false_source +++ /dev/null @@ -1,9 +0,0 @@ -sourcestates: - valid_false: - source: valid_false - state: false - readerr: false -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_valid_true_source b/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_valid_true_source deleted file mode 100644 index 4add7f5..0000000 --- a/internal/consent/testdata/TestGetConsentStates/golden/valid_true_global_file,_valid_true_source +++ /dev/null @@ -1,9 +0,0 @@ -sourcestates: - valid_true: - source: valid_true - state: true - readerr: false -globalstate: - source: global - state: true - readerr: false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_false b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_false index 28a95b6..38c7a46 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_false +++ b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_false @@ -1,31 +1,6 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false - globalstate: - source: global - state: false - readerr: false + empty: false + extra_key: false + valid_false: false + valid_true: true filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_true b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_true index bbe6efd..38c7a46 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_true +++ b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_true @@ -1,31 +1,6 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false - globalstate: - source: global - state: true - readerr: false + empty: false + extra_key: false + valid_false: false + valid_true: true filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_false b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_false index d50e411..e6f5011 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_false +++ b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_false @@ -1,35 +1,7 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - new_false: - source: new_false - state: false - readerr: false - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false - globalstate: - source: "" - state: false - readerr: false + empty: false + extra_key: false + new_false: false + valid_false: false + valid_true: true filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_true b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_true index 0491904..b5c530c 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_true +++ b/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_true @@ -1,35 +1,7 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - new_true: - source: new_true - state: true - readerr: false - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false - globalstate: - source: "" - state: false - readerr: false + empty: false + extra_key: false + new_true: true + valid_false: false + valid_true: true filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_false b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_false index 28a95b6..38c7a46 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_false +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_false @@ -1,31 +1,6 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false - globalstate: - source: global - state: false - readerr: false + empty: false + extra_key: false + valid_false: false + valid_true: true filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_true b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_true index bbe6efd..38c7a46 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_true +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_true @@ -1,31 +1,6 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false - globalstate: - source: global - state: true - readerr: false + empty: false + extra_key: false + valid_false: false + valid_true: true filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_false b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_false index e97680a..20cfa22 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_false +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_false @@ -1,31 +1,6 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: false - readerr: false - globalstate: - source: global - state: true - readerr: false + empty: false + extra_key: false + valid_false: false + valid_true: false filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_true b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_true index d035451..723440d 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_true +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_true @@ -1,31 +1,6 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: true - readerr: false - valid_true: - source: valid_true - state: true - readerr: false - globalstate: - source: global - state: true - readerr: false + empty: false + extra_key: false + valid_false: true + valid_true: true filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_false b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_false index 28a95b6..38c7a46 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_false +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_false @@ -1,31 +1,6 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false - globalstate: - source: global - state: false - readerr: false + empty: false + extra_key: false + valid_false: false + valid_true: true filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_true b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_true index bbe6efd..38c7a46 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_true +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_true @@ -1,31 +1,6 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false - globalstate: - source: global - state: true - readerr: false + empty: false + extra_key: false + valid_false: false + valid_true: true filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_false b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_false index 28a95b6..38c7a46 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_false +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_false @@ -1,31 +1,6 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false - globalstate: - source: global - state: false - readerr: false + empty: false + extra_key: false + valid_false: false + valid_true: true filecount: 9 diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_true b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_true index bbe6efd..38c7a46 100644 --- a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_true +++ b/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_true @@ -1,31 +1,6 @@ states: - sourcestates: - empty: - source: empty - state: false - readerr: false - extra_key: - source: extra_key - state: false - readerr: false - invalid_file: - source: invalid_file - state: false - readerr: true - invalid_value: - source: invalid_value - state: false - readerr: true - valid_false: - source: valid_false - state: false - readerr: false - valid_true: - source: valid_true - state: true - readerr: false - globalstate: - source: global - state: true - readerr: false + empty: false + extra_key: false + valid_false: false + valid_true: true filecount: 9 diff --git a/internal/constants/constants.go b/internal/constants/constants.go index b1ec1e5..3210a80 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -24,7 +24,7 @@ const ( // ConsentSourceBaseSeparator is the default separator between the source and the base name of the consent state files. ConsentSourceBaseSeparator = "-" - // Report extension is the default extension for the report files. + // ReportExtension is the default extension for the report files. ReportExtension = ".json" ) From 09516508443d6c2aaa1e9d504455bf29ac0e890b Mon Sep 17 00:00:00 2001 From: Trevor Shoe Date: Thu, 16 Jan 2025 13:17:14 -0500 Subject: [PATCH 13/26] Add mac and windows sysinfo bones --- internal/collector/sysinfo/sysinfo_darwin.go | 9 +++++++++ internal/collector/sysinfo/sysinfo_windows.go | 9 +++++++++ 2 files changed, 18 insertions(+) create mode 100644 internal/collector/sysinfo/sysinfo_darwin.go create mode 100644 internal/collector/sysinfo/sysinfo_windows.go diff --git a/internal/collector/sysinfo/sysinfo_darwin.go b/internal/collector/sysinfo/sysinfo_darwin.go new file mode 100644 index 0000000..08b8dca --- /dev/null +++ b/internal/collector/sysinfo/sysinfo_darwin.go @@ -0,0 +1,9 @@ +package sysinfo + +func (s Manager) collectHardware() (HwInfo, error) { + return HwInfo{}, nil +} + +func (s Manager) collectSoftware() (SwInfo, error) { + return SwInfo{}, nil +} diff --git a/internal/collector/sysinfo/sysinfo_windows.go b/internal/collector/sysinfo/sysinfo_windows.go new file mode 100644 index 0000000..08b8dca --- /dev/null +++ b/internal/collector/sysinfo/sysinfo_windows.go @@ -0,0 +1,9 @@ +package sysinfo + +func (s Manager) collectHardware() (HwInfo, error) { + return HwInfo{}, nil +} + +func (s Manager) collectSoftware() (SwInfo, error) { + return SwInfo{}, nil +} From 5b1c9f7e10b09c07fc3eef7e6fd3742573a2e290 Mon Sep 17 00:00:00 2001 From: Trevor Shoe Date: Thu, 16 Jan 2025 13:31:55 -0500 Subject: [PATCH 14/26] Apply linting --- internal/collector/sysinfo/sysinfo.go | 2 +- internal/collector/sysinfo/sysinfo_linux.go | 2 -- internal/collector/sysinfo/sysinfo_linux_test.go | 3 +-- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/internal/collector/sysinfo/sysinfo.go b/internal/collector/sysinfo/sysinfo.go index d57a211..82e1369 100644 --- a/internal/collector/sysinfo/sysinfo.go +++ b/internal/collector/sysinfo/sysinfo.go @@ -1,4 +1,4 @@ -// package sysinfo allows collecting "common" system information for all insight reports. +// Package sysinfo allows collecting "common" system information for all insight reports. package sysinfo type options struct { diff --git a/internal/collector/sysinfo/sysinfo_linux.go b/internal/collector/sysinfo/sysinfo_linux.go index 0562562..915425e 100644 --- a/internal/collector/sysinfo/sysinfo_linux.go +++ b/internal/collector/sysinfo/sysinfo_linux.go @@ -6,7 +6,6 @@ import ( ) func (s Manager) collectHardware() (hwInfo HwInfo, err error) { - // System vendor d, err := os.ReadFile(filepath.Join(s.root, "sys/class/dmi/id/sys_vendor")) if err != nil { @@ -20,6 +19,5 @@ func (s Manager) collectHardware() (hwInfo HwInfo, err error) { } func (s Manager) collectSoftware() (swInfo SwInfo, err error) { - return swInfo, nil } diff --git a/internal/collector/sysinfo/sysinfo_linux_test.go b/internal/collector/sysinfo/sysinfo_linux_test.go index c742f6d..5a3c8ce 100644 --- a/internal/collector/sysinfo/sysinfo_linux_test.go +++ b/internal/collector/sysinfo/sysinfo_linux_test.go @@ -16,7 +16,7 @@ func TestNew(t *testing.T) { }{ "Instantiate a sys info manager": {}, } - for name, _ := range tests { + for name := range tests { t.Run(name, func(t *testing.T) { t.Parallel() @@ -56,5 +56,4 @@ func TestCollect(t *testing.T) { require.Equal(t, want, got, "Collect should return expected sys information") }) } - } From 7427c2c3ee241ddba7f22f71ae935f0d4a4caaf0 Mon Sep 17 00:00:00 2001 From: kat <28567881+hk21702@users.noreply.github.com> Date: Tue, 21 Jan 2025 17:45:09 -0500 Subject: [PATCH 15/26] Add file utility functions for atomic writing and existence checking; enhance report retrieval logic --- internal/constants/constants.go | 3 + internal/fileutils/fileutils.go | 48 ++++++++ internal/fileutils/fileutils_test.go | 103 ++++++++++++++++++ internal/reportutils/reportutils.go | 37 +++++++ internal/reportutils/reportutils_test.go | 44 ++++++++ .../TestGetReports/golden/empty_directory | 1 + .../TestGetReports/golden/files_in_subdir | 1 + .../golden/get_newest_of_period | 1 + .../golden/invalid_file_extension | 1 + .../TestGetReports/golden/invalid_file_names | 1 + .../golden/mix_of_valid_and_invalid | 1 + .../golden/multiple_consequtive_windows | 3 + .../golden/multiple_non-consequtive_windows | 3 + 13 files changed, 247 insertions(+) create mode 100644 internal/fileutils/fileutils.go create mode 100644 internal/fileutils/fileutils_test.go create mode 100644 internal/reportutils/testdata/TestGetReports/golden/empty_directory create mode 100644 internal/reportutils/testdata/TestGetReports/golden/files_in_subdir create mode 100644 internal/reportutils/testdata/TestGetReports/golden/get_newest_of_period create mode 100644 internal/reportutils/testdata/TestGetReports/golden/invalid_file_extension create mode 100644 internal/reportutils/testdata/TestGetReports/golden/invalid_file_names create mode 100644 internal/reportutils/testdata/TestGetReports/golden/mix_of_valid_and_invalid create mode 100644 internal/reportutils/testdata/TestGetReports/golden/multiple_consequtive_windows create mode 100644 internal/reportutils/testdata/TestGetReports/golden/multiple_non-consequtive_windows diff --git a/internal/constants/constants.go b/internal/constants/constants.go index 448720d..b5d600c 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -26,6 +26,9 @@ const ( // ReportExt is the default extension for the report files. ReportExt = ".json" + + // OptOutJSON is the data sent in case of Opt-Out choice. + OptOutJSON = `{"OptOut": true}` ) type options struct { diff --git a/internal/fileutils/fileutils.go b/internal/fileutils/fileutils.go new file mode 100644 index 0000000..0a118d3 --- /dev/null +++ b/internal/fileutils/fileutils.go @@ -0,0 +1,48 @@ +// Package fileutils provides utility functions for handling files. +package fileutils + +import ( + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" +) + +// AtomicWrite writes data to a file atomically. +// If the file already exists, then it will be overwritten. +// Not atomic on Windows. +func AtomicWrite(path string, data []byte) error { + tmp, err := os.CreateTemp(filepath.Dir(path), "consent-*.tmp") + if err != nil { + return fmt.Errorf("could not create temporary file: %v", err) + } + defer func() { + _ = tmp.Close() + if err := os.Remove(tmp.Name()); err != nil && !os.IsNotExist(err) { + slog.Warn("Failed to remove temporary file", "file", tmp.Name(), "error", err) + } + }() + + if _, err := tmp.Write(data); err != nil { + return fmt.Errorf("could not write to temporary file: %v", err) + } + + if err := tmp.Close(); err != nil { + return fmt.Errorf("could not close temporary file: %v", err) + } + + if err := os.Rename(tmp.Name(), path); err != nil { + return fmt.Errorf("could not rename temporary file: %v", err) + } + return nil +} + +// FileExists checks if a file exists at the given path. +func FileExists(path string) (bool, error) { + _, err := os.Stat(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return false, err + } + return !errors.Is(err, os.ErrNotExist), nil +} diff --git a/internal/fileutils/fileutils_test.go b/internal/fileutils/fileutils_test.go new file mode 100644 index 0000000..fc05209 --- /dev/null +++ b/internal/fileutils/fileutils_test.go @@ -0,0 +1,103 @@ +package fileutils_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "github.com/ubuntu/ubuntu-insights/internal/fileutils" +) + +func TestAtomicWrite(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + data []byte + fileExists bool + + wantError bool + }{ + "Empty file": {data: []byte{}, fileExists: false}, + "Non-empty file": {data: []byte("data"), fileExists: false}, + "Override file": {data: []byte("data"), fileExists: true}, + "Override empty file": {data: []byte{}, fileExists: true}, + + "Existing empty file": {data: []byte{}, fileExists: true}, + "Existing non-empty file": {data: []byte("data"), fileExists: true}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + oldFile := []byte("Old File!") + tempDir := t.TempDir() + path := filepath.Join(tempDir, "file") + if tc.fileExists { + err := fileutils.AtomicWrite(path, oldFile) + require.NoError(t, err, "Setup: AtomicWrite should not return an error") + } + + err := fileutils.AtomicWrite(path, tc.data) + if tc.wantError { + require.Error(t, err, "AtomicWrite should return an error") + + // Check that the file was not overwritten + data, err := os.ReadFile(path) + require.NoError(t, err, "ReadFile should not return an error") + require.Equal(t, oldFile, data, "AtomicWrite should not overwrite the file") + } else { + require.NoError(t, err, "AtomicWrite should not return an error") + + // Check that the file was written + data, err := os.ReadFile(path) + require.NoError(t, err, "ReadFile should not return an error") + require.Equal(t, tc.data, data, "AtomicWrite should write the data to the file") + } + }) + } +} + +func TestFileExists(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + fileExists bool + parentDirIsFile bool + + wantExists bool + wantError bool + }{ + "Returns_true_when_file_exists": {fileExists: true, wantExists: true}, + "Returns_false_when_file_does_not_exist": {fileExists: false, wantExists: false}, + "Returns_false_when_parent_directory_does_not_exist": {fileExists: false, wantExists: false}, + + "Error_when_parent_directory_is_a_file": {parentDirIsFile: true, wantError: true}, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + tempDir := t.TempDir() + path := filepath.Join(tempDir, "file") + if tc.fileExists { + err := fileutils.AtomicWrite(path, []byte{}) + require.NoError(t, err, "AtomicWrite should not return an error") + } + if tc.parentDirIsFile { + path = filepath.Join(tempDir, "file", "file") + err := fileutils.AtomicWrite(filepath.Join(tempDir, "file"), []byte{}) + require.NoError(t, err, "AtomicWrite should not return an error") + } + + exists, err := fileutils.FileExists(path) + if tc.wantError { + require.Error(t, err, "FileExists should return an error") + } else { + require.NoError(t, err, "FileExists should not return an error") + } + require.Equal(t, tc.wantExists, exists, "FileExists should return the expected result") + }) + } +} diff --git a/internal/reportutils/reportutils.go b/internal/reportutils/reportutils.go index 615ceb6..307f7e0 100644 --- a/internal/reportutils/reportutils.go +++ b/internal/reportutils/reportutils.go @@ -75,3 +75,40 @@ func GetReportPath(dir string, time int64, period int) (string, error) { return mostRecentReportPath, nil } + +// GetReports returns the paths for the latest report within each period window for a given directory. +// The key of the map is the start of the period window, and the report timestamp. +func GetReports(dir string, period int) (map[int64]int64, error) { + if period <= 0 { + return nil, ErrInvalidPeriod + } + + // Get the most recent report within each period window. + files, err := os.ReadDir(dir) + if err != nil { + slog.Error("Failed to read directory", "directory", dir, "error", err) + return nil, err + } + + // Map to store the most recent report within each period window. + reports := make(map[int64]int64) + for _, file := range files { + if filepath.Ext(file.Name()) != constants.ReportExt { + slog.Info("Skipping non-report file, invalid extension", "file", file.Name()) + continue + } + + reportTime, err := GetReportTime(file.Name()) + if err != nil { + slog.Info("Skipping non-report file, invalid file name", "file", file.Name()) + continue + } + + periodStart := reportTime - (reportTime % int64(period)) + if existingReport, ok := reports[periodStart]; !ok || existingReport < reportTime { + reports[periodStart] = reportTime + } + } + + return reports, nil +} diff --git a/internal/reportutils/reportutils_test.go b/internal/reportutils/reportutils_test.go index 3216cbb..1f4a2d5 100644 --- a/internal/reportutils/reportutils_test.go +++ b/internal/reportutils/reportutils_test.go @@ -154,3 +154,47 @@ func setupTmpDir(t *testing.T, files []string, subDir string, subDirFiles []stri return dir, nil } + +func TestGetReports(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + files []string + subDir string + subDirFiles []string + period int + + wantErr error + }{ + "Empty Directory": {period: 500}, + "Files in subDir": {subDir: "subdir", subDirFiles: []string{"1.json", "2.json"}, period: 500}, + + "Invalid File Extension": {files: []string{"1.txt", "2.txt"}, period: 500}, + "Invalid File Names": {files: []string{"i-1.json", "i-2.json", "i-3.json", "test.json", "one.json"}, period: 500}, + "Mix of Valid and Invalid": {files: []string{"1.json", "2.json", "i-1.json", "i-2.json", "i-3.json", "test.json", "five.json"}, period: 500}, + + "Get Newest of Period": {files: []string{"1.json", "7.json"}, period: 100}, + "Multiple Consecutive Windows": {files: []string{"1.json", "7.json", "101.json", "107.json", "201.json", "207.json"}, period: 100}, + "Multiple Non-Consecutive Windows": {files: []string{"1.json", "7.json", "101.json", "107.json", "251.json", "257.json"}, period: 50}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + dir, err := setupTmpDir(t, tc.files, tc.subDir, tc.subDirFiles) + require.NoError(t, err, "Setup: failed to setup temporary directory") + defer os.RemoveAll(dir) + + got, err := reportutils.GetReports(dir, tc.period) + if tc.wantErr != nil { + require.ErrorIs(t, err, tc.wantErr) + return + } + require.NoError(t, err, "got an unexpected error") + + want := testutils.LoadWithUpdateFromGoldenYAML(t, got) + require.Equal(t, want, got, "GetReports should return the most recent report within each period window") + }) + } +} diff --git a/internal/reportutils/testdata/TestGetReports/golden/empty_directory b/internal/reportutils/testdata/TestGetReports/golden/empty_directory new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/internal/reportutils/testdata/TestGetReports/golden/empty_directory @@ -0,0 +1 @@ +{} diff --git a/internal/reportutils/testdata/TestGetReports/golden/files_in_subdir b/internal/reportutils/testdata/TestGetReports/golden/files_in_subdir new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/internal/reportutils/testdata/TestGetReports/golden/files_in_subdir @@ -0,0 +1 @@ +{} diff --git a/internal/reportutils/testdata/TestGetReports/golden/get_newest_of_period b/internal/reportutils/testdata/TestGetReports/golden/get_newest_of_period new file mode 100644 index 0000000..017ba7a --- /dev/null +++ b/internal/reportutils/testdata/TestGetReports/golden/get_newest_of_period @@ -0,0 +1 @@ +0: 7 diff --git a/internal/reportutils/testdata/TestGetReports/golden/invalid_file_extension b/internal/reportutils/testdata/TestGetReports/golden/invalid_file_extension new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/internal/reportutils/testdata/TestGetReports/golden/invalid_file_extension @@ -0,0 +1 @@ +{} diff --git a/internal/reportutils/testdata/TestGetReports/golden/invalid_file_names b/internal/reportutils/testdata/TestGetReports/golden/invalid_file_names new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/internal/reportutils/testdata/TestGetReports/golden/invalid_file_names @@ -0,0 +1 @@ +{} diff --git a/internal/reportutils/testdata/TestGetReports/golden/mix_of_valid_and_invalid b/internal/reportutils/testdata/TestGetReports/golden/mix_of_valid_and_invalid new file mode 100644 index 0000000..f196b73 --- /dev/null +++ b/internal/reportutils/testdata/TestGetReports/golden/mix_of_valid_and_invalid @@ -0,0 +1 @@ +0: 2 diff --git a/internal/reportutils/testdata/TestGetReports/golden/multiple_consequtive_windows b/internal/reportutils/testdata/TestGetReports/golden/multiple_consequtive_windows new file mode 100644 index 0000000..8e9a957 --- /dev/null +++ b/internal/reportutils/testdata/TestGetReports/golden/multiple_consequtive_windows @@ -0,0 +1,3 @@ +0: 7 +100: 107 +200: 207 diff --git a/internal/reportutils/testdata/TestGetReports/golden/multiple_non-consequtive_windows b/internal/reportutils/testdata/TestGetReports/golden/multiple_non-consequtive_windows new file mode 100644 index 0000000..f2cb19d --- /dev/null +++ b/internal/reportutils/testdata/TestGetReports/golden/multiple_non-consequtive_windows @@ -0,0 +1,3 @@ +0: 7 +100: 107 +250: 257 From 64e217841049798c7d5c15d06576ce4935dccad3 Mon Sep 17 00:00:00 2001 From: kat <28567881+hk21702@users.noreply.github.com> Date: Tue, 21 Jan 2025 18:06:23 -0500 Subject: [PATCH 16/26] Initial uploader --- cmd/insights/commands/upload.go | 3 +- internal/constants/constants.go | 15 +- internal/fileutils/fileutils.go | 12 +- internal/fileutils/fileutils_test.go | 73 ++---- internal/reportutils/reportutils.go | 113 ++++++--- internal/reportutils/reportutils_test.go | 38 +++ .../TestGetAllReports/golden/empty_directory | 1 + .../TestGetAllReports/golden/files_in_subdir | 2 + .../golden/invalid_file_extension | 1 + .../golden/invalid_file_names | 1 + .../golden/mix_of_valid_and_invalid | 3 + .../TestGetReports/golden/get_all_reports | 7 + .../golden/multiple_consecutive_windows | 3 + .../golden/multiple_non-consecutive_windows | 3 + internal/uploader/export_test.go | 17 ++ internal/uploader/internal_test.go | 1 + .../testdata/TestUpload/golden/bad_content | 1 + .../testdata/TestUpload/golden/bad_response | 1 + .../testdata/TestUpload/golden/bad_url | 1 + .../TestUpload/golden/consent_manager_false | 1 + .../golden/consent_manager_global_error | 1 + .../consent_manager_global_error_with_true | 1 + .../consent_manager_global_false,_source_true | 1 + .../consent_manager_global_true,_source_false | 1 + .../golden/consent_manager_source_error | 1 + .../consent_manager_source_error_with_true | 1 + .../testdata/TestUpload/golden/dry_run | 1 + .../TestUpload/golden/duplicate_upload | 2 + .../TestUpload/golden/future_timestamp | 2 + .../testdata/TestUpload/golden/min_age | 2 + .../testdata/TestUpload/golden/multi_upload | 2 + .../testdata/TestUpload/golden/no_reports | 1 + .../TestUpload/golden/no_reports_with_dummy | 33 +++ .../testdata/TestUpload/golden/offline_server | 1 + .../testdata/TestUpload/golden/single_upload | 1 + .../uploader/testdata/test_source/-1.json | 1 + .../uploader/testdata/test_source/dummy.json | 3 + .../testdata/test_source/dummy/dummy.json | 3 + .../test_source/dummy/empty-dummy.json | 0 .../testdata/test_source/empty-dummy.json | 0 internal/uploader/upload.go | 175 ++++++++++++++ internal/uploader/uploader.go | 85 +++++++ internal/uploader/uploader_test.go | 226 ++++++++++++++++++ 43 files changed, 746 insertions(+), 95 deletions(-) create mode 100644 internal/reportutils/testdata/TestGetAllReports/golden/empty_directory create mode 100644 internal/reportutils/testdata/TestGetAllReports/golden/files_in_subdir create mode 100644 internal/reportutils/testdata/TestGetAllReports/golden/invalid_file_extension create mode 100644 internal/reportutils/testdata/TestGetAllReports/golden/invalid_file_names create mode 100644 internal/reportutils/testdata/TestGetAllReports/golden/mix_of_valid_and_invalid create mode 100644 internal/reportutils/testdata/TestGetReports/golden/get_all_reports create mode 100644 internal/reportutils/testdata/TestGetReports/golden/multiple_consecutive_windows create mode 100644 internal/reportutils/testdata/TestGetReports/golden/multiple_non-consecutive_windows create mode 100644 internal/uploader/internal_test.go create mode 100644 internal/uploader/testdata/TestUpload/golden/bad_content create mode 100644 internal/uploader/testdata/TestUpload/golden/bad_response create mode 100644 internal/uploader/testdata/TestUpload/golden/bad_url create mode 100644 internal/uploader/testdata/TestUpload/golden/consent_manager_false create mode 100644 internal/uploader/testdata/TestUpload/golden/consent_manager_global_error create mode 100644 internal/uploader/testdata/TestUpload/golden/consent_manager_global_error_with_true create mode 100644 internal/uploader/testdata/TestUpload/golden/consent_manager_global_false,_source_true create mode 100644 internal/uploader/testdata/TestUpload/golden/consent_manager_global_true,_source_false create mode 100644 internal/uploader/testdata/TestUpload/golden/consent_manager_source_error create mode 100644 internal/uploader/testdata/TestUpload/golden/consent_manager_source_error_with_true create mode 100644 internal/uploader/testdata/TestUpload/golden/dry_run create mode 100644 internal/uploader/testdata/TestUpload/golden/duplicate_upload create mode 100644 internal/uploader/testdata/TestUpload/golden/future_timestamp create mode 100644 internal/uploader/testdata/TestUpload/golden/min_age create mode 100644 internal/uploader/testdata/TestUpload/golden/multi_upload create mode 100644 internal/uploader/testdata/TestUpload/golden/no_reports create mode 100644 internal/uploader/testdata/TestUpload/golden/no_reports_with_dummy create mode 100644 internal/uploader/testdata/TestUpload/golden/offline_server create mode 100644 internal/uploader/testdata/TestUpload/golden/single_upload create mode 100644 internal/uploader/testdata/test_source/-1.json create mode 100644 internal/uploader/testdata/test_source/dummy.json create mode 100644 internal/uploader/testdata/test_source/dummy/dummy.json create mode 100644 internal/uploader/testdata/test_source/dummy/empty-dummy.json create mode 100644 internal/uploader/testdata/test_source/empty-dummy.json create mode 100644 internal/uploader/upload.go create mode 100644 internal/uploader/uploader_test.go diff --git a/cmd/insights/commands/upload.go b/cmd/insights/commands/upload.go index 6f680c3..aae5175 100644 --- a/cmd/insights/commands/upload.go +++ b/cmd/insights/commands/upload.go @@ -4,6 +4,7 @@ import ( "log/slog" "github.com/spf13/cobra" + "github.com/ubuntu/ubuntu-insights/internal/constants" ) type uploadConfig struct { @@ -16,7 +17,7 @@ type uploadConfig struct { var defaultUploadConfig = uploadConfig{ sources: []string{""}, - server: "https://metrics.ubuntu.com", + server: constants.DefaultServerURL, minAge: 604800, force: false, dryRun: false, diff --git a/internal/constants/constants.go b/internal/constants/constants.go index b5d600c..670063f 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -18,6 +18,15 @@ const ( // DefaultLogLevel is the default log level selected without any verbosity flags. DefaultLogLevel = slog.LevelInfo + // DefaultServerURL is the default base URL for the server that reports are uploaded to. + DefaultServerURL = "https://metrics.ubuntu.com" + + // LocalFolder is the default name of the local collected reports folder. + LocalFolder = "local" + + // UploadedFolder is the default name of the uploaded reports folder. + UploadedFolder = "uploaded" + // GlobalFileName is the default base name of the consent state files. GlobalFileName = "consent.toml" @@ -26,11 +35,11 @@ const ( // ReportExt is the default extension for the report files. ReportExt = ".json" - - // OptOutJSON is the data sent in case of Opt-Out choice. - OptOutJSON = `{"OptOut": true}` ) +// OptOutJSON is the data sent in case of Opt-Out choice. +var OptOutJSON = struct{ OptOut bool }{OptOut: true} + type options struct { baseDir func() (string, error) } diff --git a/internal/fileutils/fileutils.go b/internal/fileutils/fileutils.go index 0a118d3..b113a2f 100644 --- a/internal/fileutils/fileutils.go +++ b/internal/fileutils/fileutils.go @@ -2,7 +2,6 @@ package fileutils import ( - "errors" "fmt" "log/slog" "os" @@ -13,7 +12,7 @@ import ( // If the file already exists, then it will be overwritten. // Not atomic on Windows. func AtomicWrite(path string, data []byte) error { - tmp, err := os.CreateTemp(filepath.Dir(path), "consent-*.tmp") + tmp, err := os.CreateTemp(filepath.Dir(path), "tmp-*.tmp") if err != nil { return fmt.Errorf("could not create temporary file: %v", err) } @@ -37,12 +36,3 @@ func AtomicWrite(path string, data []byte) error { } return nil } - -// FileExists checks if a file exists at the given path. -func FileExists(path string) (bool, error) { - _, err := os.Stat(path) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return false, err - } - return !errors.Is(err, os.ErrNotExist), nil -} diff --git a/internal/fileutils/fileutils_test.go b/internal/fileutils/fileutils_test.go index fc05209..15c2b96 100644 --- a/internal/fileutils/fileutils_test.go +++ b/internal/fileutils/fileutils_test.go @@ -15,16 +15,19 @@ func TestAtomicWrite(t *testing.T) { tests := map[string]struct { data []byte fileExists bool + invalidDir bool wantError bool }{ - "Empty file": {data: []byte{}, fileExists: false}, - "Non-empty file": {data: []byte("data"), fileExists: false}, + "Empty file": {data: []byte{}}, + "Non-empty file": {data: []byte("data")}, "Override file": {data: []byte("data"), fileExists: true}, "Override empty file": {data: []byte{}, fileExists: true}, "Existing empty file": {data: []byte{}, fileExists: true}, "Existing non-empty file": {data: []byte("data"), fileExists: true}, + + "Invalid Dir": {data: []byte("data"), invalidDir: true, wantError: true}, } for name, tc := range tests { @@ -34,6 +37,10 @@ func TestAtomicWrite(t *testing.T) { oldFile := []byte("Old File!") tempDir := t.TempDir() path := filepath.Join(tempDir, "file") + if tc.invalidDir { + path = filepath.Join(path, "fake_dir") + } + if tc.fileExists { err := fileutils.AtomicWrite(path, oldFile) require.NoError(t, err, "Setup: AtomicWrite should not return an error") @@ -44,60 +51,26 @@ func TestAtomicWrite(t *testing.T) { require.Error(t, err, "AtomicWrite should return an error") // Check that the file was not overwritten - data, err := os.ReadFile(path) - require.NoError(t, err, "ReadFile should not return an error") - require.Equal(t, oldFile, data, "AtomicWrite should not overwrite the file") - } else { - require.NoError(t, err, "AtomicWrite should not return an error") + if !tc.fileExists { + return + } + + if tc.invalidDir { + path = filepath.Dir(path) + } - // Check that the file was written data, err := os.ReadFile(path) require.NoError(t, err, "ReadFile should not return an error") - require.Equal(t, tc.data, data, "AtomicWrite should write the data to the file") - } - }) - } -} - -func TestFileExists(t *testing.T) { - t.Parallel() - - tests := map[string]struct { - fileExists bool - parentDirIsFile bool - - wantExists bool - wantError bool - }{ - "Returns_true_when_file_exists": {fileExists: true, wantExists: true}, - "Returns_false_when_file_does_not_exist": {fileExists: false, wantExists: false}, - "Returns_false_when_parent_directory_does_not_exist": {fileExists: false, wantExists: false}, - - "Error_when_parent_directory_is_a_file": {parentDirIsFile: true, wantError: true}, - } - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - t.Parallel() + require.Equal(t, oldFile, data, "AtomicWrite should not overwrite the file") - tempDir := t.TempDir() - path := filepath.Join(tempDir, "file") - if tc.fileExists { - err := fileutils.AtomicWrite(path, []byte{}) - require.NoError(t, err, "AtomicWrite should not return an error") - } - if tc.parentDirIsFile { - path = filepath.Join(tempDir, "file", "file") - err := fileutils.AtomicWrite(filepath.Join(tempDir, "file"), []byte{}) - require.NoError(t, err, "AtomicWrite should not return an error") + return } + require.NoError(t, err, "AtomicWrite should not return an error") - exists, err := fileutils.FileExists(path) - if tc.wantError { - require.Error(t, err, "FileExists should return an error") - } else { - require.NoError(t, err, "FileExists should not return an error") - } - require.Equal(t, tc.wantExists, exists, "FileExists should return the expected result") + // Check that the file was written + data, err := os.ReadFile(path) + require.NoError(t, err, "ReadFile should not return an error") + require.Equal(t, tc.data, data, "AtomicWrite should write the data to the file") }) } } diff --git a/internal/reportutils/reportutils.go b/internal/reportutils/reportutils.go index 307f7e0..f08a695 100644 --- a/internal/reportutils/reportutils.go +++ b/internal/reportutils/reportutils.go @@ -45,32 +45,41 @@ func GetReportPath(dir string, time int64, period int) (string, error) { // Reports names are utc timestamps. Get the most recent report within the period window. var mostRecentReportPath string - files, err := os.ReadDir(dir) - if err != nil { - slog.Error("Failed to read directory", "directory", dir, "error", err) - return "", err - } + err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err != nil { + slog.Error("Failed to access path", "path", path, "error", err) + return err + } - for _, file := range files { - if filepath.Ext(file.Name()) != constants.ReportExt { - slog.Info("Skipping non-report file, invalid extension", "file", file.Name()) - continue + // Skip subdirectories. + if d.IsDir() && path != dir { + return filepath.SkipDir } - reportTime, err := GetReportTime(file.Name()) + if filepath.Ext(d.Name()) != constants.ReportExt { + slog.Info("Skipping non-report file, invalid extension", "file", d.Name()) + return nil + } + + reportTime, err := GetReportTime(d.Name()) if err != nil { - slog.Info("Skipping non-report file, invalid file name", "file", file.Name()) - continue + slog.Info("Skipping non-report file, invalid file name", "file", d.Name()) + return nil } if reportTime < periodStart { - continue + return nil } if reportTime >= periodEnd { - break + return filepath.SkipDir } - mostRecentReportPath = filepath.Join(dir, file.Name()) + mostRecentReportPath = path + return nil + }) + + if err != nil { + return "", err } return mostRecentReportPath, nil @@ -78,36 +87,80 @@ func GetReportPath(dir string, time int64, period int) (string, error) { // GetReports returns the paths for the latest report within each period window for a given directory. // The key of the map is the start of the period window, and the report timestamp. +// +// If period is 1, then all reports in the dir are returned. func GetReports(dir string, period int) (map[int64]int64, error) { if period <= 0 { return nil, ErrInvalidPeriod } - // Get the most recent report within each period window. - files, err := os.ReadDir(dir) - if err != nil { - slog.Error("Failed to read directory", "directory", dir, "error", err) - return nil, err - } - - // Map to store the most recent report within each period window. reports := make(map[int64]int64) - for _, file := range files { - if filepath.Ext(file.Name()) != constants.ReportExt { - slog.Info("Skipping non-report file, invalid extension", "file", file.Name()) - continue + err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err != nil { + slog.Error("Failed to access path", "path", path, "error", err) + return err } - reportTime, err := GetReportTime(file.Name()) + if d.IsDir() && path != dir { + return filepath.SkipDir + } + + if filepath.Ext(d.Name()) != constants.ReportExt { + slog.Info("Skipping non-report file, invalid extension", "file", d.Name()) + return nil + } + + reportTime, err := GetReportTime(d.Name()) if err != nil { - slog.Info("Skipping non-report file, invalid file name", "file", file.Name()) - continue + slog.Info("Skipping non-report file, invalid file name", "file", d.Name()) + return nil } periodStart := reportTime - (reportTime % int64(period)) if existingReport, ok := reports[periodStart]; !ok || existingReport < reportTime { reports[periodStart] = reportTime } + + return nil + }) + + if err != nil { + return nil, err + } + + return reports, nil +} + +// GetAllReports returns the filename for all reports within a given directory, which match the expected pattern. +// Does not traverse subdirectories. +func GetAllReports(dir string) ([]string, error) { + reports := make([]string, 0) + err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err != nil { + slog.Error("Failed to access path", "path", path, "error", err) + return err + } + + if d.IsDir() && path != dir { + return filepath.SkipDir + } + + if filepath.Ext(d.Name()) != constants.ReportExt { + slog.Info("Skipping non-report file, invalid extension", "file", d.Name()) + return nil + } + + if _, err := GetReportTime(d.Name()); err != nil { + slog.Info("Skipping non-report file, invalid file name", "file", d.Name()) + return nil + } + + reports = append(reports, d.Name()) + return nil + }) + + if err != nil { + return nil, err } return reports, nil diff --git a/internal/reportutils/reportutils_test.go b/internal/reportutils/reportutils_test.go index 1f4a2d5..871eb21 100644 --- a/internal/reportutils/reportutils_test.go +++ b/internal/reportutils/reportutils_test.go @@ -176,6 +176,7 @@ func TestGetReports(t *testing.T) { "Get Newest of Period": {files: []string{"1.json", "7.json"}, period: 100}, "Multiple Consecutive Windows": {files: []string{"1.json", "7.json", "101.json", "107.json", "201.json", "207.json"}, period: 100}, "Multiple Non-Consecutive Windows": {files: []string{"1.json", "7.json", "101.json", "107.json", "251.json", "257.json"}, period: 50}, + "Get All Reports": {files: []string{"1.json", "2.json", "3.json", "101.json", "107.json", "251.json", "257.json"}, period: 1}, } for name, tc := range tests { @@ -198,3 +199,40 @@ func TestGetReports(t *testing.T) { }) } } + +func TestGetAllReports(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + files []string + subDir string + subDirFiles []string + + wantErr error + }{ + "Empty Directory": {}, + "Files in subDir": {files: []string{"1.json", "2.json"}, subDir: "subdir", subDirFiles: []string{"1.json", "2.json"}}, + "Invalid File Extension": {files: []string{"1.txt", "2.txt"}}, + "Invalid File Names": {files: []string{"i-1.json", "i-2.json", "i-3.json", "test.json", "one.json"}}, + "Mix of Valid and Invalid": {files: []string{"1.json", "2.json", "500.json", "i-1.json", "i-2.json", "i-3.json", "test.json", "five.json"}}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + dir, err := setupTmpDir(t, tc.files, tc.subDir, tc.subDirFiles) + require.NoError(t, err, "Setup: failed to setup temporary directory") + defer os.RemoveAll(dir) + + got, err := reportutils.GetAllReports(dir) + if tc.wantErr != nil { + require.ErrorIs(t, err, tc.wantErr) + return + } + require.NoError(t, err, "got an unexpected error") + + want := testutils.LoadWithUpdateFromGoldenYAML(t, got) + require.Equal(t, want, got, "GetAllReports should return all reports in the directory") + }) + } +} diff --git a/internal/reportutils/testdata/TestGetAllReports/golden/empty_directory b/internal/reportutils/testdata/TestGetAllReports/golden/empty_directory new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/internal/reportutils/testdata/TestGetAllReports/golden/empty_directory @@ -0,0 +1 @@ +[] diff --git a/internal/reportutils/testdata/TestGetAllReports/golden/files_in_subdir b/internal/reportutils/testdata/TestGetAllReports/golden/files_in_subdir new file mode 100644 index 0000000..6548887 --- /dev/null +++ b/internal/reportutils/testdata/TestGetAllReports/golden/files_in_subdir @@ -0,0 +1,2 @@ +- 1.json +- 2.json diff --git a/internal/reportutils/testdata/TestGetAllReports/golden/invalid_file_extension b/internal/reportutils/testdata/TestGetAllReports/golden/invalid_file_extension new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/internal/reportutils/testdata/TestGetAllReports/golden/invalid_file_extension @@ -0,0 +1 @@ +[] diff --git a/internal/reportutils/testdata/TestGetAllReports/golden/invalid_file_names b/internal/reportutils/testdata/TestGetAllReports/golden/invalid_file_names new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/internal/reportutils/testdata/TestGetAllReports/golden/invalid_file_names @@ -0,0 +1 @@ +[] diff --git a/internal/reportutils/testdata/TestGetAllReports/golden/mix_of_valid_and_invalid b/internal/reportutils/testdata/TestGetAllReports/golden/mix_of_valid_and_invalid new file mode 100644 index 0000000..a8eb048 --- /dev/null +++ b/internal/reportutils/testdata/TestGetAllReports/golden/mix_of_valid_and_invalid @@ -0,0 +1,3 @@ +- 1.json +- 2.json +- 500.json diff --git a/internal/reportutils/testdata/TestGetReports/golden/get_all_reports b/internal/reportutils/testdata/TestGetReports/golden/get_all_reports new file mode 100644 index 0000000..e295670 --- /dev/null +++ b/internal/reportutils/testdata/TestGetReports/golden/get_all_reports @@ -0,0 +1,7 @@ +1: 1 +2: 2 +3: 3 +101: 101 +107: 107 +251: 251 +257: 257 diff --git a/internal/reportutils/testdata/TestGetReports/golden/multiple_consecutive_windows b/internal/reportutils/testdata/TestGetReports/golden/multiple_consecutive_windows new file mode 100644 index 0000000..8e9a957 --- /dev/null +++ b/internal/reportutils/testdata/TestGetReports/golden/multiple_consecutive_windows @@ -0,0 +1,3 @@ +0: 7 +100: 107 +200: 207 diff --git a/internal/reportutils/testdata/TestGetReports/golden/multiple_non-consecutive_windows b/internal/reportutils/testdata/TestGetReports/golden/multiple_non-consecutive_windows new file mode 100644 index 0000000..f2cb19d --- /dev/null +++ b/internal/reportutils/testdata/TestGetReports/golden/multiple_non-consecutive_windows @@ -0,0 +1,3 @@ +0: 7 +100: 107 +250: 257 diff --git a/internal/uploader/export_test.go b/internal/uploader/export_test.go index 7619a9b..d34d9f5 100644 --- a/internal/uploader/export_test.go +++ b/internal/uploader/export_test.go @@ -1,2 +1,19 @@ package uploader +func WithCachePath(path string) Options { + return func(o *options) { + o.cachePath = path + } +} + +func WithBaseServerURL(url string) Options { + return func(o *options) { + o.baseServerURL = url + } +} + +func WithTimeProvider(tp timeProvider) Options { + return func(o *options) { + o.timeProvider = tp + } +} diff --git a/internal/uploader/internal_test.go b/internal/uploader/internal_test.go new file mode 100644 index 0000000..01aafa6 --- /dev/null +++ b/internal/uploader/internal_test.go @@ -0,0 +1 @@ +package uploader diff --git a/internal/uploader/testdata/TestUpload/golden/bad_content b/internal/uploader/testdata/TestUpload/golden/bad_content new file mode 100644 index 0000000..be447b6 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/bad_content @@ -0,0 +1 @@ +local/1.json: bad content diff --git a/internal/uploader/testdata/TestUpload/golden/bad_response b/internal/uploader/testdata/TestUpload/golden/bad_response new file mode 100644 index 0000000..0d60902 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/bad_response @@ -0,0 +1 @@ +local/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/bad_url b/internal/uploader/testdata/TestUpload/golden/bad_url new file mode 100644 index 0000000..0d60902 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/bad_url @@ -0,0 +1 @@ +local/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/consent_manager_false b/internal/uploader/testdata/TestUpload/golden/consent_manager_false new file mode 100644 index 0000000..4d2381a --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/consent_manager_false @@ -0,0 +1 @@ +uploaded/1.json: '{"OptOut":true}' diff --git a/internal/uploader/testdata/TestUpload/golden/consent_manager_global_error b/internal/uploader/testdata/TestUpload/golden/consent_manager_global_error new file mode 100644 index 0000000..0d60902 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/consent_manager_global_error @@ -0,0 +1 @@ +local/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/consent_manager_global_error_with_true b/internal/uploader/testdata/TestUpload/golden/consent_manager_global_error_with_true new file mode 100644 index 0000000..0d60902 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/consent_manager_global_error_with_true @@ -0,0 +1 @@ +local/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/consent_manager_global_false,_source_true b/internal/uploader/testdata/TestUpload/golden/consent_manager_global_false,_source_true new file mode 100644 index 0000000..4d2381a --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/consent_manager_global_false,_source_true @@ -0,0 +1 @@ +uploaded/1.json: '{"OptOut":true}' diff --git a/internal/uploader/testdata/TestUpload/golden/consent_manager_global_true,_source_false b/internal/uploader/testdata/TestUpload/golden/consent_manager_global_true,_source_false new file mode 100644 index 0000000..4d2381a --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/consent_manager_global_true,_source_false @@ -0,0 +1 @@ +uploaded/1.json: '{"OptOut":true}' diff --git a/internal/uploader/testdata/TestUpload/golden/consent_manager_source_error b/internal/uploader/testdata/TestUpload/golden/consent_manager_source_error new file mode 100644 index 0000000..0d60902 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/consent_manager_source_error @@ -0,0 +1 @@ +local/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/consent_manager_source_error_with_true b/internal/uploader/testdata/TestUpload/golden/consent_manager_source_error_with_true new file mode 100644 index 0000000..0d60902 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/consent_manager_source_error_with_true @@ -0,0 +1 @@ +local/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/dry_run b/internal/uploader/testdata/TestUpload/golden/dry_run new file mode 100644 index 0000000..0d60902 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/dry_run @@ -0,0 +1 @@ +local/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/duplicate_upload b/internal/uploader/testdata/TestUpload/golden/duplicate_upload new file mode 100644 index 0000000..382d755 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/duplicate_upload @@ -0,0 +1,2 @@ +local/1.json: '{"Content":"normal content"}' +uploaded/1.json: bad content diff --git a/internal/uploader/testdata/TestUpload/golden/future_timestamp b/internal/uploader/testdata/TestUpload/golden/future_timestamp new file mode 100644 index 0000000..f8a9041 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/future_timestamp @@ -0,0 +1,2 @@ +local/11.json: '{"Content":"normal content"}' +uploaded/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/min_age b/internal/uploader/testdata/TestUpload/golden/min_age new file mode 100644 index 0000000..463c163 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/min_age @@ -0,0 +1,2 @@ +local/9.json: '{"Content":"normal content"}' +uploaded/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/multi_upload b/internal/uploader/testdata/TestUpload/golden/multi_upload new file mode 100644 index 0000000..55b3ee3 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/multi_upload @@ -0,0 +1,2 @@ +uploaded/1.json: '{"Content":"normal content"}' +uploaded/5.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/no_reports b/internal/uploader/testdata/TestUpload/golden/no_reports new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/no_reports @@ -0,0 +1 @@ +{} diff --git a/internal/uploader/testdata/TestUpload/golden/no_reports_with_dummy b/internal/uploader/testdata/TestUpload/golden/no_reports_with_dummy new file mode 100644 index 0000000..0a52bdc --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/no_reports_with_dummy @@ -0,0 +1,33 @@ +-1.json: bad contents +dummy.json: |- + { + "OS": "something" + } +dummy/dummy.json: |- + { + "OS": "something" + } +dummy/empty-dummy.json: "" +empty-dummy.json: "" +local/-1.json: bad contents +local/dummy.json: |- + { + "OS": "something" + } +local/dummy/dummy.json: |- + { + "OS": "something" + } +local/dummy/empty-dummy.json: "" +local/empty-dummy.json: "" +uploaded/-1.json: bad contents +uploaded/dummy.json: |- + { + "OS": "something" + } +uploaded/dummy/dummy.json: |- + { + "OS": "something" + } +uploaded/dummy/empty-dummy.json: "" +uploaded/empty-dummy.json: "" diff --git a/internal/uploader/testdata/TestUpload/golden/offline_server b/internal/uploader/testdata/TestUpload/golden/offline_server new file mode 100644 index 0000000..0d60902 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/offline_server @@ -0,0 +1 @@ +local/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/single_upload b/internal/uploader/testdata/TestUpload/golden/single_upload new file mode 100644 index 0000000..9dc4645 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/single_upload @@ -0,0 +1 @@ +uploaded/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/test_source/-1.json b/internal/uploader/testdata/test_source/-1.json new file mode 100644 index 0000000..89d4cda --- /dev/null +++ b/internal/uploader/testdata/test_source/-1.json @@ -0,0 +1 @@ +bad contents \ No newline at end of file diff --git a/internal/uploader/testdata/test_source/dummy.json b/internal/uploader/testdata/test_source/dummy.json new file mode 100644 index 0000000..6012d3c --- /dev/null +++ b/internal/uploader/testdata/test_source/dummy.json @@ -0,0 +1,3 @@ +{ + "OS": "something" +} \ No newline at end of file diff --git a/internal/uploader/testdata/test_source/dummy/dummy.json b/internal/uploader/testdata/test_source/dummy/dummy.json new file mode 100644 index 0000000..6012d3c --- /dev/null +++ b/internal/uploader/testdata/test_source/dummy/dummy.json @@ -0,0 +1,3 @@ +{ + "OS": "something" +} \ No newline at end of file diff --git a/internal/uploader/testdata/test_source/dummy/empty-dummy.json b/internal/uploader/testdata/test_source/dummy/empty-dummy.json new file mode 100644 index 0000000..e69de29 diff --git a/internal/uploader/testdata/test_source/empty-dummy.json b/internal/uploader/testdata/test_source/empty-dummy.json new file mode 100644 index 0000000..e69de29 diff --git a/internal/uploader/upload.go b/internal/uploader/upload.go new file mode 100644 index 0000000..4775582 --- /dev/null +++ b/internal/uploader/upload.go @@ -0,0 +1,175 @@ +package uploader + +import ( + "bytes" + "encoding/json" + "fmt" + "log/slog" + "math" + "net/http" + "net/url" + "os" + "path" + "sync" + "time" + + "github.com/ubuntu/ubuntu-insights/internal/constants" + "github.com/ubuntu/ubuntu-insights/internal/fileutils" + "github.com/ubuntu/ubuntu-insights/internal/reportutils" +) + +// Upload uploads the reports corresponding to the source to the configured server. +// Does not do duplicate checks. +func (um Manager) Upload() error { + slog.Debug("Uploading reports") + + gConsent, err := um.consentManager.GetConsentState("") + if err != nil { + return fmt.Errorf("upload failed to get global consent state: %v", err) + } + + sConsent, err := um.consentManager.GetConsentState(um.source) + if err != nil { + return fmt.Errorf("upload failed to get source consent state: %v", err) + } + + reports, err := reportutils.GetAllReports(um.collectedDir) + if err != nil { + return fmt.Errorf("failed to get reports: %v", err) + } + + url, err := um.getURL() + if err != nil { + return fmt.Errorf("failed to get URL: %v", err) + } + + var wg sync.WaitGroup + for _, file := range reports { + wg.Add(1) + go func(file string) { + defer wg.Done() + if err := um.upload(file, url, gConsent && sConsent); err != nil { + slog.Warn("Failed to upload report", "file", file, "source", um.source, "error", err) + } + }(file) + } + wg.Wait() + + return nil +} + +func (um Manager) upload(file, url string, consent bool) error { + slog.Debug("Uploading report", "file", file, "consent", consent) + + ts, err := reportutils.GetReportTime(file) + if err != nil { + return fmt.Errorf("failed to parse report time from filename: %v", err) + } + + // Report maturity check + if um.minAge > math.MaxInt64 { + return fmt.Errorf("min age is too large: %d", um.minAge) + } + if ts+int64(um.minAge) > um.timeProvider.NowUnix() { + slog.Debug("Skipping report due to min age", "timestamp", file, "minAge", um.minAge) + return ErrReportNotMature + } + + payload, err := um.getPayload(file, consent) + if err != nil { + return fmt.Errorf("failed to get payload: %v", err) + } + slog.Debug("Uploading", "payload", payload) + + if !um.dryRun { + if err := send(url, payload); err != nil { + return fmt.Errorf("failed to send data: %v", err) + } + + if err := um.moveReport(file, payload); err != nil { + return fmt.Errorf("failed to move report after uploading: %v", err) + } + } + + return nil +} + +func (um Manager) getURL() (string, error) { + u, err := url.Parse(um.baseServerURL) + if err != nil { + return "", fmt.Errorf("failed to parse base server URL %s: %v", um.baseServerURL, err) + } + u.Path = path.Join(u.Path, um.source) + return u.String(), nil +} + +func (um Manager) getPayload(file string, consent bool) ([]byte, error) { + path := path.Join(um.collectedDir, file) + var jsonData map[string]interface{} + + data, err := json.Marshal(constants.OptOutJSON) + if err != nil { + return nil, fmt.Errorf("failed to marshal JSON data") + } + if consent { + // Read the report file + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read report file: %v", err) + } + + // Remashal the JSON data to ensure it is valid + if err := json.Unmarshal(data, &jsonData); err != nil { + return nil, fmt.Errorf("failed to unmarshal JSON data: %v", err) + } + + // Marshal the JSON data back to bytes + data, err = json.Marshal(jsonData) + if err != nil { + return nil, fmt.Errorf("failed to marshal JSON data: %v", err) + } + + return data, nil + } + + return data, nil +} + +// moveReport writes the uploaded report to the uploaded directory, and removes it from the collected directory. +func (um Manager) moveReport(file string, data []byte) error { + err := fileutils.AtomicWrite(path.Join(um.uploadedDir, file), data) + if err != nil { + return fmt.Errorf("failed to write report to uploaded directory: %v", err) + } + + err = os.Remove(path.Join(um.collectedDir, file)) + if err != nil { + return fmt.Errorf("failed to remove report from collected directory: %v", err) + } + + return nil +} + +func send(url string, data []byte) error { + slog.Debug("Sending data to server", "url", url, "data", data) + req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(data)) + if err != nil { + return fmt.Errorf("failed to create request: %v", err) + } + + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{Timeout: time.Second * 10} + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("failed to send data: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("server returned status code %d", resp.StatusCode) + } + + return nil +} diff --git a/internal/uploader/uploader.go b/internal/uploader/uploader.go index b1c189e..a1eebe3 100644 --- a/internal/uploader/uploader.go +++ b/internal/uploader/uploader.go @@ -1,3 +1,88 @@ // Package uploader implements the uploader component. // The uploader component is responsible for uploading reports to the Ubuntu Insights server. package uploader + +import ( + "errors" + "log/slog" + "path/filepath" + "time" + + "github.com/ubuntu/ubuntu-insights/internal/constants" +) + +var ( + // ErrReportNotMature is returned when a report is not mature enough to be uploaded based on min age. + ErrReportNotMature = errors.New("report is not mature enough to be uploaded") + // ErrDuplicateReport is returned when a report has already been uploaded for this period. + ErrDuplicateReport = errors.New("report has already been uploaded for this period") + // ErrEmptySource is returned when the passed source is incorrectly an empty string. + ErrEmptySource = errors.New("source cannot be an empty string") +) + +type timeProvider interface { + NowUnix() int64 +} + +type realTimeProvider struct{} + +func (realTimeProvider) NowUnix() int64 { + return time.Now().Unix() +} + +// Manager is an abstraction of the uploader component. +type Manager struct { + source string + consentManager consentManager + minAge uint + dryRun bool + + baseServerURL string + collectedDir string + uploadedDir string + timeProvider timeProvider +} + +type options struct { + // Private members exported for tests. + baseServerURL string + cachePath string + timeProvider timeProvider +} + +// Options represents an optional function to override Upload Manager default values. +type Options func(*options) + +type consentManager interface { + GetConsentState(source string) (bool, error) +} + +// New returns a new UploaderManager. +func New(cm consentManager, source string, minAge uint, dryRun bool, args ...Options) (Manager, error) { + slog.Debug("Creating new uploader manager", "source", source, "minAge", minAge, "dryRun", dryRun) + + if source == "" { + return Manager{}, ErrEmptySource + } + + opts := options{ + baseServerURL: constants.DefaultServerURL, + cachePath: constants.GetDefaultCachePath(), + timeProvider: realTimeProvider{}, + } + for _, opt := range args { + opt(&opts) + } + + return Manager{ + source: source, + consentManager: cm, + minAge: minAge, + dryRun: dryRun, + timeProvider: opts.timeProvider, + + baseServerURL: opts.baseServerURL, + collectedDir: filepath.Join(opts.cachePath, constants.LocalFolder), + uploadedDir: filepath.Join(opts.cachePath, constants.UploadedFolder), + }, nil +} diff --git a/internal/uploader/uploader_test.go b/internal/uploader/uploader_test.go new file mode 100644 index 0000000..720c2e4 --- /dev/null +++ b/internal/uploader/uploader_test.go @@ -0,0 +1,226 @@ +package uploader_test + +import ( + "bytes" + "encoding/json" + "fmt" + "io/fs" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "github.com/ubuntu/ubuntu-insights/internal/fileutils" + "github.com/ubuntu/ubuntu-insights/internal/testutils" + "github.com/ubuntu/ubuntu-insights/internal/uploader" +) + +type reportType any + +var ( + normal reportType = struct{ Content string }{Content: "normal content"} + badContent = `bad content` +) + +type mockTimeProvider struct { + currentTime int64 +} + +func (m mockTimeProvider) NowUnix() int64 { + return m.currentTime +} + +func TestUpload(t *testing.T) { + t.Parallel() + + var ( + cmSErr = testConsentManager{sErr: fmt.Errorf("consent error")} + cmTrueSErr = testConsentManager{sState: true, gState: true, sErr: fmt.Errorf("consent error")} + cmGErr = testConsentManager{gErr: fmt.Errorf("consent error")} + cmTrueGErr = testConsentManager{gState: true, gErr: fmt.Errorf("consent error")} + cmTrue = testConsentManager{sState: true, gState: true} + cmFalse = testConsentManager{sState: false, gState: false} + cmSTrue = testConsentManager{sState: true, gState: false} + cmGTrue = testConsentManager{sState: false, gState: true} + ) + + const mockTime = 10 + + tests := map[string]struct { + localFiles, uploadedFiles map[string]reportType + dummy bool + serverResponse int + serverOffline bool + url string + + cm testConsentManager + minAge uint + dryRun bool + + wantErr bool + }{ + "No Reports": {cm: cmTrue, serverResponse: http.StatusOK}, + "No Reports with Dummy": {dummy: true, cm: cmTrue, serverResponse: http.StatusOK}, + "Single Upload": {localFiles: map[string]reportType{"1.json": normal}, cm: cmTrue, serverResponse: http.StatusOK}, + "Multi Upload": {localFiles: map[string]reportType{"1.json": normal, "5.json": normal}, cm: cmTrue, serverResponse: http.StatusOK}, + "Min Age": {localFiles: map[string]reportType{"1.json": normal, "9.json": normal}, cm: cmTrue, minAge: 5, serverResponse: http.StatusOK}, + "Future Timestamp": {localFiles: map[string]reportType{"1.json": normal, "11.json": normal}, cm: cmTrue, serverResponse: http.StatusOK}, + "Duplicate Upload": {localFiles: map[string]reportType{"1.json": normal}, uploadedFiles: map[string]reportType{"1.json": badContent}, cm: cmTrue, serverResponse: http.StatusAccepted}, + "Bad Content": {localFiles: map[string]reportType{"1.json": badContent}, cm: cmTrue, serverResponse: http.StatusOK}, + + "Consent Manager Source Error": {localFiles: map[string]reportType{"1.json": normal}, cm: cmSErr, serverResponse: http.StatusOK, wantErr: true}, + "Consent Manager Source Error with True": {localFiles: map[string]reportType{"1.json": normal}, cm: cmTrueSErr, serverResponse: http.StatusOK, wantErr: true}, + "Consent Manager Global Error": {localFiles: map[string]reportType{"1.json": normal}, cm: cmGErr, serverResponse: http.StatusOK, wantErr: true}, + "Consent Manager Global Error with True": {localFiles: map[string]reportType{"1.json": normal}, cm: cmTrueGErr, serverResponse: http.StatusOK, wantErr: true}, + "Consent Manager False": {localFiles: map[string]reportType{"1.json": normal}, cm: cmFalse, serverResponse: http.StatusOK}, + "Consent Manager Global True, Source False": {localFiles: map[string]reportType{"1.json": normal}, cm: cmGTrue, serverResponse: http.StatusOK}, + "Consent Manager Global False, Source True": {localFiles: map[string]reportType{"1.json": normal}, cm: cmSTrue, serverResponse: http.StatusOK}, + + "Dry run": {localFiles: map[string]reportType{"1.json": normal}, cm: cmTrue, dryRun: true}, + + "Bad URL": {localFiles: map[string]reportType{"1.json": normal}, cm: cmTrue, url: "http://a b.com/", wantErr: true}, + "Bad Response": {localFiles: map[string]reportType{"1.json": normal}, cm: cmTrue, serverResponse: http.StatusForbidden}, + "Offline Server": {localFiles: map[string]reportType{"1.json": normal}, cm: cmTrue, serverOffline: true}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + dir := setupTmpDir(t, tc.localFiles, tc.uploadedFiles, tc.dummy) + + if !tc.serverOffline { + status := statusHandler(tc.serverResponse) + ts := httptest.NewServer(&status) + t.Cleanup(func() { ts.Close() }) + if tc.url == "" { + tc.url = ts.URL + } + } + + mgr, err := uploader.New(tc.cm, "source", tc.minAge, tc.dryRun, + uploader.WithBaseServerURL(tc.url), uploader.WithCachePath(dir), uploader.WithTimeProvider(mockTimeProvider{currentTime: mockTime})) + require.NoError(t, err, "Setup: failed to create new uploader manager") + + err = mgr.Upload() + if tc.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + got, err := getDirResult(t, dir, 3) + require.NoError(t, err) + want := testutils.LoadWithUpdateFromGoldenYAML(t, got) + require.EqualValues(t, want, got) + }) + } +} + +func setupTmpDir(t *testing.T, localFiles, uploadedFiles map[string]reportType, dummy bool) string { + t.Helper() + + dir, err := os.MkdirTemp("", "uploader-test") + require.NoError(t, err, "Setup: failed to create temporary directory") + t.Cleanup(func() { os.RemoveAll(dir) }) + + localDir := filepath.Join(dir, "local") + uploadedDir := filepath.Join(dir, "uploaded") + require.NoError(t, os.Mkdir(localDir, 0750), "Setup: failed to create local directory") + require.NoError(t, os.Mkdir(uploadedDir, 0750), "Setup: failed to create uploaded directory") + + if dummy { + copyDummyData(t, "testdata/test_source", dir, localDir, uploadedDir) + } + + writeFiles(t, localDir, localFiles) + writeFiles(t, uploadedDir, uploadedFiles) + + return dir +} + +func copyDummyData(t *testing.T, sourceDir, dir, localDir, uploadedDir string) { + t.Helper() + require.NoError(t, testutils.CopyDir(sourceDir, dir), "Setup: failed to copy dummy data to temporary directory") + require.NoError(t, testutils.CopyDir(sourceDir, localDir), "Setup: failed to copy dummy data to local") + require.NoError(t, testutils.CopyDir(sourceDir, uploadedDir), "Setup: failed to copy dummy data to uploaded") +} + +func writeFiles(t *testing.T, targetDir string, files map[string]reportType) { + t.Helper() + for file, content := range files { + var data []byte + var err error + + switch v := content.(type) { + case string: + data = []byte(v) + default: + data, err = json.Marshal(content) + require.NoError(t, err, "Setup: failed to marshal sample data") + } + require.NoError(t, fileutils.AtomicWrite(filepath.Join(targetDir, file), data), "Setup: failed to write file") + } +} + +func getDirResult(t *testing.T, dir string, maxDepth uint) (map[string]string, error) { + t.Helper() + + files := make(map[string]string) + + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if path == dir { + return nil + } + + relPath, err := filepath.Rel(dir, path) + if err != nil { + return err + } + + depth := uint(len(filepath.SplitList(relPath))) + if depth > maxDepth { + return fmt.Errorf("max depth %d exceeded at %s", maxDepth, relPath) + } + + if !d.IsDir() { + content, err := os.ReadFile(path) + if err != nil { + return err + } + // Normalize content between windows and linux + content = bytes.ReplaceAll(content, []byte("\r\n"), []byte("\n")) + files[filepath.ToSlash(relPath)] = string(content) + } + + return nil + }) + + return files, err +} + +type testConsentManager struct { + sState bool + gState bool + sErr error + gErr error +} + +func (m testConsentManager) GetConsentState(source string) (bool, error) { + if source != "" { + return m.sState, m.sErr + } + return m.gState, m.gErr +} + +type statusHandler int + +func (h *statusHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(int(*h)) +} From 5218a6ce4a27ef58055436d84a5b0ead0afe4b27 Mon Sep 17 00:00:00 2001 From: kat <28567881+hk21702@users.noreply.github.com> Date: Thu, 23 Jan 2025 17:34:31 -0500 Subject: [PATCH 17/26] Improve reportutils test coverage --- internal/reportutils/reportutils_test.go | 118 ++++++++++++++--------- 1 file changed, 75 insertions(+), 43 deletions(-) diff --git a/internal/reportutils/reportutils_test.go b/internal/reportutils/reportutils_test.go index 871eb21..9df7102 100644 --- a/internal/reportutils/reportutils_test.go +++ b/internal/reportutils/reportutils_test.go @@ -80,8 +80,10 @@ func TestGetReportPath(t *testing.T) { subDirFiles []string time int64 period int + invalidDir bool - wantErr error + wantSpecificErr error + wantGenericErr bool }{ "Empty Directory": {time: 1, period: 500}, "Files in subDir": {subDir: "subdir", subDirFiles: []string{"1.json", "2.json"}, time: 1, period: 500}, @@ -93,8 +95,10 @@ func TestGetReportPath(t *testing.T) { "Negative Timestamp": {files: []string{"-100.json", "-101.json"}, time: -150, period: 100}, "Not Inclusive Period": {files: []string{"1.json", "7.json"}, time: 2, period: 7}, - "Invalid Negative Period": {files: []string{"1.json", "7.json"}, time: 2, period: -7, wantErr: reportutils.ErrInvalidPeriod}, - "Invalid Zero Period": {files: []string{"1.json", "7.json"}, time: 2, period: 0, wantErr: reportutils.ErrInvalidPeriod}, + "Invalid Negative Period": {files: []string{"1.json", "7.json"}, time: 2, period: -7, wantSpecificErr: reportutils.ErrInvalidPeriod}, + "Invalid Zero Period": {files: []string{"1.json", "7.json"}, time: 2, period: 0, wantSpecificErr: reportutils.ErrInvalidPeriod}, + + "Invalid Dir": {period: 1, invalidDir: true, wantGenericErr: true}, } for name, tc := range tests { @@ -104,12 +108,20 @@ func TestGetReportPath(t *testing.T) { dir, err := setupTmpDir(t, tc.files, tc.subDir, tc.subDirFiles) require.NoError(t, err, "Setup: failed to setup temporary directory") defer os.RemoveAll(dir) + if tc.invalidDir { + dir = filepath.Join(dir, "invalid dir") + } got, err := reportutils.GetReportPath(dir, tc.time, tc.period) - if tc.wantErr != nil { - require.ErrorIs(t, err, tc.wantErr) + if tc.wantSpecificErr != nil { + require.ErrorIs(t, err, tc.wantSpecificErr) + return + } + if tc.wantGenericErr { + require.Error(t, err, "expected an error but got none") return } + require.NoError(t, err, "got an unexpected error") if got != "" { @@ -123,38 +135,6 @@ func TestGetReportPath(t *testing.T) { } } -func setupTmpDir(t *testing.T, files []string, subDir string, subDirFiles []string) (string, error) { - t.Helper() - - dir, err := os.MkdirTemp("", "reportutils-test") - if err != nil { - return "", err - } - - for _, file := range files { - path := filepath.Join(dir, file) - if err := os.WriteFile(path, []byte{}, 0600); err != nil { - return "", err - } - } - - if subDir != "" { - subDirPath := filepath.Join(dir, subDir) - if err := os.Mkdir(subDirPath, 0700); err != nil { - return "", err - } - - for _, file := range subDirFiles { - path := filepath.Join(subDirPath, file) - if err := os.WriteFile(path, []byte{}, 0600); err != nil { - return "", err - } - } - } - - return dir, nil -} - func TestGetReports(t *testing.T) { t.Parallel() @@ -163,8 +143,10 @@ func TestGetReports(t *testing.T) { subDir string subDirFiles []string period int + invalidDir bool - wantErr error + wantSpecificErr error + wantGenericErr bool }{ "Empty Directory": {period: 500}, "Files in subDir": {subDir: "subdir", subDirFiles: []string{"1.json", "2.json"}, period: 500}, @@ -177,6 +159,11 @@ func TestGetReports(t *testing.T) { "Multiple Consecutive Windows": {files: []string{"1.json", "7.json", "101.json", "107.json", "201.json", "207.json"}, period: 100}, "Multiple Non-Consecutive Windows": {files: []string{"1.json", "7.json", "101.json", "107.json", "251.json", "257.json"}, period: 50}, "Get All Reports": {files: []string{"1.json", "2.json", "3.json", "101.json", "107.json", "251.json", "257.json"}, period: 1}, + + "Invalid Negative Period": {files: []string{"1.json", "7.json"}, period: -7, wantSpecificErr: reportutils.ErrInvalidPeriod}, + "Invalid Zero Period": {files: []string{"1.json", "7.json"}, period: 0, wantSpecificErr: reportutils.ErrInvalidPeriod}, + + "Invalid Dir": {period: 1, invalidDir: true, wantGenericErr: true}, } for name, tc := range tests { @@ -186,10 +173,17 @@ func TestGetReports(t *testing.T) { dir, err := setupTmpDir(t, tc.files, tc.subDir, tc.subDirFiles) require.NoError(t, err, "Setup: failed to setup temporary directory") defer os.RemoveAll(dir) + if tc.invalidDir { + dir = filepath.Join(dir, "invalid dir") + } got, err := reportutils.GetReports(dir, tc.period) - if tc.wantErr != nil { - require.ErrorIs(t, err, tc.wantErr) + if tc.wantSpecificErr != nil { + require.ErrorIs(t, err, tc.wantSpecificErr) + return + } + if tc.wantGenericErr { + require.Error(t, err, "expected an error but got none") return } require.NoError(t, err, "got an unexpected error") @@ -207,14 +201,17 @@ func TestGetAllReports(t *testing.T) { files []string subDir string subDirFiles []string + invalidDir bool - wantErr error + wantErr bool }{ "Empty Directory": {}, "Files in subDir": {files: []string{"1.json", "2.json"}, subDir: "subdir", subDirFiles: []string{"1.json", "2.json"}}, "Invalid File Extension": {files: []string{"1.txt", "2.txt"}}, "Invalid File Names": {files: []string{"i-1.json", "i-2.json", "i-3.json", "test.json", "one.json"}}, "Mix of Valid and Invalid": {files: []string{"1.json", "2.json", "500.json", "i-1.json", "i-2.json", "i-3.json", "test.json", "five.json"}}, + + "Invalid Dir": {invalidDir: true, wantErr: true}, } for name, tc := range tests { @@ -223,10 +220,13 @@ func TestGetAllReports(t *testing.T) { dir, err := setupTmpDir(t, tc.files, tc.subDir, tc.subDirFiles) require.NoError(t, err, "Setup: failed to setup temporary directory") defer os.RemoveAll(dir) + if tc.invalidDir { + dir = filepath.Join(dir, "invalid dir") + } got, err := reportutils.GetAllReports(dir) - if tc.wantErr != nil { - require.ErrorIs(t, err, tc.wantErr) + if tc.wantErr { + require.Error(t, err, "expected an error but got none") return } require.NoError(t, err, "got an unexpected error") @@ -236,3 +236,35 @@ func TestGetAllReports(t *testing.T) { }) } } + +func setupTmpDir(t *testing.T, files []string, subDir string, subDirFiles []string) (string, error) { + t.Helper() + + dir, err := os.MkdirTemp("", "reportutils-test") + if err != nil { + return "", err + } + + for _, file := range files { + path := filepath.Join(dir, file) + if err := os.WriteFile(path, []byte{}, 0600); err != nil { + return "", err + } + } + + if subDir != "" { + subDirPath := filepath.Join(dir, subDir) + if err := os.Mkdir(subDirPath, 0700); err != nil { + return "", err + } + + for _, file := range subDirFiles { + path := filepath.Join(subDirPath, file) + if err := os.WriteFile(path, []byte{}, 0600); err != nil { + return "", err + } + } + } + + return dir, nil +} From ce11cbe29f19ded48a9c4bb69ba08ac95346bd4c Mon Sep 17 00:00:00 2001 From: kat <28567881+hk21702@users.noreply.github.com> Date: Fri, 24 Jan 2025 11:47:22 -0500 Subject: [PATCH 18/26] Improve uploader tests --- internal/constants/constants.go | 2 +- internal/fileutils/fileutils.go | 10 + internal/fileutils/fileutils_test.go | 36 ++++ internal/testutils/files.go | 46 +++++ internal/uploader/internal_test.go | 191 ++++++++++++++++++ .../testdata/TestUpload/golden/force_cm_false | 1 + .../TestUpload/golden/force_duplicate | 1 + .../testdata/TestUpload/golden/force_min_age | 2 + .../TestUpload/golden/invalid_directory | 1 + .../TestUpload/golden/optout_payload_cm_false | 1 + .../TestUpload/golden/optout_payload_cm_true | 1 + internal/uploader/upload.go | 111 +++++----- internal/uploader/uploader.go | 36 ++-- internal/uploader/uploader_test.go | 134 ++++++------ 14 files changed, 433 insertions(+), 140 deletions(-) create mode 100644 internal/uploader/testdata/TestUpload/golden/force_cm_false create mode 100644 internal/uploader/testdata/TestUpload/golden/force_duplicate create mode 100644 internal/uploader/testdata/TestUpload/golden/force_min_age create mode 100644 internal/uploader/testdata/TestUpload/golden/invalid_directory create mode 100644 internal/uploader/testdata/TestUpload/golden/optout_payload_cm_false create mode 100644 internal/uploader/testdata/TestUpload/golden/optout_payload_cm_true diff --git a/internal/constants/constants.go b/internal/constants/constants.go index 670063f..6f9acf6 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -38,7 +38,7 @@ const ( ) // OptOutJSON is the data sent in case of Opt-Out choice. -var OptOutJSON = struct{ OptOut bool }{OptOut: true} +var OptOutJSON = struct{ OptOut bool }{true} type options struct { baseDir func() (string, error) diff --git a/internal/fileutils/fileutils.go b/internal/fileutils/fileutils.go index b113a2f..b07bfde 100644 --- a/internal/fileutils/fileutils.go +++ b/internal/fileutils/fileutils.go @@ -2,6 +2,7 @@ package fileutils import ( + "errors" "fmt" "log/slog" "os" @@ -36,3 +37,12 @@ func AtomicWrite(path string, data []byte) error { } return nil } + +// FileExists checks if a file exists. +func FileExists(path string) (bool, error) { + _, err := os.Stat(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return false, err + } + return !errors.Is(err, os.ErrNotExist), nil +} diff --git a/internal/fileutils/fileutils_test.go b/internal/fileutils/fileutils_test.go index 15c2b96..ec6f776 100644 --- a/internal/fileutils/fileutils_test.go +++ b/internal/fileutils/fileutils_test.go @@ -74,3 +74,39 @@ func TestAtomicWrite(t *testing.T) { }) } } + +func TestFileExists(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + fileExists bool + + wantExists bool + wantError bool + }{ + "Returns_true_when_file_exists": {fileExists: true, wantExists: true}, + "Returns_false_when_file_does_not_exist": {fileExists: false, wantExists: false}, + "Returns_false_when_parent_directory_does_not_exist": {fileExists: false, wantExists: false}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + tempDir := t.TempDir() + path := filepath.Join(tempDir, "file") + if tc.fileExists { + err := fileutils.AtomicWrite(path, []byte("")) + require.NoError(t, err, "Setup: AtomicWrite should not return an error") + } + + exists, err := fileutils.FileExists(path) + if tc.wantError { + require.Error(t, err, "FileExists should return an error") + } else { + require.NoError(t, err, "FileExists should not return an error") + } + require.Equal(t, tc.wantExists, exists, "FileExists should return the expected result") + }) + } +} diff --git a/internal/testutils/files.go b/internal/testutils/files.go index 84a0cf1..681cb19 100644 --- a/internal/testutils/files.go +++ b/internal/testutils/files.go @@ -3,7 +3,10 @@ package testutils import ( + "bytes" + "fmt" "io" + "io/fs" "os" "path/filepath" "testing" @@ -52,3 +55,46 @@ func CopyDir(srcDir, dstDir string) error { return CopyFile(path, dstPath) }) } + +// GetDirContents returns the contents of a directory as a map of file paths to file contents. +// The contents are read as strings. +// The maxDepth parameter limits the depth of the directory tree to read. +func GetDirContents(t *testing.T, dir string, maxDepth uint) (map[string]string, error) { + t.Helper() + + files := make(map[string]string) + + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if path == dir { + return nil + } + + relPath, err := filepath.Rel(dir, path) + if err != nil { + return err + } + + depth := uint(len(filepath.SplitList(relPath))) + if depth > maxDepth { + return fmt.Errorf("max depth %d exceeded at %s", maxDepth, relPath) + } + + if !d.IsDir() { + content, err := os.ReadFile(path) + if err != nil { + return err + } + // Normalize content between Windows and Linux + content = bytes.ReplaceAll(content, []byte("\r\n"), []byte("\n")) + files[filepath.ToSlash(relPath)] = string(content) + } + + return nil + }) + + return files, err +} diff --git a/internal/uploader/internal_test.go b/internal/uploader/internal_test.go index 01aafa6..d7a0147 100644 --- a/internal/uploader/internal_test.go +++ b/internal/uploader/internal_test.go @@ -1 +1,192 @@ package uploader + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "github.com/ubuntu/ubuntu-insights/internal/constants" + "github.com/ubuntu/ubuntu-insights/internal/fileutils" +) + +type mockTimeProvider struct { + currentTime int64 +} + +func (m mockTimeProvider) NowUnix() int64 { + return m.currentTime +} + +func TestUploadBadFile(t *testing.T) { + t.Parallel() + basicContent := `{"Content":true, "string": "string"}` + badContent := `bad content` + + tests := map[string]struct { + fName string + fileContents string + missingFile bool + fileIsDir bool + url string + consent bool + + wantErr bool + }{ + "Ok": {fName: "0.json", fileContents: basicContent, wantErr: false}, + "Missing File": {fName: "0.json", fileContents: basicContent, missingFile: true, wantErr: true}, + "File Is Dir": {fName: "0.json", fileIsDir: true, wantErr: true}, + "Non-numeric": {fName: "not-numeric.json", fileContents: basicContent, wantErr: true}, + "Bad File Ext": {fName: "0.txt", fileContents: basicContent}, + "Bad Contents": {fName: "0.json", fileContents: badContent, wantErr: true}, + "Bad URL": {fName: "0.json", fileContents: basicContent, url: "http://bad host:1234", wantErr: true}, + + "Ok Consent": {fName: "0.json", fileContents: basicContent, consent: true, wantErr: false}, + "Missing File Consent": {fName: "0.json", fileContents: basicContent, missingFile: true, consent: true, wantErr: true}, + "File Is Dir Consent": {fName: "0.json", fileIsDir: true, consent: true, wantErr: true}, + "Non-numeric Consent": {fName: "not-numeric.json", fileContents: basicContent, consent: true, wantErr: true}, + "Bad File Ext Consent": {fName: "0.txt", fileContents: basicContent, consent: true}, + "Bad Contents Consent": {fName: "0.json", fileContents: badContent, consent: true, wantErr: true}, + "Bad URL Consent": {fName: "0.json", fileContents: basicContent, url: "http://bad host:1234", consent: true, wantErr: true}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + require.False(t, tc.missingFile && tc.fileIsDir, "Test case cannot have both missing file and file is dir") + + dir := t.TempDir() + + um := &Manager{ + collectedDir: filepath.Join(dir, constants.LocalFolder), + uploadedDir: filepath.Join(dir, constants.UploadedFolder), + minAge: 0, + timeProvider: mockTimeProvider{currentTime: 0}, + } + + require.NoError(t, os.Mkdir(um.collectedDir, 0750), "Setup: failed to create uploaded folder") + require.NoError(t, os.Mkdir(um.uploadedDir, 0750), "Setup: failed to create collected folder") + fPath := filepath.Join(um.collectedDir, tc.fName) + + if !tc.missingFile && !tc.fileIsDir { + require.NoError(t, fileutils.AtomicWrite(fPath, []byte(tc.fileContents)), "Setup: failed to create report file") + } + if tc.fileIsDir { + require.NoError(t, os.Mkdir(fPath, 0750), "Setup: failed to create directory") + } + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(func() { ts.Close() }) + + if tc.url == "" { + tc.url = ts.URL + } + + err := um.upload(tc.fName, tc.url, tc.consent, false) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + }) + } +} + +func TestMoveReport(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + fileExists bool + + wantErr bool + }{ + "File Exists": {fileExists: true}, + + "File Not Found": {fileExists: false, wantErr: true}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + + um := &Manager{ + collectedDir: filepath.Join(dir, constants.LocalFolder), + uploadedDir: filepath.Join(dir, constants.UploadedFolder), + } + + require.NoError(t, os.MkdirAll(um.collectedDir, 0750), "Setup: failed to create uploaded folder") + require.NoError(t, os.MkdirAll(um.uploadedDir, 0750), "Setup: failed to create collected folder") + + cDir := filepath.Join(um.collectedDir, "report.json") + uDir := filepath.Join(um.uploadedDir, "report.json") + if tc.fileExists { + f, err := os.Create(cDir) + require.NoError(t, err) + f.Close() + } + + err := um.moveReport(uDir, cDir, []byte("payload")) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + + _, err = os.Stat(uDir) + if !tc.fileExists { + require.Error(t, err, "File should not exist in the uploaded directory") + return + } + + require.NoError(t, err, "File should exist in the uploaded directory") + }) + } +} + +func TestSend(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + url string + noServer bool + serverResponse int + + wantErr bool + }{ + "No Server": {noServer: true, wantErr: true}, + "Bad URL": {url: "http://local host:1234", serverResponse: http.StatusOK, wantErr: true}, + "Bad Response": {serverResponse: http.StatusForbidden, wantErr: true}, + + "Success": {serverResponse: http.StatusOK}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tc.serverResponse) + })) + t.Cleanup(func() { ts.Close() }) + + if tc.url == "" { + tc.url = ts.URL + } + if tc.noServer { + ts.Close() + } + + err := send(tc.url, []byte("payload")) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + }) + } +} diff --git a/internal/uploader/testdata/TestUpload/golden/force_cm_false b/internal/uploader/testdata/TestUpload/golden/force_cm_false new file mode 100644 index 0000000..4d2381a --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/force_cm_false @@ -0,0 +1 @@ +uploaded/1.json: '{"OptOut":true}' diff --git a/internal/uploader/testdata/TestUpload/golden/force_duplicate b/internal/uploader/testdata/TestUpload/golden/force_duplicate new file mode 100644 index 0000000..9dc4645 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/force_duplicate @@ -0,0 +1 @@ +uploaded/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/force_min_age b/internal/uploader/testdata/TestUpload/golden/force_min_age new file mode 100644 index 0000000..f12da56 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/force_min_age @@ -0,0 +1,2 @@ +uploaded/1.json: '{"Content":"normal content"}' +uploaded/9.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/invalid_directory b/internal/uploader/testdata/TestUpload/golden/invalid_directory new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/invalid_directory @@ -0,0 +1 @@ +{} diff --git a/internal/uploader/testdata/TestUpload/golden/optout_payload_cm_false b/internal/uploader/testdata/TestUpload/golden/optout_payload_cm_false new file mode 100644 index 0000000..4d2381a --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/optout_payload_cm_false @@ -0,0 +1 @@ +uploaded/1.json: '{"OptOut":true}' diff --git a/internal/uploader/testdata/TestUpload/golden/optout_payload_cm_true b/internal/uploader/testdata/TestUpload/golden/optout_payload_cm_true new file mode 100644 index 0000000..4d2381a --- /dev/null +++ b/internal/uploader/testdata/TestUpload/golden/optout_payload_cm_true @@ -0,0 +1 @@ +uploaded/1.json: '{"OptOut":true}' diff --git a/internal/uploader/upload.go b/internal/uploader/upload.go index 4775582..cc15bdd 100644 --- a/internal/uploader/upload.go +++ b/internal/uploader/upload.go @@ -5,11 +5,11 @@ import ( "encoding/json" "fmt" "log/slog" - "math" "net/http" "net/url" "os" "path" + "path/filepath" "sync" "time" @@ -20,15 +20,15 @@ import ( // Upload uploads the reports corresponding to the source to the configured server. // Does not do duplicate checks. -func (um Manager) Upload() error { +func (um Manager) Upload(force bool) error { slog.Debug("Uploading reports") - gConsent, err := um.consentManager.GetConsentState("") + gConsent, err := um.consentM.GetConsentState("") if err != nil { return fmt.Errorf("upload failed to get global consent state: %v", err) } - sConsent, err := um.consentManager.GetConsentState(um.source) + sConsent, err := um.consentM.GetConsentState(um.source) if err != nil { return fmt.Errorf("upload failed to get source consent state: %v", err) } @@ -48,7 +48,7 @@ func (um Manager) Upload() error { wg.Add(1) go func(file string) { defer wg.Done() - if err := um.upload(file, url, gConsent && sConsent); err != nil { + if err := um.upload(file, url, gConsent && sConsent, force); err != nil { slog.Warn("Failed to upload report", "file", file, "source", um.source, "error", err) } }(file) @@ -58,37 +58,56 @@ func (um Manager) Upload() error { return nil } -func (um Manager) upload(file, url string, consent bool) error { - slog.Debug("Uploading report", "file", file, "consent", consent) +// upload uploads an individual report to the server. It returns an error if the report is not mature enough to be uploaded, or if the upload fails. +// It also moves the report to the uploaded directory after a successful upload. +func (um Manager) upload(fName, url string, consent, force bool) error { + slog.Debug("Uploading report", "file", fName, "consent", consent, "force", force) - ts, err := reportutils.GetReportTime(file) + ts, err := reportutils.GetReportTime(fName) if err != nil { return fmt.Errorf("failed to parse report time from filename: %v", err) } - // Report maturity check - if um.minAge > math.MaxInt64 { - return fmt.Errorf("min age is too large: %d", um.minAge) + if ts > um.timeProvider.NowUnix()-um.minAge && !force { + return fmt.Errorf("report is not mature enough to be uploaded") } - if ts+int64(um.minAge) > um.timeProvider.NowUnix() { - slog.Debug("Skipping report due to min age", "timestamp", file, "minAge", um.minAge) - return ErrReportNotMature + + // Check for duplicate reports. + fileExists, err := fileutils.FileExists(filepath.Join(um.uploadedDir, fName)) + if err != nil { + return fmt.Errorf("failed to check if report has already been uploaded: %v", err) + } + if fileExists && !force { + return fmt.Errorf("report has already been uploaded") } - payload, err := um.getPayload(file, consent) + origData, err := um.readPayload(fName) if err != nil { return fmt.Errorf("failed to get payload: %v", err) } - slog.Debug("Uploading", "payload", payload) - - if !um.dryRun { - if err := send(url, payload); err != nil { - return fmt.Errorf("failed to send data: %v", err) + data := origData + if !consent { + data, err = json.Marshal(constants.OptOutJSON) + if err != nil { + return fmt.Errorf("failed to marshal opt-out JSON data: %v", err) } + } + slog.Debug("Uploading", "payload", data) - if err := um.moveReport(file, payload); err != nil { - return fmt.Errorf("failed to move report after uploading: %v", err) + if um.dryRun { + slog.Debug("Dry run, skipping upload") + return nil + } + + // Move report first to avoid the situation where the report is sent, but not marked as sent. + if err := um.moveReport(filepath.Join(um.uploadedDir, fName), filepath.Join(um.collectedDir, fName), data); err != nil { + return fmt.Errorf("failed to move report after uploading: %v", err) + } + if err := send(url, data); err != nil { + if moveErr := um.moveReport(filepath.Join(um.collectedDir, fName), filepath.Join(um.uploadedDir, fName), origData); moveErr != nil { + return fmt.Errorf("failed to send data: %v, and failed to restore the original report: %v", err, moveErr) } + return fmt.Errorf("failed to send data: %v", err) } return nil @@ -103,48 +122,38 @@ func (um Manager) getURL() (string, error) { return u.String(), nil } -func (um Manager) getPayload(file string, consent bool) ([]byte, error) { - path := path.Join(um.collectedDir, file) +func (um Manager) readPayload(file string) ([]byte, error) { var jsonData map[string]interface{} - data, err := json.Marshal(constants.OptOutJSON) + // Read the report file + data, err := os.ReadFile(path.Join(um.collectedDir, file)) if err != nil { - return nil, fmt.Errorf("failed to marshal JSON data") + return nil, fmt.Errorf("failed to read report file: %v", err) } - if consent { - // Read the report file - data, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("failed to read report file: %v", err) - } - - // Remashal the JSON data to ensure it is valid - if err := json.Unmarshal(data, &jsonData); err != nil { - return nil, fmt.Errorf("failed to unmarshal JSON data: %v", err) - } - // Marshal the JSON data back to bytes - data, err = json.Marshal(jsonData) - if err != nil { - return nil, fmt.Errorf("failed to marshal JSON data: %v", err) - } + // Remarshal the JSON data to ensure it is valid + if err := json.Unmarshal(data, &jsonData); err != nil { + return nil, fmt.Errorf("failed to unmarshal JSON data: %v", err) + } - return data, nil + data, err = json.Marshal(jsonData) + if err != nil { + return nil, fmt.Errorf("failed to marshal JSON data: %v", err) } return data, nil } -// moveReport writes the uploaded report to the uploaded directory, and removes it from the collected directory. -func (um Manager) moveReport(file string, data []byte) error { - err := fileutils.AtomicWrite(path.Join(um.uploadedDir, file), data) +// moveReport writes the data to the writePath, and removes the matching file from the removePath. +func (um Manager) moveReport(writePath, removePath string, data []byte) error { + err := fileutils.AtomicWrite(writePath, data) if err != nil { - return fmt.Errorf("failed to write report to uploaded directory: %v", err) + return fmt.Errorf("failed to write report: %v", err) } - err = os.Remove(path.Join(um.collectedDir, file)) + err = os.Remove(removePath) if err != nil { - return fmt.Errorf("failed to remove report from collected directory: %v", err) + return fmt.Errorf("failed to remove report: %v", err) } return nil @@ -156,14 +165,12 @@ func send(url string, data []byte) error { if err != nil { return fmt.Errorf("failed to create request: %v", err) } - req.Header.Set("Content-Type", "application/json") client := &http.Client{Timeout: time.Second * 10} - resp, err := client.Do(req) if err != nil { - return fmt.Errorf("failed to send data: %v", err) + return fmt.Errorf("failed to send HTTP request: %v", err) } defer resp.Body.Close() diff --git a/internal/uploader/uploader.go b/internal/uploader/uploader.go index a1eebe3..c304583 100644 --- a/internal/uploader/uploader.go +++ b/internal/uploader/uploader.go @@ -3,23 +3,15 @@ package uploader import ( - "errors" + "fmt" "log/slog" + "math" "path/filepath" "time" "github.com/ubuntu/ubuntu-insights/internal/constants" ) -var ( - // ErrReportNotMature is returned when a report is not mature enough to be uploaded based on min age. - ErrReportNotMature = errors.New("report is not mature enough to be uploaded") - // ErrDuplicateReport is returned when a report has already been uploaded for this period. - ErrDuplicateReport = errors.New("report has already been uploaded for this period") - // ErrEmptySource is returned when the passed source is incorrectly an empty string. - ErrEmptySource = errors.New("source cannot be an empty string") -) - type timeProvider interface { NowUnix() int64 } @@ -32,10 +24,10 @@ func (realTimeProvider) NowUnix() int64 { // Manager is an abstraction of the uploader component. type Manager struct { - source string - consentManager consentManager - minAge uint - dryRun bool + source string + consentM consentManager + minAge int64 + dryRun bool baseServerURL string collectedDir string @@ -62,7 +54,11 @@ func New(cm consentManager, source string, minAge uint, dryRun bool, args ...Opt slog.Debug("Creating new uploader manager", "source", source, "minAge", minAge, "dryRun", dryRun) if source == "" { - return Manager{}, ErrEmptySource + return Manager{}, fmt.Errorf("source cannot be an empty string") + } + + if minAge > math.MaxInt64 { + return Manager{}, fmt.Errorf("min age %d is too large, would overflow", minAge) } opts := options{ @@ -75,11 +71,11 @@ func New(cm consentManager, source string, minAge uint, dryRun bool, args ...Opt } return Manager{ - source: source, - consentManager: cm, - minAge: minAge, - dryRun: dryRun, - timeProvider: opts.timeProvider, + source: source, + consentM: cm, + minAge: int64(minAge), + dryRun: dryRun, + timeProvider: opts.timeProvider, baseServerURL: opts.baseServerURL, collectedDir: filepath.Join(opts.cachePath, constants.LocalFolder), diff --git a/internal/uploader/uploader_test.go b/internal/uploader/uploader_test.go index 720c2e4..f57a5ff 100644 --- a/internal/uploader/uploader_test.go +++ b/internal/uploader/uploader_test.go @@ -1,10 +1,9 @@ package uploader_test import ( - "bytes" "encoding/json" "fmt" - "io/fs" + "math" "net/http" "net/http/httptest" "os" @@ -12,6 +11,7 @@ import ( "testing" "github.com/stretchr/testify/require" + "github.com/ubuntu/ubuntu-insights/internal/constants" "github.com/ubuntu/ubuntu-insights/internal/fileutils" "github.com/ubuntu/ubuntu-insights/internal/testutils" "github.com/ubuntu/ubuntu-insights/internal/uploader" @@ -21,6 +21,7 @@ type reportType any var ( normal reportType = struct{ Content string }{Content: "normal content"} + optOut = constants.OptOutJSON badContent = `bad content` ) @@ -32,19 +33,51 @@ func (m mockTimeProvider) NowUnix() int64 { return m.currentTime } -func TestUpload(t *testing.T) { +var ( + cmSErr = testConsentManager{sErr: fmt.Errorf("consent error")} + cmTrueSErr = testConsentManager{sState: true, gState: true, sErr: fmt.Errorf("consent error")} + cmGErr = testConsentManager{gErr: fmt.Errorf("consent error")} + cmTrueGErr = testConsentManager{gState: true, gErr: fmt.Errorf("consent error")} + cmTrue = testConsentManager{sState: true, gState: true} + cmFalse = testConsentManager{sState: false, gState: false} + cmSTrue = testConsentManager{sState: true, gState: false} + cmGTrue = testConsentManager{sState: false, gState: true} +) + +func TestNew(t *testing.T) { t.Parallel() - var ( - cmSErr = testConsentManager{sErr: fmt.Errorf("consent error")} - cmTrueSErr = testConsentManager{sState: true, gState: true, sErr: fmt.Errorf("consent error")} - cmGErr = testConsentManager{gErr: fmt.Errorf("consent error")} - cmTrueGErr = testConsentManager{gState: true, gErr: fmt.Errorf("consent error")} - cmTrue = testConsentManager{sState: true, gState: true} - cmFalse = testConsentManager{sState: false, gState: false} - cmSTrue = testConsentManager{sState: true, gState: false} - cmGTrue = testConsentManager{sState: false, gState: true} - ) + tests := map[string]struct { + cm testConsentManager + source string + minAge uint + dryRun bool + + wantErr bool + }{ + "Valid": {cm: cmTrue, source: "source", minAge: 5, dryRun: true}, + "Zero Min Age": {cm: cmTrue, source: "source", minAge: 0}, + + "Empty Source": {cm: cmTrue, source: "", wantErr: true}, + "Minage Overflow": {cm: cmTrue, source: "source", minAge: math.MaxUint64, wantErr: true}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + _, err := uploader.New(tc.cm, tc.source, tc.minAge, tc.dryRun) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + }) + } +} + +func TestUpload(t *testing.T) { + t.Parallel() const mockTime = 10 @@ -54,10 +87,12 @@ func TestUpload(t *testing.T) { serverResponse int serverOffline bool url string + invalidDir bool cm testConsentManager minAge uint dryRun bool + force bool wantErr bool }{ @@ -78,11 +113,20 @@ func TestUpload(t *testing.T) { "Consent Manager Global True, Source False": {localFiles: map[string]reportType{"1.json": normal}, cm: cmGTrue, serverResponse: http.StatusOK}, "Consent Manager Global False, Source True": {localFiles: map[string]reportType{"1.json": normal}, cm: cmSTrue, serverResponse: http.StatusOK}, + "Force CM False": {localFiles: map[string]reportType{"1.json": normal}, cm: cmFalse, force: true, serverResponse: http.StatusOK}, + "Force Min Age": {localFiles: map[string]reportType{"1.json": normal, "9.json": normal}, cm: cmTrue, minAge: 5, force: true, serverResponse: http.StatusOK}, + "Force Duplicate": {localFiles: map[string]reportType{"1.json": normal}, uploadedFiles: map[string]reportType{"1.json": badContent}, cm: cmTrue, force: true, serverResponse: http.StatusOK}, + + "OptOut Payload CM True": {localFiles: map[string]reportType{"1.json": optOut}, cm: cmTrue, serverResponse: http.StatusOK}, + "OptOut Payload CM False": {localFiles: map[string]reportType{"1.json": optOut}, cm: cmFalse, serverResponse: http.StatusOK}, + "Dry run": {localFiles: map[string]reportType{"1.json": normal}, cm: cmTrue, dryRun: true}, "Bad URL": {localFiles: map[string]reportType{"1.json": normal}, cm: cmTrue, url: "http://a b.com/", wantErr: true}, "Bad Response": {localFiles: map[string]reportType{"1.json": normal}, cm: cmTrue, serverResponse: http.StatusForbidden}, "Offline Server": {localFiles: map[string]reportType{"1.json": normal}, cm: cmTrue, serverOffline: true}, + + "Invalid Directory": {localFiles: map[string]reportType{"1.json": normal}, cm: cmTrue, invalidDir: true, wantErr: true}, } for name, tc := range tests { @@ -92,26 +136,31 @@ func TestUpload(t *testing.T) { dir := setupTmpDir(t, tc.localFiles, tc.uploadedFiles, tc.dummy) if !tc.serverOffline { - status := statusHandler(tc.serverResponse) - ts := httptest.NewServer(&status) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tc.serverResponse) + })) t.Cleanup(func() { ts.Close() }) if tc.url == "" { tc.url = ts.URL } } + if tc.invalidDir { + require.NoError(t, os.RemoveAll(filepath.Join(dir, "local")), "Setup: failed to remove local directory") + } + mgr, err := uploader.New(tc.cm, "source", tc.minAge, tc.dryRun, uploader.WithBaseServerURL(tc.url), uploader.WithCachePath(dir), uploader.WithTimeProvider(mockTimeProvider{currentTime: mockTime})) require.NoError(t, err, "Setup: failed to create new uploader manager") - err = mgr.Upload() + err = mgr.Upload(tc.force) if tc.wantErr { require.Error(t, err) } else { require.NoError(t, err) } - got, err := getDirResult(t, dir, 3) + got, err := testutils.GetDirContents(t, dir, 3) require.NoError(t, err) want := testutils.LoadWithUpdateFromGoldenYAML(t, got) require.EqualValues(t, want, got) @@ -121,10 +170,7 @@ func TestUpload(t *testing.T) { func setupTmpDir(t *testing.T, localFiles, uploadedFiles map[string]reportType, dummy bool) string { t.Helper() - - dir, err := os.MkdirTemp("", "uploader-test") - require.NoError(t, err, "Setup: failed to create temporary directory") - t.Cleanup(func() { os.RemoveAll(dir) }) + dir := t.TempDir() localDir := filepath.Join(dir, "local") uploadedDir := filepath.Join(dir, "uploaded") @@ -165,46 +211,6 @@ func writeFiles(t *testing.T, targetDir string, files map[string]reportType) { } } -func getDirResult(t *testing.T, dir string, maxDepth uint) (map[string]string, error) { - t.Helper() - - files := make(map[string]string) - - err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - - if path == dir { - return nil - } - - relPath, err := filepath.Rel(dir, path) - if err != nil { - return err - } - - depth := uint(len(filepath.SplitList(relPath))) - if depth > maxDepth { - return fmt.Errorf("max depth %d exceeded at %s", maxDepth, relPath) - } - - if !d.IsDir() { - content, err := os.ReadFile(path) - if err != nil { - return err - } - // Normalize content between windows and linux - content = bytes.ReplaceAll(content, []byte("\r\n"), []byte("\n")) - files[filepath.ToSlash(relPath)] = string(content) - } - - return nil - }) - - return files, err -} - type testConsentManager struct { sState bool gState bool @@ -218,9 +224,3 @@ func (m testConsentManager) GetConsentState(source string) (bool, error) { } return m.gState, m.gErr } - -type statusHandler int - -func (h *statusHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(int(*h)) -} From 66bef8dc62c4e802c15970f094f0923abb727361 Mon Sep 17 00:00:00 2001 From: kat <28567881+hk21702@users.noreply.github.com> Date: Mon, 27 Jan 2025 16:41:20 -0500 Subject: [PATCH 19/26] Minor test adjustments --- internal/consent/consent_test.go | 11 +++-------- internal/testutils/files.go | 21 ++++++++------------- internal/uploader/uploader_test.go | 6 +++--- 3 files changed, 14 insertions(+), 24 deletions(-) diff --git a/internal/consent/consent_test.go b/internal/consent/consent_test.go index fc1568c..4393831 100644 --- a/internal/consent/consent_test.go +++ b/internal/consent/consent_test.go @@ -55,7 +55,6 @@ func TestGetConsentState(t *testing.T) { t.Parallel() dir, err := setupTmpConsentFiles(t, tc.globalFile) require.NoError(t, err, "Setup: failed to setup temporary consent files") - defer testutils.CleanupDir(t, dir) cm := consent.New(dir) got, err := cm.GetConsentState(tc.source) @@ -112,7 +111,6 @@ func TestSetConsentStates(t *testing.T) { t.Parallel() dir, err := setupTmpConsentFiles(t, tc.globalFile) require.NoError(t, err, "Setup: failed to setup temporary consent files") - defer testutils.CleanupDir(t, dir) cm := consent.New(dir) err = cm.SetConsentState(tc.writeSource, tc.writeState) @@ -140,18 +138,15 @@ func setupTmpConsentFiles(t *testing.T, globalFile string) (string, error) { // Setup temporary directory var err error - dir, err := os.MkdirTemp("", "consent-files") - if err != nil { - return dir, fmt.Errorf("failed to create temporary directory: %v", err) - } + dir := t.TempDir() - if err = testutils.CopyDir(filepath.Join("testdata", "consent_files"), dir); err != nil { + if err = testutils.CopyDir(t, filepath.Join("testdata", "consent_files"), dir); err != nil { return dir, fmt.Errorf("failed to copy testdata directory to temporary directory: %v", err) } // Setup globalFile if provided if globalFile != "" { - if err = testutils.CopyFile(filepath.Join(dir, globalFile), filepath.Join(dir, "consent.toml")); err != nil { + if err = testutils.CopyFile(t, filepath.Join(dir, globalFile), filepath.Join(dir, "consent.toml")); err != nil { return dir, fmt.Errorf("failed to copy requested global consent file: %v", err) } } diff --git a/internal/testutils/files.go b/internal/testutils/files.go index 681cb19..6ac9cb7 100644 --- a/internal/testutils/files.go +++ b/internal/testutils/files.go @@ -10,18 +10,12 @@ import ( "os" "path/filepath" "testing" - - "github.com/stretchr/testify/assert" ) -// CleanupDir removes the temporary directory including its contents. -func CleanupDir(t *testing.T, dir string) { +// CopyFile copies a file from source to destination. +func CopyFile(t *testing.T, src, dst string) error { t.Helper() - assert.NoError(t, os.RemoveAll(dir), "Cleanup: failed to remove temporary directory") -} -// CopyFile copies a file from source to destination. -func CopyFile(src, dst string) error { sourceFile, err := os.Open(src) if err != nil { return err @@ -39,8 +33,9 @@ func CopyFile(src, dst string) error { } // CopyDir copies the contents of a directory to another directory. -func CopyDir(srcDir, dstDir string) error { - return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error { +func CopyDir(t *testing.T, srcDir, dstDir string) error { + t.Helper() + return filepath.WalkDir(srcDir, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } @@ -49,10 +44,10 @@ func CopyDir(srcDir, dstDir string) error { return err } dstPath := filepath.Join(dstDir, relPath) - if info.IsDir() { - return os.MkdirAll(dstPath, info.Mode()) + if d.IsDir() { + return os.MkdirAll(dstPath, 0700) } - return CopyFile(path, dstPath) + return CopyFile(t, path, dstPath) }) } diff --git a/internal/uploader/uploader_test.go b/internal/uploader/uploader_test.go index f57a5ff..7dc4bd5 100644 --- a/internal/uploader/uploader_test.go +++ b/internal/uploader/uploader_test.go @@ -189,9 +189,9 @@ func setupTmpDir(t *testing.T, localFiles, uploadedFiles map[string]reportType, func copyDummyData(t *testing.T, sourceDir, dir, localDir, uploadedDir string) { t.Helper() - require.NoError(t, testutils.CopyDir(sourceDir, dir), "Setup: failed to copy dummy data to temporary directory") - require.NoError(t, testutils.CopyDir(sourceDir, localDir), "Setup: failed to copy dummy data to local") - require.NoError(t, testutils.CopyDir(sourceDir, uploadedDir), "Setup: failed to copy dummy data to uploaded") + require.NoError(t, testutils.CopyDir(t, sourceDir, dir), "Setup: failed to copy dummy data to temporary directory") + require.NoError(t, testutils.CopyDir(t, sourceDir, localDir), "Setup: failed to copy dummy data to local") + require.NoError(t, testutils.CopyDir(t, sourceDir, uploadedDir), "Setup: failed to copy dummy data to uploaded") } func writeFiles(t *testing.T, targetDir string, files map[string]reportType) { From 288f605b60eb16e104f8e433eb7387468e0e5bd4 Mon Sep 17 00:00:00 2001 From: kat <28567881+hk21702@users.noreply.github.com> Date: Wed, 29 Jan 2025 13:32:58 -0500 Subject: [PATCH 20/26] Refactor uploader and consent management: rename methods, remove unused code, and improve time provider abstraction --- internal/consent/consent.go | 33 +++-- internal/consent/consent_test.go | 57 +++++++- internal/consent/export_test.go | 4 +- ...alid_value_global_file,_valid_false_source | 0 ...valid_value_global_file,_valid_true_source | 0 .../golden/no_global_file,_valid_false_source | 0 .../golden/no_global_file,_valid_true_source | 0 .../golden/valid_false_global_file | 0 .../golden/valid_true_global_file | 0 ...valid_true_global_file,_valid_false_source | 0 .../valid_true_global_file,_valid_true_source | 0 .../golden/new_file,_write_global_false | 0 .../golden/new_file,_write_global_true | 0 .../golden/new_file,_write_source_false | 0 .../golden/new_file,_write_source_true | 0 .../overwrite_file,_write_diff_global_false | 0 .../overwrite_file,_write_diff_global_true | 0 .../overwrite_file,_write_diff_source_false | 0 .../overwrite_file,_write_diff_source_true | 0 .../golden/overwrite_file,_write_global_false | 0 .../golden/overwrite_file,_write_global_true | 0 .../golden/overwrite_file,_write_source_false | 0 .../golden/overwrite_file,_write_source_true | 0 internal/fileutils/fileutils.go | 10 -- internal/fileutils/fileutils_test.go | 36 ----- internal/reportutils/reportutils.go | 10 ++ internal/uploader/export_test.go | 13 ++ internal/uploader/internal_test.go | 14 +- internal/uploader/upload.go | 87 +++++++------ internal/uploader/uploader.go | 20 +-- internal/uploader/uploader_test.go | 123 ++++++++---------- 31 files changed, 210 insertions(+), 197 deletions(-) rename internal/consent/testdata/{TestGetConsentState => TestGetState}/golden/invalid_value_global_file,_valid_false_source (100%) rename internal/consent/testdata/{TestGetConsentState => TestGetState}/golden/invalid_value_global_file,_valid_true_source (100%) rename internal/consent/testdata/{TestGetConsentState => TestGetState}/golden/no_global_file,_valid_false_source (100%) rename internal/consent/testdata/{TestGetConsentState => TestGetState}/golden/no_global_file,_valid_true_source (100%) rename internal/consent/testdata/{TestGetConsentState => TestGetState}/golden/valid_false_global_file (100%) rename internal/consent/testdata/{TestGetConsentState => TestGetState}/golden/valid_true_global_file (100%) rename internal/consent/testdata/{TestGetConsentState => TestGetState}/golden/valid_true_global_file,_valid_false_source (100%) rename internal/consent/testdata/{TestGetConsentState => TestGetState}/golden/valid_true_global_file,_valid_true_source (100%) rename internal/consent/testdata/{TestSetConsentStates => TestSetState}/golden/new_file,_write_global_false (100%) rename internal/consent/testdata/{TestSetConsentStates => TestSetState}/golden/new_file,_write_global_true (100%) rename internal/consent/testdata/{TestSetConsentStates => TestSetState}/golden/new_file,_write_source_false (100%) rename internal/consent/testdata/{TestSetConsentStates => TestSetState}/golden/new_file,_write_source_true (100%) rename internal/consent/testdata/{TestSetConsentStates => TestSetState}/golden/overwrite_file,_write_diff_global_false (100%) rename internal/consent/testdata/{TestSetConsentStates => TestSetState}/golden/overwrite_file,_write_diff_global_true (100%) rename internal/consent/testdata/{TestSetConsentStates => TestSetState}/golden/overwrite_file,_write_diff_source_false (100%) rename internal/consent/testdata/{TestSetConsentStates => TestSetState}/golden/overwrite_file,_write_diff_source_true (100%) rename internal/consent/testdata/{TestSetConsentStates => TestSetState}/golden/overwrite_file,_write_global_false (100%) rename internal/consent/testdata/{TestSetConsentStates => TestSetState}/golden/overwrite_file,_write_global_true (100%) rename internal/consent/testdata/{TestSetConsentStates => TestSetState}/golden/overwrite_file,_write_source_false (100%) rename internal/consent/testdata/{TestSetConsentStates => TestSetState}/golden/overwrite_file,_write_source_true (100%) diff --git a/internal/consent/consent.go b/internal/consent/consent.go index d13a145..b1c0749 100644 --- a/internal/consent/consent.go +++ b/internal/consent/consent.go @@ -30,12 +30,12 @@ func New(path string) *Manager { return &Manager{path: path} } -// GetConsentState gets the consent state for the given source. +// GetState gets the consent state for the given source. // If the source do not have a consent file, it will be considered as a false state. // If the source is an empty string, then the global consent state will be returned. // If the target consent file does not exist, it will not be created. -func (cm Manager) GetConsentState(source string) (bool, error) { - sourceConsent, err := readConsentFile(cm.getConsentFile(source)) +func (cm Manager) GetState(source string) (bool, error) { + sourceConsent, err := readFile(cm.getFile(source)) if err != nil { slog.Error("Error reading source consent file", "source", source, "error", err) return false, err @@ -46,20 +46,33 @@ func (cm Manager) GetConsentState(source string) (bool, error) { var consentSourceFilePattern = `%s` + constants.ConsentSourceBaseSeparator + constants.GlobalFileName -// SetConsentState updates the consent state for the given source. +// SetState updates the consent state for the given source. // If the source is an empty string, then the global consent state will be set. // If the target consent file does not exist, it will be created. -func (cm Manager) SetConsentState(source string, state bool) (err error) { +func (cm Manager) SetState(source string, state bool) (err error) { defer decorate.OnError(&err, "could not set consent state") consent := consentFile{ConsentState: state} - return consent.write(cm.getConsentFile(source)) + return consent.write(cm.getFile(source)) } -// getConsentFile returns the expected path to the consent file for the given source. +// HasConsent returns true if there is consent for the given source, based on the hierarchy rules. +// If the source has a consent file, its value is returned. +// Otherwise, the global consent state is returned. +func (cm Manager) HasConsent(source string) (bool, error) { + consent, err := cm.GetState(source) + if err != nil { + slog.Warn("Could not get source specific consent state, falling back to global consent state", "source", source, "error", err) + return cm.GetState("") + } + + return consent, nil +} + +// getFile returns the expected path to the consent file for the given source. // If source is blank, it returns the path to the global consent file. // It does not check if the file exists, or if it is valid. -func (cm Manager) getConsentFile(source string) string { +func (cm Manager) getFile(source string) string { p := filepath.Join(cm.path, constants.GlobalFileName) if source != "" { p = filepath.Join(cm.path, fmt.Sprintf(consentSourceFilePattern, source)) @@ -69,7 +82,7 @@ func (cm Manager) getConsentFile(source string) string { } // getSourceConsentFiles returns a map of all paths to validly named consent files in the folder, other than the global file. -func (cm Manager) getConsentFiles() (map[string]string, error) { +func (cm Manager) getFiles() (map[string]string, error) { sourceFiles := make(map[string]string) entries, err := os.ReadDir(cm.path) @@ -94,7 +107,7 @@ func (cm Manager) getConsentFiles() (map[string]string, error) { return sourceFiles, nil } -func readConsentFile(path string) (consentFile, error) { +func readFile(path string) (consentFile, error) { var consent consentFile _, err := toml.DecodeFile(path, &consent) slog.Debug("Read consent file", "file", path, "consent", consent.ConsentState) diff --git a/internal/consent/consent_test.go b/internal/consent/consent_test.go index 4393831..612fdb5 100644 --- a/internal/consent/consent_test.go +++ b/internal/consent/consent_test.go @@ -11,7 +11,7 @@ import ( "github.com/ubuntu/ubuntu-insights/internal/testutils" ) -func TestGetConsentState(t *testing.T) { +func TestGetState(t *testing.T) { t.Parallel() tests := map[string]struct { @@ -57,7 +57,7 @@ func TestGetConsentState(t *testing.T) { require.NoError(t, err, "Setup: failed to setup temporary consent files") cm := consent.New(dir) - got, err := cm.GetConsentState(tc.source) + got, err := cm.GetState(tc.source) if tc.wantErr { require.Error(t, err, "expected an error but got none") return @@ -70,7 +70,7 @@ func TestGetConsentState(t *testing.T) { } } -func TestSetConsentStates(t *testing.T) { +func TestSetState(t *testing.T) { t.Parallel() tests := map[string]struct { @@ -113,7 +113,7 @@ func TestSetConsentStates(t *testing.T) { require.NoError(t, err, "Setup: failed to setup temporary consent files") cm := consent.New(dir) - err = cm.SetConsentState(tc.writeSource, tc.writeState) + err = cm.SetState(tc.writeSource, tc.writeState) if tc.wantErr { require.Error(t, err, "expected an error but got none") return @@ -133,6 +133,55 @@ func TestSetConsentStates(t *testing.T) { } } +func TestHasConsent(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + source string + + globalFile string + + want bool + wantErr bool + }{ + "True Global-True Source": {source: "valid_true", globalFile: "valid_true-consent.toml", want: true}, + "True Global-False Source": {source: "valid_false", globalFile: "valid_true-consent.toml", want: false}, + "True Global-Invalid Value Source": {source: "invalid_value", globalFile: "valid_true-consent.toml", want: true}, + "True Global-Invalid File Source": {source: "invalid_file", globalFile: "valid_true-consent.toml", want: true}, + "True Global-Not A File Source": {source: "not_a_file", globalFile: "valid_true-consent.toml", want: true}, + + "False Global-True Source": {source: "valid_true", globalFile: "valid_false-consent.toml", want: true}, + "False Global-False Source": {source: "valid_false", globalFile: "valid_false-consent.toml", want: false}, + "False Global-Invalid Value Source": {source: "invalid_value", globalFile: "valid_false-consent.toml", want: false}, + "False Global-Invalid File Source": {source: "invalid_file", globalFile: "valid_false-consent.toml", want: false}, + "False Global-Not A File Source": {source: "not_a_file", globalFile: "valid_false-consent.toml", want: false}, + + "No Global-True Source": {source: "valid_true", want: true}, + "No Global-False Source": {source: "valid_false", want: false}, + "No Global-Invalid Value Source": {source: "invalid_value", wantErr: true}, + "No Global-Invalid File Source": {source: "invalid_file", wantErr: true}, + "No Global-Not A File Source": {source: "not_a_file", wantErr: true}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + dir, err := setupTmpConsentFiles(t, tc.globalFile) + require.NoError(t, err, "Setup: failed to setup temporary consent files") + cm := consent.New(dir) + + got, err := cm.HasConsent(tc.source) + if tc.wantErr { + require.Error(t, err, "expected an error but got none") + return + } + require.NoError(t, err, "got an unexpected error") + + require.Equal(t, tc.want, got, "HasConsent should return expected consent state") + }) + } +} + func setupTmpConsentFiles(t *testing.T, globalFile string) (string, error) { t.Helper() diff --git a/internal/consent/export_test.go b/internal/consent/export_test.go index f887256..8f88838 100644 --- a/internal/consent/export_test.go +++ b/internal/consent/export_test.go @@ -4,14 +4,14 @@ package consent // It does not get the global consent state. // If continueOnErr is true, it will continue to the next source if an error occurs. func (cm Manager) GetAllSourceConsentStates(continueOnErr bool) (map[string]bool, error) { - p, err := cm.getConsentFiles() + p, err := cm.getFiles() if err != nil { return nil, err } consentStates := make(map[string]bool) for source, path := range p { - consent, err := readConsentFile(path) + consent, err := readFile(path) if err != nil && !continueOnErr { return nil, err } diff --git a/internal/consent/testdata/TestGetConsentState/golden/invalid_value_global_file,_valid_false_source b/internal/consent/testdata/TestGetState/golden/invalid_value_global_file,_valid_false_source similarity index 100% rename from internal/consent/testdata/TestGetConsentState/golden/invalid_value_global_file,_valid_false_source rename to internal/consent/testdata/TestGetState/golden/invalid_value_global_file,_valid_false_source diff --git a/internal/consent/testdata/TestGetConsentState/golden/invalid_value_global_file,_valid_true_source b/internal/consent/testdata/TestGetState/golden/invalid_value_global_file,_valid_true_source similarity index 100% rename from internal/consent/testdata/TestGetConsentState/golden/invalid_value_global_file,_valid_true_source rename to internal/consent/testdata/TestGetState/golden/invalid_value_global_file,_valid_true_source diff --git a/internal/consent/testdata/TestGetConsentState/golden/no_global_file,_valid_false_source b/internal/consent/testdata/TestGetState/golden/no_global_file,_valid_false_source similarity index 100% rename from internal/consent/testdata/TestGetConsentState/golden/no_global_file,_valid_false_source rename to internal/consent/testdata/TestGetState/golden/no_global_file,_valid_false_source diff --git a/internal/consent/testdata/TestGetConsentState/golden/no_global_file,_valid_true_source b/internal/consent/testdata/TestGetState/golden/no_global_file,_valid_true_source similarity index 100% rename from internal/consent/testdata/TestGetConsentState/golden/no_global_file,_valid_true_source rename to internal/consent/testdata/TestGetState/golden/no_global_file,_valid_true_source diff --git a/internal/consent/testdata/TestGetConsentState/golden/valid_false_global_file b/internal/consent/testdata/TestGetState/golden/valid_false_global_file similarity index 100% rename from internal/consent/testdata/TestGetConsentState/golden/valid_false_global_file rename to internal/consent/testdata/TestGetState/golden/valid_false_global_file diff --git a/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file b/internal/consent/testdata/TestGetState/golden/valid_true_global_file similarity index 100% rename from internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file rename to internal/consent/testdata/TestGetState/golden/valid_true_global_file diff --git a/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_false_source b/internal/consent/testdata/TestGetState/golden/valid_true_global_file,_valid_false_source similarity index 100% rename from internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_false_source rename to internal/consent/testdata/TestGetState/golden/valid_true_global_file,_valid_false_source diff --git a/internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_true_source b/internal/consent/testdata/TestGetState/golden/valid_true_global_file,_valid_true_source similarity index 100% rename from internal/consent/testdata/TestGetConsentState/golden/valid_true_global_file,_valid_true_source rename to internal/consent/testdata/TestGetState/golden/valid_true_global_file,_valid_true_source diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_false b/internal/consent/testdata/TestSetState/golden/new_file,_write_global_false similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_false rename to internal/consent/testdata/TestSetState/golden/new_file,_write_global_false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_true b/internal/consent/testdata/TestSetState/golden/new_file,_write_global_true similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_global_true rename to internal/consent/testdata/TestSetState/golden/new_file,_write_global_true diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_false b/internal/consent/testdata/TestSetState/golden/new_file,_write_source_false similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_false rename to internal/consent/testdata/TestSetState/golden/new_file,_write_source_false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_true b/internal/consent/testdata/TestSetState/golden/new_file,_write_source_true similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/new_file,_write_source_true rename to internal/consent/testdata/TestSetState/golden/new_file,_write_source_true diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_false b/internal/consent/testdata/TestSetState/golden/overwrite_file,_write_diff_global_false similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_false rename to internal/consent/testdata/TestSetState/golden/overwrite_file,_write_diff_global_false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_true b/internal/consent/testdata/TestSetState/golden/overwrite_file,_write_diff_global_true similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_global_true rename to internal/consent/testdata/TestSetState/golden/overwrite_file,_write_diff_global_true diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_false b/internal/consent/testdata/TestSetState/golden/overwrite_file,_write_diff_source_false similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_false rename to internal/consent/testdata/TestSetState/golden/overwrite_file,_write_diff_source_false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_true b/internal/consent/testdata/TestSetState/golden/overwrite_file,_write_diff_source_true similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_diff_source_true rename to internal/consent/testdata/TestSetState/golden/overwrite_file,_write_diff_source_true diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_false b/internal/consent/testdata/TestSetState/golden/overwrite_file,_write_global_false similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_false rename to internal/consent/testdata/TestSetState/golden/overwrite_file,_write_global_false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_true b/internal/consent/testdata/TestSetState/golden/overwrite_file,_write_global_true similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_global_true rename to internal/consent/testdata/TestSetState/golden/overwrite_file,_write_global_true diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_false b/internal/consent/testdata/TestSetState/golden/overwrite_file,_write_source_false similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_false rename to internal/consent/testdata/TestSetState/golden/overwrite_file,_write_source_false diff --git a/internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_true b/internal/consent/testdata/TestSetState/golden/overwrite_file,_write_source_true similarity index 100% rename from internal/consent/testdata/TestSetConsentStates/golden/overwrite_file,_write_source_true rename to internal/consent/testdata/TestSetState/golden/overwrite_file,_write_source_true diff --git a/internal/fileutils/fileutils.go b/internal/fileutils/fileutils.go index b07bfde..b113a2f 100644 --- a/internal/fileutils/fileutils.go +++ b/internal/fileutils/fileutils.go @@ -2,7 +2,6 @@ package fileutils import ( - "errors" "fmt" "log/slog" "os" @@ -37,12 +36,3 @@ func AtomicWrite(path string, data []byte) error { } return nil } - -// FileExists checks if a file exists. -func FileExists(path string) (bool, error) { - _, err := os.Stat(path) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return false, err - } - return !errors.Is(err, os.ErrNotExist), nil -} diff --git a/internal/fileutils/fileutils_test.go b/internal/fileutils/fileutils_test.go index ec6f776..15c2b96 100644 --- a/internal/fileutils/fileutils_test.go +++ b/internal/fileutils/fileutils_test.go @@ -74,39 +74,3 @@ func TestAtomicWrite(t *testing.T) { }) } } - -func TestFileExists(t *testing.T) { - t.Parallel() - - tests := map[string]struct { - fileExists bool - - wantExists bool - wantError bool - }{ - "Returns_true_when_file_exists": {fileExists: true, wantExists: true}, - "Returns_false_when_file_does_not_exist": {fileExists: false, wantExists: false}, - "Returns_false_when_parent_directory_does_not_exist": {fileExists: false, wantExists: false}, - } - - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - t.Parallel() - - tempDir := t.TempDir() - path := filepath.Join(tempDir, "file") - if tc.fileExists { - err := fileutils.AtomicWrite(path, []byte("")) - require.NoError(t, err, "Setup: AtomicWrite should not return an error") - } - - exists, err := fileutils.FileExists(path) - if tc.wantError { - require.Error(t, err, "FileExists should return an error") - } else { - require.NoError(t, err, "FileExists should not return an error") - } - require.Equal(t, tc.wantExists, exists, "FileExists should return the expected result") - }) - } -} diff --git a/internal/reportutils/reportutils.go b/internal/reportutils/reportutils.go index f08a695..932d99c 100644 --- a/internal/reportutils/reportutils.go +++ b/internal/reportutils/reportutils.go @@ -1,6 +1,8 @@ // Package reportutils provides utility functions for handling reports. package reportutils +// package report + import ( "errors" "log/slog" @@ -16,6 +18,8 @@ import ( // ErrInvalidPeriod is returned when a function requiring a period, received an invalid, period that isn't a non-negative integer. var ErrInvalidPeriod = errors.New("invalid period, period should be a positive integer") +// type Report struct{} + // GetPeriodStart returns the start of the period window for a given period in seconds. func GetPeriodStart(period int) (int64, error) { if period <= 0 { @@ -31,6 +35,8 @@ func GetReportTime(path string) (int64, error) { return strconv.ParseInt(strings.TrimSuffix(fileName, filepath.Ext(fileName)), 10, 64) } +// Path is a method on Report + // GetReportPath returns the path for the most recent report within a period window, returning an empty string if no report is found. // Not inclusive of the period end (periodStart + period). // @@ -85,6 +91,8 @@ func GetReportPath(dir string, time int64, period int) (string, error) { return mostRecentReportPath, nil } +// -> map[int64]Report + // GetReports returns the paths for the latest report within each period window for a given directory. // The key of the map is the start of the period window, and the report timestamp. // @@ -131,6 +139,8 @@ func GetReports(dir string, period int) (map[int64]int64, error) { return reports, nil } +// reports.GetAll(dir string) ([]Report, error) + // GetAllReports returns the filename for all reports within a given directory, which match the expected pattern. // Does not traverse subdirectories. func GetAllReports(dir string) ([]string, error) { diff --git a/internal/uploader/export_test.go b/internal/uploader/export_test.go index d34d9f5..a9a4c3f 100644 --- a/internal/uploader/export_test.go +++ b/internal/uploader/export_test.go @@ -1,17 +1,30 @@ package uploader +import "time" + +type MockTimeProvider struct { + CurrentTime int64 +} + +func (m MockTimeProvider) Now() time.Time { + return time.Unix(m.CurrentTime, 0) +} + +// WithCachePath sets the cache path for the uploader. func WithCachePath(path string) Options { return func(o *options) { o.cachePath = path } } +// WithBaseServerURL sets the base server URL for the uploader. func WithBaseServerURL(url string) Options { return func(o *options) { o.baseServerURL = url } } +// WithTimeProvider sets the time provider for the uploader. func WithTimeProvider(tp timeProvider) Options { return func(o *options) { o.timeProvider = tp diff --git a/internal/uploader/internal_test.go b/internal/uploader/internal_test.go index d7a0147..f9882d3 100644 --- a/internal/uploader/internal_test.go +++ b/internal/uploader/internal_test.go @@ -12,14 +12,6 @@ import ( "github.com/ubuntu/ubuntu-insights/internal/fileutils" ) -type mockTimeProvider struct { - currentTime int64 -} - -func (m mockTimeProvider) NowUnix() int64 { - return m.currentTime -} - func TestUploadBadFile(t *testing.T) { t.Parallel() basicContent := `{"Content":true, "string": "string"}` @@ -59,11 +51,11 @@ func TestUploadBadFile(t *testing.T) { dir := t.TempDir() - um := &Manager{ + um := &Uploader{ collectedDir: filepath.Join(dir, constants.LocalFolder), uploadedDir: filepath.Join(dir, constants.UploadedFolder), minAge: 0, - timeProvider: mockTimeProvider{currentTime: 0}, + timeProvider: MockTimeProvider{CurrentTime: 0}, } require.NoError(t, os.Mkdir(um.collectedDir, 0750), "Setup: failed to create uploaded folder") @@ -114,7 +106,7 @@ func TestMoveReport(t *testing.T) { t.Parallel() dir := t.TempDir() - um := &Manager{ + um := &Uploader{ collectedDir: filepath.Join(dir, constants.LocalFolder), uploadedDir: filepath.Join(dir, constants.UploadedFolder), } diff --git a/internal/uploader/upload.go b/internal/uploader/upload.go index cc15bdd..2f181e3 100644 --- a/internal/uploader/upload.go +++ b/internal/uploader/upload.go @@ -3,6 +3,7 @@ package uploader import ( "bytes" "encoding/json" + "errors" "fmt" "log/slog" "net/http" @@ -18,19 +19,19 @@ import ( "github.com/ubuntu/ubuntu-insights/internal/reportutils" ) +var ( + // ErrReportNotMature is returned when a report is not mature enough to be uploaded. + ErrReportNotMature = errors.New("report is not mature enough to be uploaded") +) + // Upload uploads the reports corresponding to the source to the configured server. // Does not do duplicate checks. -func (um Manager) Upload(force bool) error { +func (um Uploader) Upload(force bool) error { slog.Debug("Uploading reports") - gConsent, err := um.consentM.GetConsentState("") + consent, err := um.consentM.HasConsent(um.source) if err != nil { - return fmt.Errorf("upload failed to get global consent state: %v", err) - } - - sConsent, err := um.consentM.GetConsentState(um.source) - if err != nil { - return fmt.Errorf("upload failed to get source consent state: %v", err) + return fmt.Errorf("upload failed to get consent state: %v", err) } reports, err := reportutils.GetAllReports(um.collectedDir) @@ -44,14 +45,17 @@ func (um Manager) Upload(force bool) error { } var wg sync.WaitGroup - for _, file := range reports { + for _, name := range reports { wg.Add(1) - go func(file string) { + go func(name string) { defer wg.Done() - if err := um.upload(file, url, gConsent && sConsent, force); err != nil { - slog.Warn("Failed to upload report", "file", file, "source", um.source, "error", err) + err := um.upload(name, url, consent, force) + if errors.Is(err, ErrReportNotMature) { + slog.Debug("Skipped report upload, not mature enough", "file", name, "source", um.source) + } else if err != nil { + slog.Warn("Failed to upload report", "file", name, "source", um.source, "error", err) } - }(file) + }(name) } wg.Wait() @@ -60,28 +64,31 @@ func (um Manager) Upload(force bool) error { // upload uploads an individual report to the server. It returns an error if the report is not mature enough to be uploaded, or if the upload fails. // It also moves the report to the uploaded directory after a successful upload. -func (um Manager) upload(fName, url string, consent, force bool) error { - slog.Debug("Uploading report", "file", fName, "consent", consent, "force", force) +func (um Uploader) upload(name, url string, consent, force bool) error { + slog.Debug("Uploading report", "file", name, "consent", consent, "force", force) - ts, err := reportutils.GetReportTime(fName) + // TODO… pass the Report object directly. + ts, err := reportutils.GetReportTime(name) if err != nil { return fmt.Errorf("failed to parse report time from filename: %v", err) } - if ts > um.timeProvider.NowUnix()-um.minAge && !force { - return fmt.Errorf("report is not mature enough to be uploaded") + if um.timeProvider.Now().Add(time.Duration(-um.minAge)*time.Second).Before(time.Unix(ts, 0)) && !force { + return ErrReportNotMature } // Check for duplicate reports. - fileExists, err := fileutils.FileExists(filepath.Join(um.uploadedDir, fName)) - if err != nil { + _, err = os.Stat(filepath.Join(um.uploadedDir, name)) + if err != nil && !os.IsNotExist(err) { return fmt.Errorf("failed to check if report has already been uploaded: %v", err) } - if fileExists && !force { + if err == nil && !force { + // TODO: What to do with the original file? Should we clean it up? + // Should we move it elsewhere for investigation in a "tmp" and clean it afterwards? return fmt.Errorf("report has already been uploaded") } - origData, err := um.readPayload(fName) + origData, err := um.readJSON(name) if err != nil { return fmt.Errorf("failed to get payload: %v", err) } @@ -100,11 +107,12 @@ func (um Manager) upload(fName, url string, consent, force bool) error { } // Move report first to avoid the situation where the report is sent, but not marked as sent. - if err := um.moveReport(filepath.Join(um.uploadedDir, fName), filepath.Join(um.collectedDir, fName), data); err != nil { + // TODO: maybe a method on Reports ? + if err := um.moveReport(filepath.Join(um.uploadedDir, name), filepath.Join(um.collectedDir, name), data); err != nil { return fmt.Errorf("failed to move report after uploading: %v", err) } if err := send(url, data); err != nil { - if moveErr := um.moveReport(filepath.Join(um.collectedDir, fName), filepath.Join(um.uploadedDir, fName), origData); moveErr != nil { + if moveErr := um.moveReport(filepath.Join(um.collectedDir, name), filepath.Join(um.uploadedDir, name), origData); moveErr != nil { return fmt.Errorf("failed to send data: %v, and failed to restore the original report: %v", err, moveErr) } return fmt.Errorf("failed to send data: %v", err) @@ -113,7 +121,7 @@ func (um Manager) upload(fName, url string, consent, force bool) error { return nil } -func (um Manager) getURL() (string, error) { +func (um Uploader) getURL() (string, error) { u, err := url.Parse(um.baseServerURL) if err != nil { return "", fmt.Errorf("failed to parse base server URL %s: %v", um.baseServerURL, err) @@ -122,37 +130,32 @@ func (um Manager) getURL() (string, error) { return u.String(), nil } -func (um Manager) readPayload(file string) ([]byte, error) { - var jsonData map[string]interface{} - +// readJSON reads the JSON data from the report file. +func (um Uploader) readJSON(file string) ([]byte, error) { // Read the report file data, err := os.ReadFile(path.Join(um.collectedDir, file)) if err != nil { return nil, fmt.Errorf("failed to read report file: %v", err) } - // Remarshal the JSON data to ensure it is valid - if err := json.Unmarshal(data, &jsonData); err != nil { - return nil, fmt.Errorf("failed to unmarshal JSON data: %v", err) - } - - data, err = json.Marshal(jsonData) - if err != nil { - return nil, fmt.Errorf("failed to marshal JSON data: %v", err) + if !json.Valid(data) { + return nil, fmt.Errorf("invalid JSON data in report file") } return data, nil } -// moveReport writes the data to the writePath, and removes the matching file from the removePath. -func (um Manager) moveReport(writePath, removePath string, data []byte) error { - err := fileutils.AtomicWrite(writePath, data) - if err != nil { +func (um Uploader) moveReport(writePath, removePath string, data []byte) error { + // (Report).MarkAsProcessed(data) + // (Report).UndoProcessed + // moveReport writes the data to the writePath, and removes the matching file from the removePath. + // dest, src, data + + if err := fileutils.AtomicWrite(writePath, data); err != nil { return fmt.Errorf("failed to write report: %v", err) } - err = os.Remove(removePath) - if err != nil { + if err := os.Remove(removePath); err != nil { return fmt.Errorf("failed to remove report: %v", err) } diff --git a/internal/uploader/uploader.go b/internal/uploader/uploader.go index c304583..2469319 100644 --- a/internal/uploader/uploader.go +++ b/internal/uploader/uploader.go @@ -13,17 +13,17 @@ import ( ) type timeProvider interface { - NowUnix() int64 + Now() time.Time } type realTimeProvider struct{} -func (realTimeProvider) NowUnix() int64 { - return time.Now().Unix() +func (realTimeProvider) Now() time.Time { + return time.Now() } -// Manager is an abstraction of the uploader component. -type Manager struct { +// Uploader is an abstraction of the uploader component. +type Uploader struct { source string consentM consentManager minAge int64 @@ -46,19 +46,19 @@ type options struct { type Options func(*options) type consentManager interface { - GetConsentState(source string) (bool, error) + HasConsent(source string) (bool, error) } // New returns a new UploaderManager. -func New(cm consentManager, source string, minAge uint, dryRun bool, args ...Options) (Manager, error) { +func New(cm consentManager, source string, minAge uint, dryRun bool, args ...Options) (Uploader, error) { slog.Debug("Creating new uploader manager", "source", source, "minAge", minAge, "dryRun", dryRun) if source == "" { - return Manager{}, fmt.Errorf("source cannot be an empty string") + return Uploader{}, fmt.Errorf("source cannot be an empty string") } if minAge > math.MaxInt64 { - return Manager{}, fmt.Errorf("min age %d is too large, would overflow", minAge) + return Uploader{}, fmt.Errorf("min age %d is too large, would overflow", minAge) } opts := options{ @@ -70,7 +70,7 @@ func New(cm consentManager, source string, minAge uint, dryRun bool, args ...Opt opt(&opts) } - return Manager{ + return Uploader{ source: source, consentM: cm, minAge: int64(minAge), diff --git a/internal/uploader/uploader_test.go b/internal/uploader/uploader_test.go index 7dc4bd5..2a4d3d1 100644 --- a/internal/uploader/uploader_test.go +++ b/internal/uploader/uploader_test.go @@ -25,48 +25,36 @@ var ( badContent = `bad content` ) -type mockTimeProvider struct { - currentTime int64 -} - -func (m mockTimeProvider) NowUnix() int64 { - return m.currentTime -} - var ( - cmSErr = testConsentManager{sErr: fmt.Errorf("consent error")} - cmTrueSErr = testConsentManager{sState: true, gState: true, sErr: fmt.Errorf("consent error")} - cmGErr = testConsentManager{gErr: fmt.Errorf("consent error")} - cmTrueGErr = testConsentManager{gState: true, gErr: fmt.Errorf("consent error")} - cmTrue = testConsentManager{sState: true, gState: true} - cmFalse = testConsentManager{sState: false, gState: false} - cmSTrue = testConsentManager{sState: true, gState: false} - cmGTrue = testConsentManager{sState: false, gState: true} + cTrue = testConsentChecker{consent: true} + cFalse = testConsentChecker{consent: false} + cErr = testConsentChecker{err: fmt.Errorf("consent error")} + cErrTrue = testConsentChecker{consent: true, err: fmt.Errorf("consent error")} ) func TestNew(t *testing.T) { t.Parallel() tests := map[string]struct { - cm testConsentManager - source string - minAge uint - dryRun bool + consent testConsentChecker + source string + minAge uint + dryRun bool wantErr bool }{ - "Valid": {cm: cmTrue, source: "source", minAge: 5, dryRun: true}, - "Zero Min Age": {cm: cmTrue, source: "source", minAge: 0}, + "Valid": {consent: cTrue, source: "source", minAge: 5, dryRun: true}, + "Zero Min Age": {consent: cTrue, source: "source", minAge: 0}, - "Empty Source": {cm: cmTrue, source: "", wantErr: true}, - "Minage Overflow": {cm: cmTrue, source: "source", minAge: math.MaxUint64, wantErr: true}, + "Empty Source": {consent: cTrue, source: "", wantErr: true}, + "Minage Overflow": {consent: cTrue, source: "source", minAge: math.MaxUint64, wantErr: true}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { t.Parallel() - _, err := uploader.New(tc.cm, tc.source, tc.minAge, tc.dryRun) + _, err := uploader.New(tc.consent, tc.source, tc.minAge, tc.dryRun) if tc.wantErr { require.Error(t, err) return @@ -89,44 +77,40 @@ func TestUpload(t *testing.T) { url string invalidDir bool - cm testConsentManager - minAge uint - dryRun bool - force bool + consent testConsentChecker + minAge uint + dryRun bool + force bool wantErr bool }{ - "No Reports": {cm: cmTrue, serverResponse: http.StatusOK}, - "No Reports with Dummy": {dummy: true, cm: cmTrue, serverResponse: http.StatusOK}, - "Single Upload": {localFiles: map[string]reportType{"1.json": normal}, cm: cmTrue, serverResponse: http.StatusOK}, - "Multi Upload": {localFiles: map[string]reportType{"1.json": normal, "5.json": normal}, cm: cmTrue, serverResponse: http.StatusOK}, - "Min Age": {localFiles: map[string]reportType{"1.json": normal, "9.json": normal}, cm: cmTrue, minAge: 5, serverResponse: http.StatusOK}, - "Future Timestamp": {localFiles: map[string]reportType{"1.json": normal, "11.json": normal}, cm: cmTrue, serverResponse: http.StatusOK}, - "Duplicate Upload": {localFiles: map[string]reportType{"1.json": normal}, uploadedFiles: map[string]reportType{"1.json": badContent}, cm: cmTrue, serverResponse: http.StatusAccepted}, - "Bad Content": {localFiles: map[string]reportType{"1.json": badContent}, cm: cmTrue, serverResponse: http.StatusOK}, - - "Consent Manager Source Error": {localFiles: map[string]reportType{"1.json": normal}, cm: cmSErr, serverResponse: http.StatusOK, wantErr: true}, - "Consent Manager Source Error with True": {localFiles: map[string]reportType{"1.json": normal}, cm: cmTrueSErr, serverResponse: http.StatusOK, wantErr: true}, - "Consent Manager Global Error": {localFiles: map[string]reportType{"1.json": normal}, cm: cmGErr, serverResponse: http.StatusOK, wantErr: true}, - "Consent Manager Global Error with True": {localFiles: map[string]reportType{"1.json": normal}, cm: cmTrueGErr, serverResponse: http.StatusOK, wantErr: true}, - "Consent Manager False": {localFiles: map[string]reportType{"1.json": normal}, cm: cmFalse, serverResponse: http.StatusOK}, - "Consent Manager Global True, Source False": {localFiles: map[string]reportType{"1.json": normal}, cm: cmGTrue, serverResponse: http.StatusOK}, - "Consent Manager Global False, Source True": {localFiles: map[string]reportType{"1.json": normal}, cm: cmSTrue, serverResponse: http.StatusOK}, - - "Force CM False": {localFiles: map[string]reportType{"1.json": normal}, cm: cmFalse, force: true, serverResponse: http.StatusOK}, - "Force Min Age": {localFiles: map[string]reportType{"1.json": normal, "9.json": normal}, cm: cmTrue, minAge: 5, force: true, serverResponse: http.StatusOK}, - "Force Duplicate": {localFiles: map[string]reportType{"1.json": normal}, uploadedFiles: map[string]reportType{"1.json": badContent}, cm: cmTrue, force: true, serverResponse: http.StatusOK}, - - "OptOut Payload CM True": {localFiles: map[string]reportType{"1.json": optOut}, cm: cmTrue, serverResponse: http.StatusOK}, - "OptOut Payload CM False": {localFiles: map[string]reportType{"1.json": optOut}, cm: cmFalse, serverResponse: http.StatusOK}, - - "Dry run": {localFiles: map[string]reportType{"1.json": normal}, cm: cmTrue, dryRun: true}, - - "Bad URL": {localFiles: map[string]reportType{"1.json": normal}, cm: cmTrue, url: "http://a b.com/", wantErr: true}, - "Bad Response": {localFiles: map[string]reportType{"1.json": normal}, cm: cmTrue, serverResponse: http.StatusForbidden}, - "Offline Server": {localFiles: map[string]reportType{"1.json": normal}, cm: cmTrue, serverOffline: true}, - - "Invalid Directory": {localFiles: map[string]reportType{"1.json": normal}, cm: cmTrue, invalidDir: true, wantErr: true}, + "No Reports": {consent: cTrue, serverResponse: http.StatusOK}, + "No Reports with Dummy": {dummy: true, consent: cTrue, serverResponse: http.StatusOK}, + "Single Upload": {localFiles: map[string]reportType{"1.json": normal}, consent: cTrue, serverResponse: http.StatusOK}, + "Multi Upload": {localFiles: map[string]reportType{"1.json": normal, "5.json": normal}, consent: cTrue, serverResponse: http.StatusOK}, + "Min Age": {localFiles: map[string]reportType{"1.json": normal, "9.json": normal}, consent: cTrue, minAge: 5, serverResponse: http.StatusOK}, + "Future Timestamp": {localFiles: map[string]reportType{"1.json": normal, "11.json": normal}, consent: cTrue, serverResponse: http.StatusOK}, + "Duplicate Upload": {localFiles: map[string]reportType{"1.json": normal}, uploadedFiles: map[string]reportType{"1.json": badContent}, consent: cTrue, serverResponse: http.StatusAccepted}, + "Bad Content": {localFiles: map[string]reportType{"1.json": badContent}, consent: cTrue, serverResponse: http.StatusOK}, + + "Consent Manager Source Error": {localFiles: map[string]reportType{"1.json": normal}, consent: cErr, serverResponse: http.StatusOK, wantErr: true}, + "Consent Manager Source Error with True": {localFiles: map[string]reportType{"1.json": normal}, consent: cErrTrue, serverResponse: http.StatusOK, wantErr: true}, + "Consent Manager False": {localFiles: map[string]reportType{"1.json": normal}, consent: cFalse, serverResponse: http.StatusOK}, + + "Force CM False": {localFiles: map[string]reportType{"1.json": normal}, consent: cFalse, force: true, serverResponse: http.StatusOK}, + "Force Min Age": {localFiles: map[string]reportType{"1.json": normal, "9.json": normal}, consent: cTrue, minAge: 5, force: true, serverResponse: http.StatusOK}, + "Force Duplicate": {localFiles: map[string]reportType{"1.json": normal}, uploadedFiles: map[string]reportType{"1.json": badContent}, consent: cTrue, force: true, serverResponse: http.StatusOK}, + + "OptOut Payload CM True": {localFiles: map[string]reportType{"1.json": optOut}, consent: cTrue, serverResponse: http.StatusOK}, + "OptOut Payload CM False": {localFiles: map[string]reportType{"1.json": optOut}, consent: cFalse, serverResponse: http.StatusOK}, + + "Dry run": {localFiles: map[string]reportType{"1.json": normal}, consent: cTrue, dryRun: true}, + + "Bad URL": {localFiles: map[string]reportType{"1.json": normal}, consent: cTrue, url: "http://a b.com/", wantErr: true}, + "Bad Response": {localFiles: map[string]reportType{"1.json": normal}, consent: cTrue, serverResponse: http.StatusForbidden}, + "Offline Server": {localFiles: map[string]reportType{"1.json": normal}, consent: cTrue, serverOffline: true}, + + "Invalid Directory": {localFiles: map[string]reportType{"1.json": normal}, consent: cTrue, invalidDir: true, wantErr: true}, } for name, tc := range tests { @@ -149,8 +133,8 @@ func TestUpload(t *testing.T) { require.NoError(t, os.RemoveAll(filepath.Join(dir, "local")), "Setup: failed to remove local directory") } - mgr, err := uploader.New(tc.cm, "source", tc.minAge, tc.dryRun, - uploader.WithBaseServerURL(tc.url), uploader.WithCachePath(dir), uploader.WithTimeProvider(mockTimeProvider{currentTime: mockTime})) + mgr, err := uploader.New(tc.consent, "source", tc.minAge, tc.dryRun, + uploader.WithBaseServerURL(tc.url), uploader.WithCachePath(dir), uploader.WithTimeProvider(uploader.MockTimeProvider{CurrentTime: mockTime})) require.NoError(t, err, "Setup: failed to create new uploader manager") err = mgr.Upload(tc.force) @@ -211,16 +195,11 @@ func writeFiles(t *testing.T, targetDir string, files map[string]reportType) { } } -type testConsentManager struct { - sState bool - gState bool - sErr error - gErr error +type testConsentChecker struct { + consent bool + err error } -func (m testConsentManager) GetConsentState(source string) (bool, error) { - if source != "" { - return m.sState, m.sErr - } - return m.gState, m.gErr +func (m testConsentChecker) HasConsent(source string) (bool, error) { + return m.consent, m.err } From 24e17725bbb1b8807861e278606b3c86f088d25d Mon Sep 17 00:00:00 2001 From: kat <28567881+hk21702@users.noreply.github.com> Date: Wed, 29 Jan 2025 18:04:33 -0500 Subject: [PATCH 21/26] Report package for handling report files --- internal/report/export_test.go | 10 + internal/report/report.go | 223 ++++++++++++++++++ .../report_test.go} | 84 ++++--- .../TestGetAll}/golden/empty_directory | 0 .../TestGetAll/golden/files_in_subdir | 12 + .../TestGetAll}/golden/invalid_file_extension | 0 .../TestGetAll}/golden/invalid_file_names | 0 .../golden/mix_of_valid_and_invalid | 18 ++ .../TestGetForPeriod/golden/empty_directory | 3 + .../TestGetForPeriod/golden/empty_subdir | 3 + .../TestGetForPeriod/golden/files_in_subdir | 3 + .../golden/invalid_file_extension | 3 + .../golden/invalid_file_names | 3 + .../golden/negative_timestamp | 3 + .../golden/not_inclusive_period | 3 + .../golden/specific_time_single_valid_report | 3 + .../TestGetPerPeriod}/golden/empty_directory | 0 .../TestGetPerPeriod}/golden/files_in_subdir | 0 .../TestGetPerPeriod/golden/get_all_reports | 28 +++ .../golden/get_newest_of_period | 4 + .../golden/invalid_file_extension | 0 .../golden/invalid_file_names | 0 .../golden/mix_of_valid_and_invalid | 4 + .../golden/multiple_consecutive_windows | 12 + .../golden/multiple_non-consecutive_windows | 12 + .../testdata/TestNew/golden/valid_report | 3 + .../TestNew/golden/valid_report_with_path | 3 + internal/reportutils/reportutils.go | 177 -------------- .../TestGetAllReports/golden/files_in_subdir | 2 - .../golden/mix_of_valid_and_invalid | 3 - .../TestGetReportPath/golden/empty_directory | 0 .../TestGetReportPath/golden/empty_subdir | 0 .../TestGetReportPath/golden/files_in_subdir | 0 .../golden/invalid_file_extension | 0 .../golden/invalid_file_names | 0 .../golden/negative_timestamp | 1 - .../golden/not_inclusive_period | 1 - .../golden/specific_time_single_valid_report | 1 - .../TestGetReportTime/golden/alt_extension | 1 - .../golden/valid_report_time | 1 - .../golden/valid_report_time_with_path | 1 - .../TestGetReports/golden/get_all_reports | 7 - .../golden/get_newest_of_period | 1 - .../golden/mix_of_valid_and_invalid | 1 - .../golden/multiple_consecutive_windows | 3 - .../golden/multiple_consequtive_windows | 3 - .../golden/multiple_non-consecutive_windows | 3 - .../golden/multiple_non-consequtive_windows | 3 - internal/uploader/internal_test.go | 19 +- internal/uploader/upload.go | 53 ++--- 50 files changed, 432 insertions(+), 286 deletions(-) create mode 100644 internal/report/export_test.go create mode 100644 internal/report/report.go rename internal/{reportutils/reportutils_test.go => report/report_test.go} (79%) rename internal/{reportutils/testdata/TestGetAllReports => report/testdata/TestGetAll}/golden/empty_directory (100%) create mode 100644 internal/report/testdata/TestGetAll/golden/files_in_subdir rename internal/{reportutils/testdata/TestGetAllReports => report/testdata/TestGetAll}/golden/invalid_file_extension (100%) rename internal/{reportutils/testdata/TestGetAllReports => report/testdata/TestGetAll}/golden/invalid_file_names (100%) create mode 100644 internal/report/testdata/TestGetAll/golden/mix_of_valid_and_invalid create mode 100644 internal/report/testdata/TestGetForPeriod/golden/empty_directory create mode 100644 internal/report/testdata/TestGetForPeriod/golden/empty_subdir create mode 100644 internal/report/testdata/TestGetForPeriod/golden/files_in_subdir create mode 100644 internal/report/testdata/TestGetForPeriod/golden/invalid_file_extension create mode 100644 internal/report/testdata/TestGetForPeriod/golden/invalid_file_names create mode 100644 internal/report/testdata/TestGetForPeriod/golden/negative_timestamp create mode 100644 internal/report/testdata/TestGetForPeriod/golden/not_inclusive_period create mode 100644 internal/report/testdata/TestGetForPeriod/golden/specific_time_single_valid_report rename internal/{reportutils/testdata/TestGetReports => report/testdata/TestGetPerPeriod}/golden/empty_directory (100%) rename internal/{reportutils/testdata/TestGetReports => report/testdata/TestGetPerPeriod}/golden/files_in_subdir (100%) create mode 100644 internal/report/testdata/TestGetPerPeriod/golden/get_all_reports create mode 100644 internal/report/testdata/TestGetPerPeriod/golden/get_newest_of_period rename internal/{reportutils/testdata/TestGetReports => report/testdata/TestGetPerPeriod}/golden/invalid_file_extension (100%) rename internal/{reportutils/testdata/TestGetReports => report/testdata/TestGetPerPeriod}/golden/invalid_file_names (100%) create mode 100644 internal/report/testdata/TestGetPerPeriod/golden/mix_of_valid_and_invalid create mode 100644 internal/report/testdata/TestGetPerPeriod/golden/multiple_consecutive_windows create mode 100644 internal/report/testdata/TestGetPerPeriod/golden/multiple_non-consecutive_windows create mode 100644 internal/report/testdata/TestNew/golden/valid_report create mode 100644 internal/report/testdata/TestNew/golden/valid_report_with_path delete mode 100644 internal/reportutils/reportutils.go delete mode 100644 internal/reportutils/testdata/TestGetAllReports/golden/files_in_subdir delete mode 100644 internal/reportutils/testdata/TestGetAllReports/golden/mix_of_valid_and_invalid delete mode 100644 internal/reportutils/testdata/TestGetReportPath/golden/empty_directory delete mode 100644 internal/reportutils/testdata/TestGetReportPath/golden/empty_subdir delete mode 100644 internal/reportutils/testdata/TestGetReportPath/golden/files_in_subdir delete mode 100644 internal/reportutils/testdata/TestGetReportPath/golden/invalid_file_extension delete mode 100644 internal/reportutils/testdata/TestGetReportPath/golden/invalid_file_names delete mode 100644 internal/reportutils/testdata/TestGetReportPath/golden/negative_timestamp delete mode 100644 internal/reportutils/testdata/TestGetReportPath/golden/not_inclusive_period delete mode 100644 internal/reportutils/testdata/TestGetReportPath/golden/specific_time_single_valid_report delete mode 100644 internal/reportutils/testdata/TestGetReportTime/golden/alt_extension delete mode 100644 internal/reportutils/testdata/TestGetReportTime/golden/valid_report_time delete mode 100644 internal/reportutils/testdata/TestGetReportTime/golden/valid_report_time_with_path delete mode 100644 internal/reportutils/testdata/TestGetReports/golden/get_all_reports delete mode 100644 internal/reportutils/testdata/TestGetReports/golden/get_newest_of_period delete mode 100644 internal/reportutils/testdata/TestGetReports/golden/mix_of_valid_and_invalid delete mode 100644 internal/reportutils/testdata/TestGetReports/golden/multiple_consecutive_windows delete mode 100644 internal/reportutils/testdata/TestGetReports/golden/multiple_consequtive_windows delete mode 100644 internal/reportutils/testdata/TestGetReports/golden/multiple_non-consecutive_windows delete mode 100644 internal/reportutils/testdata/TestGetReports/golden/multiple_non-consequtive_windows diff --git a/internal/report/export_test.go b/internal/report/export_test.go new file mode 100644 index 0000000..6a8629f --- /dev/null +++ b/internal/report/export_test.go @@ -0,0 +1,10 @@ +package report + +type ReportStash = reportStash + +// ReportStash returns the reportStash of the report. +// +//nolint:revive // This is a false positive as we returned a typed alias and not the private type. +func (r Report) ReportStash() ReportStash { + return r.reportStash +} diff --git a/internal/report/report.go b/internal/report/report.go new file mode 100644 index 0000000..eb22687 --- /dev/null +++ b/internal/report/report.go @@ -0,0 +1,223 @@ +// Package report provides utility functions for handling reports. +package report + +// package report + +import ( + "encoding/json" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/ubuntu/ubuntu-insights/internal/constants" +) + +var ( + // ErrInvalidPeriod is returned when a function requiring a period, received an invalid, period that isn't a non-negative integer. + ErrInvalidPeriod = errors.New("invalid period, period should be a positive integer") + + // ErrInvalidReportExt is returned when a report file has an invalid extension. + ErrInvalidReportExt = errors.New("invalid report file extension") + + // ErrInvalidReportName is returned when a report file has an invalid name that can't be parsed. + ErrInvalidReportName = errors.New("invalid report file name") +) + +// Report represents a report file. +type Report struct { + Path string // Path is the path to the report. + Name string // Name is the name of the report file, including extension. + TimeStamp int64 // TimeStamp is the timestamp of the report. + + reportStash reportStash +} + +// reportStash is a helper struct to store a report and its data for movement. +type reportStash struct { + Path string + Data []byte +} + +// New creates a new Report object from a path. +// It does not write to the file system, or validate the path. +func New(path string) (Report, error) { + if filepath.Ext(path) != constants.ReportExt { + return Report{}, ErrInvalidReportExt + } + + rTime, err := getReportTime(filepath.Base(path)) + if err != nil { + return Report{}, err + } + + return Report{Path: path, Name: filepath.Base(path), TimeStamp: rTime}, nil +} + +// ReadJSON reads the JSON data from the report file. +func (r Report) ReadJSON() ([]byte, error) { + // Read the report file + data, err := os.ReadFile(r.Path) + if err != nil { + return nil, fmt.Errorf("failed to read report file: %v", err) + } + + if !json.Valid(data) { + return nil, fmt.Errorf("invalid JSON data in report file") + } + + return data, nil +} + +// getReportTime returns a int64 representation of the report time from the report path. +func getReportTime(path string) (int64, error) { + fileName := filepath.Base(path) + i, err := strconv.ParseInt(strings.TrimSuffix(fileName, filepath.Ext(fileName)), 10, 64) + if err != nil { + return i, fmt.Errorf("%w: %v", ErrInvalidReportName, err) + } + return i, nil +} + +// GetPeriodStart returns the start of the period window for a given period in seconds. +func GetPeriodStart(period int) (int64, error) { + if period <= 0 { + return 0, ErrInvalidPeriod + } + utcTime := time.Now().UTC().Unix() + return utcTime - (utcTime % int64(period)), nil +} + +// GetForPeriod returns the most recent report within a period window for a given directory. +// Not inclusive of the period end (periodStart + period). +// +// For example, given reports 1 and 7, with time 2 and period 7, the function will return the path for report 1. +func GetForPeriod(dir string, time time.Time, period int) (Report, error) { + if period <= 0 { + return Report{}, ErrInvalidPeriod + } + + periodStart := time.Unix() - (time.Unix() % int64(period)) + periodEnd := periodStart + int64(period) + + // Reports names are utc timestamps. Get the most recent report within the period window. + var report Report + err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err != nil { + slog.Error("Failed to access path", "path", path, "error", err) + return err + } + + // Skip subdirectories. + if d.IsDir() && path != dir { + return filepath.SkipDir + } + + r, err := New(path) + if errors.Is(err, ErrInvalidReportExt) || errors.Is(err, ErrInvalidReportName) { + slog.Info("Skipping non-report file", "file", d.Name(), "error", err) + return nil + } else if err != nil { + slog.Error("Failed to create report object", "error", err) + return err + } + + if r.TimeStamp < periodStart { + return nil + } + if r.TimeStamp >= periodEnd { + return filepath.SkipDir + } + + report = r + return nil + }) + + if err != nil { + return Report{}, err + } + + return report, nil +} + +// GetPerPeriod returns the latest report within each period window for a given directory. +// The key of the map is the start of the period window, and the value is a Report object. +// +// If period is 1, then all reports in the dir are returned. +func GetPerPeriod(dir string, period int) (map[int64]Report, error) { + if period <= 0 { + return nil, ErrInvalidPeriod + } + + reports := make(map[int64]Report) + err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err != nil { + slog.Error("Failed to access path", "path", path, "error", err) + return err + } + + if d.IsDir() && path != dir { + return filepath.SkipDir + } + + r, err := New(path) + if errors.Is(err, ErrInvalidReportExt) || errors.Is(err, ErrInvalidReportName) { + slog.Info("Skipping non-report file", "file", d.Name(), "error", err) + return nil + } else if err != nil { + slog.Error("Failed to create report object", "error", err) + return err + } + + periodStart := r.TimeStamp - (r.TimeStamp % int64(period)) + if existingReport, ok := reports[periodStart]; !ok || existingReport.TimeStamp < r.TimeStamp { + reports[periodStart] = r + } + + return nil + }) + + if err != nil { + return nil, err + } + + return reports, nil +} + +// GetAll returns all reports in a given directory. +// Reports are expected to have the correct file extension, and have a name which can be parsed by a timestamp. +// Does not traverse subdirectories. +func GetAll(dir string) ([]Report, error) { + reports := make([]Report, 0) + err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err != nil { + slog.Error("Failed to access path", "path", path, "error", err) + return err + } + + if d.IsDir() && path != dir { + return filepath.SkipDir + } + + r, err := New(path) + if errors.Is(err, ErrInvalidReportExt) || errors.Is(err, ErrInvalidReportName) { + slog.Info("Skipping non-report file", "file", d.Name(), "error", err) + return nil + } else if err != nil { + slog.Error("Failed to create report object", "error", err) + return err + } + + reports = append(reports, r) + return nil + }) + if err != nil { + return nil, err + } + + return reports, nil +} diff --git a/internal/reportutils/reportutils_test.go b/internal/report/report_test.go similarity index 79% rename from internal/reportutils/reportutils_test.go rename to internal/report/report_test.go index 9df7102..96b41be 100644 --- a/internal/reportutils/reportutils_test.go +++ b/internal/report/report_test.go @@ -1,12 +1,13 @@ -package reportutils_test +package report_test import ( "os" "path/filepath" "testing" + "time" "github.com/stretchr/testify/require" - "github.com/ubuntu/ubuntu-insights/internal/reportutils" + report "github.com/ubuntu/ubuntu-insights/internal/report" "github.com/ubuntu/ubuntu-insights/internal/testutils" ) @@ -20,15 +21,15 @@ func TestGetPeriodStart(t *testing.T) { }{ "Valid Period": {period: 500}, - "Invalid Negative Period": {period: -500, wantErr: reportutils.ErrInvalidPeriod}, - "Invalid Zero Period": {period: 0, wantErr: reportutils.ErrInvalidPeriod}, + "Invalid Negative Period": {period: -500, wantErr: report.ErrInvalidPeriod}, + "Invalid Zero Period": {period: 0, wantErr: report.ErrInvalidPeriod}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { t.Parallel() - got, err := reportutils.GetPeriodStart(tc.period) + got, err := report.GetPeriodStart(tc.period) if tc.wantErr != nil { require.ErrorIs(t, err, tc.wantErr) return @@ -40,25 +41,28 @@ func TestGetPeriodStart(t *testing.T) { } } -func TestGetReportTime(t *testing.T) { +func TestNew(t *testing.T) { + t.Parallel() + tests := map[string]struct { path string wantErr bool }{ - "Valid Report Time": {path: "1627847285.json", wantErr: false}, - "Valid Report Time with Path": {path: "/some/dir/1627847285.json", wantErr: false}, - "Alt Extension": {path: "1627847285.txt", wantErr: false}, + "Valid Report": {path: "1627847285.json"}, + "Valid Report with Path": {path: "/some/dir/1627847285.json"}, "Empty File Name": {path: ".json", wantErr: true}, "Invalid Report Time": {path: "invalid.json", wantErr: true}, "Invalid Report Mixed": {path: "i-1.json", wantErr: true}, "Invalid Report Time with Path": {path: "/123/123/invalid.json", wantErr: true}, + "Alt Extension": {path: "1627847285.txt", wantErr: true}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { - got, err := reportutils.GetReportTime(tc.path) + t.Parallel() + got, err := report.New(tc.path) if tc.wantErr { require.Error(t, err, "expected an error but got none") return @@ -66,12 +70,12 @@ func TestGetReportTime(t *testing.T) { require.NoError(t, err, "got an unexpected error") want := testutils.LoadWithUpdateFromGoldenYAML(t, got) - require.Equal(t, want, got, "GetReportTime should return the report time from the report path") + require.Equal(t, want, got, "New should return a new report object") }) } } -func TestGetReportPath(t *testing.T) { +func TestGetForPeriod(t *testing.T) { t.Parallel() tests := map[string]struct { @@ -95,8 +99,8 @@ func TestGetReportPath(t *testing.T) { "Negative Timestamp": {files: []string{"-100.json", "-101.json"}, time: -150, period: 100}, "Not Inclusive Period": {files: []string{"1.json", "7.json"}, time: 2, period: 7}, - "Invalid Negative Period": {files: []string{"1.json", "7.json"}, time: 2, period: -7, wantSpecificErr: reportutils.ErrInvalidPeriod}, - "Invalid Zero Period": {files: []string{"1.json", "7.json"}, time: 2, period: 0, wantSpecificErr: reportutils.ErrInvalidPeriod}, + "Invalid Negative Period": {files: []string{"1.json", "7.json"}, time: 2, period: -7, wantSpecificErr: report.ErrInvalidPeriod}, + "Invalid Zero Period": {files: []string{"1.json", "7.json"}, time: 2, period: 0, wantSpecificErr: report.ErrInvalidPeriod}, "Invalid Dir": {period: 1, invalidDir: true, wantGenericErr: true}, } @@ -107,12 +111,11 @@ func TestGetReportPath(t *testing.T) { dir, err := setupTmpDir(t, tc.files, tc.subDir, tc.subDirFiles) require.NoError(t, err, "Setup: failed to setup temporary directory") - defer os.RemoveAll(dir) if tc.invalidDir { dir = filepath.Join(dir, "invalid dir") } - got, err := reportutils.GetReportPath(dir, tc.time, tc.period) + r, err := report.GetForPeriod(dir, time.Unix(tc.time, 0), tc.period) if tc.wantSpecificErr != nil { require.ErrorIs(t, err, tc.wantSpecificErr) return @@ -121,21 +124,17 @@ func TestGetReportPath(t *testing.T) { require.Error(t, err, "expected an error but got none") return } - require.NoError(t, err, "got an unexpected error") - if got != "" { - got, err = filepath.Rel(dir, got) - require.NoError(t, err, "failed to get relative path") - } + got := sanitizeReportPath(t, r, dir) - want := testutils.LoadWithUpdateFromGolden(t, got) + want := testutils.LoadWithUpdateFromGoldenYAML(t, got) require.Equal(t, want, got, "GetReportPath should return the most recent report within the period window") }) } } -func TestGetReports(t *testing.T) { +func TestGetPerPeriod(t *testing.T) { t.Parallel() tests := map[string]struct { @@ -160,8 +159,8 @@ func TestGetReports(t *testing.T) { "Multiple Non-Consecutive Windows": {files: []string{"1.json", "7.json", "101.json", "107.json", "251.json", "257.json"}, period: 50}, "Get All Reports": {files: []string{"1.json", "2.json", "3.json", "101.json", "107.json", "251.json", "257.json"}, period: 1}, - "Invalid Negative Period": {files: []string{"1.json", "7.json"}, period: -7, wantSpecificErr: reportutils.ErrInvalidPeriod}, - "Invalid Zero Period": {files: []string{"1.json", "7.json"}, period: 0, wantSpecificErr: reportutils.ErrInvalidPeriod}, + "Invalid Negative Period": {files: []string{"1.json", "7.json"}, period: -7, wantSpecificErr: report.ErrInvalidPeriod}, + "Invalid Zero Period": {files: []string{"1.json", "7.json"}, period: 0, wantSpecificErr: report.ErrInvalidPeriod}, "Invalid Dir": {period: 1, invalidDir: true, wantGenericErr: true}, } @@ -172,12 +171,11 @@ func TestGetReports(t *testing.T) { dir, err := setupTmpDir(t, tc.files, tc.subDir, tc.subDirFiles) require.NoError(t, err, "Setup: failed to setup temporary directory") - defer os.RemoveAll(dir) if tc.invalidDir { dir = filepath.Join(dir, "invalid dir") } - got, err := reportutils.GetReports(dir, tc.period) + reports, err := report.GetPerPeriod(dir, tc.period) if tc.wantSpecificErr != nil { require.ErrorIs(t, err, tc.wantSpecificErr) return @@ -188,13 +186,17 @@ func TestGetReports(t *testing.T) { } require.NoError(t, err, "got an unexpected error") + got := make(map[int64]report.Report, len(reports)) + for n, r := range reports { + got[n] = sanitizeReportPath(t, r, dir) + } want := testutils.LoadWithUpdateFromGoldenYAML(t, got) require.Equal(t, want, got, "GetReports should return the most recent report within each period window") }) } } -func TestGetAllReports(t *testing.T) { +func TestGetAll(t *testing.T) { t.Parallel() tests := map[string]struct { @@ -219,18 +221,21 @@ func TestGetAllReports(t *testing.T) { t.Parallel() dir, err := setupTmpDir(t, tc.files, tc.subDir, tc.subDirFiles) require.NoError(t, err, "Setup: failed to setup temporary directory") - defer os.RemoveAll(dir) if tc.invalidDir { dir = filepath.Join(dir, "invalid dir") } - got, err := reportutils.GetAllReports(dir) + reports, err := report.GetAll(dir) if tc.wantErr { require.Error(t, err, "expected an error but got none") return } require.NoError(t, err, "got an unexpected error") + got := make([]report.Report, len(reports)) + for _, r := range reports { + got = append(got, sanitizeReportPath(t, r, dir)) + } want := testutils.LoadWithUpdateFromGoldenYAML(t, got) require.Equal(t, want, got, "GetAllReports should return all reports in the directory") }) @@ -240,10 +245,7 @@ func TestGetAllReports(t *testing.T) { func setupTmpDir(t *testing.T, files []string, subDir string, subDirFiles []string) (string, error) { t.Helper() - dir, err := os.MkdirTemp("", "reportutils-test") - if err != nil { - return "", err - } + dir := t.TempDir() for _, file := range files { path := filepath.Join(dir, file) @@ -268,3 +270,17 @@ func setupTmpDir(t *testing.T, files []string, subDir string, subDirFiles []stri return dir, nil } + +func sanitizeReportPath(t *testing.T, r report.Report, dir string) report.Report { + t.Helper() + if r.Path == "" { + return r + } + + fp, err := filepath.Rel(dir, r.Path) + if err != nil { + require.NoError(t, err, "failed to get relative path") + return report.Report{} + } + return report.Report{Path: fp, Name: r.Name, TimeStamp: r.TimeStamp} +} diff --git a/internal/reportutils/testdata/TestGetAllReports/golden/empty_directory b/internal/report/testdata/TestGetAll/golden/empty_directory similarity index 100% rename from internal/reportutils/testdata/TestGetAllReports/golden/empty_directory rename to internal/report/testdata/TestGetAll/golden/empty_directory diff --git a/internal/report/testdata/TestGetAll/golden/files_in_subdir b/internal/report/testdata/TestGetAll/golden/files_in_subdir new file mode 100644 index 0000000..ccc5761 --- /dev/null +++ b/internal/report/testdata/TestGetAll/golden/files_in_subdir @@ -0,0 +1,12 @@ +- path: "" + name: "" + timestamp: 0 +- path: "" + name: "" + timestamp: 0 +- path: 1.json + name: 1.json + timestamp: 1 +- path: 2.json + name: 2.json + timestamp: 2 diff --git a/internal/reportutils/testdata/TestGetAllReports/golden/invalid_file_extension b/internal/report/testdata/TestGetAll/golden/invalid_file_extension similarity index 100% rename from internal/reportutils/testdata/TestGetAllReports/golden/invalid_file_extension rename to internal/report/testdata/TestGetAll/golden/invalid_file_extension diff --git a/internal/reportutils/testdata/TestGetAllReports/golden/invalid_file_names b/internal/report/testdata/TestGetAll/golden/invalid_file_names similarity index 100% rename from internal/reportutils/testdata/TestGetAllReports/golden/invalid_file_names rename to internal/report/testdata/TestGetAll/golden/invalid_file_names diff --git a/internal/report/testdata/TestGetAll/golden/mix_of_valid_and_invalid b/internal/report/testdata/TestGetAll/golden/mix_of_valid_and_invalid new file mode 100644 index 0000000..e2ad51f --- /dev/null +++ b/internal/report/testdata/TestGetAll/golden/mix_of_valid_and_invalid @@ -0,0 +1,18 @@ +- path: "" + name: "" + timestamp: 0 +- path: "" + name: "" + timestamp: 0 +- path: "" + name: "" + timestamp: 0 +- path: 1.json + name: 1.json + timestamp: 1 +- path: 2.json + name: 2.json + timestamp: 2 +- path: 500.json + name: 500.json + timestamp: 500 diff --git a/internal/report/testdata/TestGetForPeriod/golden/empty_directory b/internal/report/testdata/TestGetForPeriod/golden/empty_directory new file mode 100644 index 0000000..ec9107d --- /dev/null +++ b/internal/report/testdata/TestGetForPeriod/golden/empty_directory @@ -0,0 +1,3 @@ +path: "" +name: "" +timestamp: 0 diff --git a/internal/report/testdata/TestGetForPeriod/golden/empty_subdir b/internal/report/testdata/TestGetForPeriod/golden/empty_subdir new file mode 100644 index 0000000..ec9107d --- /dev/null +++ b/internal/report/testdata/TestGetForPeriod/golden/empty_subdir @@ -0,0 +1,3 @@ +path: "" +name: "" +timestamp: 0 diff --git a/internal/report/testdata/TestGetForPeriod/golden/files_in_subdir b/internal/report/testdata/TestGetForPeriod/golden/files_in_subdir new file mode 100644 index 0000000..ec9107d --- /dev/null +++ b/internal/report/testdata/TestGetForPeriod/golden/files_in_subdir @@ -0,0 +1,3 @@ +path: "" +name: "" +timestamp: 0 diff --git a/internal/report/testdata/TestGetForPeriod/golden/invalid_file_extension b/internal/report/testdata/TestGetForPeriod/golden/invalid_file_extension new file mode 100644 index 0000000..ec9107d --- /dev/null +++ b/internal/report/testdata/TestGetForPeriod/golden/invalid_file_extension @@ -0,0 +1,3 @@ +path: "" +name: "" +timestamp: 0 diff --git a/internal/report/testdata/TestGetForPeriod/golden/invalid_file_names b/internal/report/testdata/TestGetForPeriod/golden/invalid_file_names new file mode 100644 index 0000000..ec9107d --- /dev/null +++ b/internal/report/testdata/TestGetForPeriod/golden/invalid_file_names @@ -0,0 +1,3 @@ +path: "" +name: "" +timestamp: 0 diff --git a/internal/report/testdata/TestGetForPeriod/golden/negative_timestamp b/internal/report/testdata/TestGetForPeriod/golden/negative_timestamp new file mode 100644 index 0000000..431aa05 --- /dev/null +++ b/internal/report/testdata/TestGetForPeriod/golden/negative_timestamp @@ -0,0 +1,3 @@ +path: -100.json +name: -100.json +timestamp: -100 diff --git a/internal/report/testdata/TestGetForPeriod/golden/not_inclusive_period b/internal/report/testdata/TestGetForPeriod/golden/not_inclusive_period new file mode 100644 index 0000000..56c8ba5 --- /dev/null +++ b/internal/report/testdata/TestGetForPeriod/golden/not_inclusive_period @@ -0,0 +1,3 @@ +path: 1.json +name: 1.json +timestamp: 1 diff --git a/internal/report/testdata/TestGetForPeriod/golden/specific_time_single_valid_report b/internal/report/testdata/TestGetForPeriod/golden/specific_time_single_valid_report new file mode 100644 index 0000000..7c4d2c9 --- /dev/null +++ b/internal/report/testdata/TestGetForPeriod/golden/specific_time_single_valid_report @@ -0,0 +1,3 @@ +path: 2.json +name: 2.json +timestamp: 2 diff --git a/internal/reportutils/testdata/TestGetReports/golden/empty_directory b/internal/report/testdata/TestGetPerPeriod/golden/empty_directory similarity index 100% rename from internal/reportutils/testdata/TestGetReports/golden/empty_directory rename to internal/report/testdata/TestGetPerPeriod/golden/empty_directory diff --git a/internal/reportutils/testdata/TestGetReports/golden/files_in_subdir b/internal/report/testdata/TestGetPerPeriod/golden/files_in_subdir similarity index 100% rename from internal/reportutils/testdata/TestGetReports/golden/files_in_subdir rename to internal/report/testdata/TestGetPerPeriod/golden/files_in_subdir diff --git a/internal/report/testdata/TestGetPerPeriod/golden/get_all_reports b/internal/report/testdata/TestGetPerPeriod/golden/get_all_reports new file mode 100644 index 0000000..e3e067a --- /dev/null +++ b/internal/report/testdata/TestGetPerPeriod/golden/get_all_reports @@ -0,0 +1,28 @@ +1: + path: 1.json + name: 1.json + timestamp: 1 +2: + path: 2.json + name: 2.json + timestamp: 2 +3: + path: 3.json + name: 3.json + timestamp: 3 +101: + path: 101.json + name: 101.json + timestamp: 101 +107: + path: 107.json + name: 107.json + timestamp: 107 +251: + path: 251.json + name: 251.json + timestamp: 251 +257: + path: 257.json + name: 257.json + timestamp: 257 diff --git a/internal/report/testdata/TestGetPerPeriod/golden/get_newest_of_period b/internal/report/testdata/TestGetPerPeriod/golden/get_newest_of_period new file mode 100644 index 0000000..d6fdc70 --- /dev/null +++ b/internal/report/testdata/TestGetPerPeriod/golden/get_newest_of_period @@ -0,0 +1,4 @@ +0: + path: 7.json + name: 7.json + timestamp: 7 diff --git a/internal/reportutils/testdata/TestGetReports/golden/invalid_file_extension b/internal/report/testdata/TestGetPerPeriod/golden/invalid_file_extension similarity index 100% rename from internal/reportutils/testdata/TestGetReports/golden/invalid_file_extension rename to internal/report/testdata/TestGetPerPeriod/golden/invalid_file_extension diff --git a/internal/reportutils/testdata/TestGetReports/golden/invalid_file_names b/internal/report/testdata/TestGetPerPeriod/golden/invalid_file_names similarity index 100% rename from internal/reportutils/testdata/TestGetReports/golden/invalid_file_names rename to internal/report/testdata/TestGetPerPeriod/golden/invalid_file_names diff --git a/internal/report/testdata/TestGetPerPeriod/golden/mix_of_valid_and_invalid b/internal/report/testdata/TestGetPerPeriod/golden/mix_of_valid_and_invalid new file mode 100644 index 0000000..5f45d53 --- /dev/null +++ b/internal/report/testdata/TestGetPerPeriod/golden/mix_of_valid_and_invalid @@ -0,0 +1,4 @@ +0: + path: 2.json + name: 2.json + timestamp: 2 diff --git a/internal/report/testdata/TestGetPerPeriod/golden/multiple_consecutive_windows b/internal/report/testdata/TestGetPerPeriod/golden/multiple_consecutive_windows new file mode 100644 index 0000000..d777da2 --- /dev/null +++ b/internal/report/testdata/TestGetPerPeriod/golden/multiple_consecutive_windows @@ -0,0 +1,12 @@ +0: + path: 7.json + name: 7.json + timestamp: 7 +100: + path: 107.json + name: 107.json + timestamp: 107 +200: + path: 207.json + name: 207.json + timestamp: 207 diff --git a/internal/report/testdata/TestGetPerPeriod/golden/multiple_non-consecutive_windows b/internal/report/testdata/TestGetPerPeriod/golden/multiple_non-consecutive_windows new file mode 100644 index 0000000..9fb936e --- /dev/null +++ b/internal/report/testdata/TestGetPerPeriod/golden/multiple_non-consecutive_windows @@ -0,0 +1,12 @@ +0: + path: 7.json + name: 7.json + timestamp: 7 +100: + path: 107.json + name: 107.json + timestamp: 107 +250: + path: 257.json + name: 257.json + timestamp: 257 diff --git a/internal/report/testdata/TestNew/golden/valid_report b/internal/report/testdata/TestNew/golden/valid_report new file mode 100644 index 0000000..7ce2121 --- /dev/null +++ b/internal/report/testdata/TestNew/golden/valid_report @@ -0,0 +1,3 @@ +path: 1627847285.json +name: 1627847285.json +timestamp: 1627847285 diff --git a/internal/report/testdata/TestNew/golden/valid_report_with_path b/internal/report/testdata/TestNew/golden/valid_report_with_path new file mode 100644 index 0000000..252b842 --- /dev/null +++ b/internal/report/testdata/TestNew/golden/valid_report_with_path @@ -0,0 +1,3 @@ +path: /some/dir/1627847285.json +name: 1627847285.json +timestamp: 1627847285 diff --git a/internal/reportutils/reportutils.go b/internal/reportutils/reportutils.go deleted file mode 100644 index 932d99c..0000000 --- a/internal/reportutils/reportutils.go +++ /dev/null @@ -1,177 +0,0 @@ -// Package reportutils provides utility functions for handling reports. -package reportutils - -// package report - -import ( - "errors" - "log/slog" - "os" - "path/filepath" - "strconv" - "strings" - "time" - - "github.com/ubuntu/ubuntu-insights/internal/constants" -) - -// ErrInvalidPeriod is returned when a function requiring a period, received an invalid, period that isn't a non-negative integer. -var ErrInvalidPeriod = errors.New("invalid period, period should be a positive integer") - -// type Report struct{} - -// GetPeriodStart returns the start of the period window for a given period in seconds. -func GetPeriodStart(period int) (int64, error) { - if period <= 0 { - return 0, ErrInvalidPeriod - } - utcTime := time.Now().UTC().Unix() - return utcTime - (utcTime % int64(period)), nil -} - -// GetReportTime returns a int64 representation of the report time from the report path. -func GetReportTime(path string) (int64, error) { - fileName := filepath.Base(path) - return strconv.ParseInt(strings.TrimSuffix(fileName, filepath.Ext(fileName)), 10, 64) -} - -// Path is a method on Report - -// GetReportPath returns the path for the most recent report within a period window, returning an empty string if no report is found. -// Not inclusive of the period end (periodStart + period). -// -// For example, given reports 1 and 7, with time 2 and period 7, the function will return the path for report 1. -func GetReportPath(dir string, time int64, period int) (string, error) { - if period <= 0 { - return "", ErrInvalidPeriod - } - - periodStart := time - (time % int64(period)) - periodEnd := periodStart + int64(period) - - // Reports names are utc timestamps. Get the most recent report within the period window. - var mostRecentReportPath string - err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { - if err != nil { - slog.Error("Failed to access path", "path", path, "error", err) - return err - } - - // Skip subdirectories. - if d.IsDir() && path != dir { - return filepath.SkipDir - } - - if filepath.Ext(d.Name()) != constants.ReportExt { - slog.Info("Skipping non-report file, invalid extension", "file", d.Name()) - return nil - } - - reportTime, err := GetReportTime(d.Name()) - if err != nil { - slog.Info("Skipping non-report file, invalid file name", "file", d.Name()) - return nil - } - - if reportTime < periodStart { - return nil - } - if reportTime >= periodEnd { - return filepath.SkipDir - } - - mostRecentReportPath = path - return nil - }) - - if err != nil { - return "", err - } - - return mostRecentReportPath, nil -} - -// -> map[int64]Report - -// GetReports returns the paths for the latest report within each period window for a given directory. -// The key of the map is the start of the period window, and the report timestamp. -// -// If period is 1, then all reports in the dir are returned. -func GetReports(dir string, period int) (map[int64]int64, error) { - if period <= 0 { - return nil, ErrInvalidPeriod - } - - reports := make(map[int64]int64) - err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { - if err != nil { - slog.Error("Failed to access path", "path", path, "error", err) - return err - } - - if d.IsDir() && path != dir { - return filepath.SkipDir - } - - if filepath.Ext(d.Name()) != constants.ReportExt { - slog.Info("Skipping non-report file, invalid extension", "file", d.Name()) - return nil - } - - reportTime, err := GetReportTime(d.Name()) - if err != nil { - slog.Info("Skipping non-report file, invalid file name", "file", d.Name()) - return nil - } - - periodStart := reportTime - (reportTime % int64(period)) - if existingReport, ok := reports[periodStart]; !ok || existingReport < reportTime { - reports[periodStart] = reportTime - } - - return nil - }) - - if err != nil { - return nil, err - } - - return reports, nil -} - -// reports.GetAll(dir string) ([]Report, error) - -// GetAllReports returns the filename for all reports within a given directory, which match the expected pattern. -// Does not traverse subdirectories. -func GetAllReports(dir string) ([]string, error) { - reports := make([]string, 0) - err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { - if err != nil { - slog.Error("Failed to access path", "path", path, "error", err) - return err - } - - if d.IsDir() && path != dir { - return filepath.SkipDir - } - - if filepath.Ext(d.Name()) != constants.ReportExt { - slog.Info("Skipping non-report file, invalid extension", "file", d.Name()) - return nil - } - - if _, err := GetReportTime(d.Name()); err != nil { - slog.Info("Skipping non-report file, invalid file name", "file", d.Name()) - return nil - } - - reports = append(reports, d.Name()) - return nil - }) - - if err != nil { - return nil, err - } - - return reports, nil -} diff --git a/internal/reportutils/testdata/TestGetAllReports/golden/files_in_subdir b/internal/reportutils/testdata/TestGetAllReports/golden/files_in_subdir deleted file mode 100644 index 6548887..0000000 --- a/internal/reportutils/testdata/TestGetAllReports/golden/files_in_subdir +++ /dev/null @@ -1,2 +0,0 @@ -- 1.json -- 2.json diff --git a/internal/reportutils/testdata/TestGetAllReports/golden/mix_of_valid_and_invalid b/internal/reportutils/testdata/TestGetAllReports/golden/mix_of_valid_and_invalid deleted file mode 100644 index a8eb048..0000000 --- a/internal/reportutils/testdata/TestGetAllReports/golden/mix_of_valid_and_invalid +++ /dev/null @@ -1,3 +0,0 @@ -- 1.json -- 2.json -- 500.json diff --git a/internal/reportutils/testdata/TestGetReportPath/golden/empty_directory b/internal/reportutils/testdata/TestGetReportPath/golden/empty_directory deleted file mode 100644 index e69de29..0000000 diff --git a/internal/reportutils/testdata/TestGetReportPath/golden/empty_subdir b/internal/reportutils/testdata/TestGetReportPath/golden/empty_subdir deleted file mode 100644 index e69de29..0000000 diff --git a/internal/reportutils/testdata/TestGetReportPath/golden/files_in_subdir b/internal/reportutils/testdata/TestGetReportPath/golden/files_in_subdir deleted file mode 100644 index e69de29..0000000 diff --git a/internal/reportutils/testdata/TestGetReportPath/golden/invalid_file_extension b/internal/reportutils/testdata/TestGetReportPath/golden/invalid_file_extension deleted file mode 100644 index e69de29..0000000 diff --git a/internal/reportutils/testdata/TestGetReportPath/golden/invalid_file_names b/internal/reportutils/testdata/TestGetReportPath/golden/invalid_file_names deleted file mode 100644 index e69de29..0000000 diff --git a/internal/reportutils/testdata/TestGetReportPath/golden/negative_timestamp b/internal/reportutils/testdata/TestGetReportPath/golden/negative_timestamp deleted file mode 100644 index 411e61e..0000000 --- a/internal/reportutils/testdata/TestGetReportPath/golden/negative_timestamp +++ /dev/null @@ -1 +0,0 @@ --100.json \ No newline at end of file diff --git a/internal/reportutils/testdata/TestGetReportPath/golden/not_inclusive_period b/internal/reportutils/testdata/TestGetReportPath/golden/not_inclusive_period deleted file mode 100644 index a3d3d4c..0000000 --- a/internal/reportutils/testdata/TestGetReportPath/golden/not_inclusive_period +++ /dev/null @@ -1 +0,0 @@ -1.json \ No newline at end of file diff --git a/internal/reportutils/testdata/TestGetReportPath/golden/specific_time_single_valid_report b/internal/reportutils/testdata/TestGetReportPath/golden/specific_time_single_valid_report deleted file mode 100644 index e1f33a0..0000000 --- a/internal/reportutils/testdata/TestGetReportPath/golden/specific_time_single_valid_report +++ /dev/null @@ -1 +0,0 @@ -2.json \ No newline at end of file diff --git a/internal/reportutils/testdata/TestGetReportTime/golden/alt_extension b/internal/reportutils/testdata/TestGetReportTime/golden/alt_extension deleted file mode 100644 index d29665a..0000000 --- a/internal/reportutils/testdata/TestGetReportTime/golden/alt_extension +++ /dev/null @@ -1 +0,0 @@ -1627847285 diff --git a/internal/reportutils/testdata/TestGetReportTime/golden/valid_report_time b/internal/reportutils/testdata/TestGetReportTime/golden/valid_report_time deleted file mode 100644 index d29665a..0000000 --- a/internal/reportutils/testdata/TestGetReportTime/golden/valid_report_time +++ /dev/null @@ -1 +0,0 @@ -1627847285 diff --git a/internal/reportutils/testdata/TestGetReportTime/golden/valid_report_time_with_path b/internal/reportutils/testdata/TestGetReportTime/golden/valid_report_time_with_path deleted file mode 100644 index d29665a..0000000 --- a/internal/reportutils/testdata/TestGetReportTime/golden/valid_report_time_with_path +++ /dev/null @@ -1 +0,0 @@ -1627847285 diff --git a/internal/reportutils/testdata/TestGetReports/golden/get_all_reports b/internal/reportutils/testdata/TestGetReports/golden/get_all_reports deleted file mode 100644 index e295670..0000000 --- a/internal/reportutils/testdata/TestGetReports/golden/get_all_reports +++ /dev/null @@ -1,7 +0,0 @@ -1: 1 -2: 2 -3: 3 -101: 101 -107: 107 -251: 251 -257: 257 diff --git a/internal/reportutils/testdata/TestGetReports/golden/get_newest_of_period b/internal/reportutils/testdata/TestGetReports/golden/get_newest_of_period deleted file mode 100644 index 017ba7a..0000000 --- a/internal/reportutils/testdata/TestGetReports/golden/get_newest_of_period +++ /dev/null @@ -1 +0,0 @@ -0: 7 diff --git a/internal/reportutils/testdata/TestGetReports/golden/mix_of_valid_and_invalid b/internal/reportutils/testdata/TestGetReports/golden/mix_of_valid_and_invalid deleted file mode 100644 index f196b73..0000000 --- a/internal/reportutils/testdata/TestGetReports/golden/mix_of_valid_and_invalid +++ /dev/null @@ -1 +0,0 @@ -0: 2 diff --git a/internal/reportutils/testdata/TestGetReports/golden/multiple_consecutive_windows b/internal/reportutils/testdata/TestGetReports/golden/multiple_consecutive_windows deleted file mode 100644 index 8e9a957..0000000 --- a/internal/reportutils/testdata/TestGetReports/golden/multiple_consecutive_windows +++ /dev/null @@ -1,3 +0,0 @@ -0: 7 -100: 107 -200: 207 diff --git a/internal/reportutils/testdata/TestGetReports/golden/multiple_consequtive_windows b/internal/reportutils/testdata/TestGetReports/golden/multiple_consequtive_windows deleted file mode 100644 index 8e9a957..0000000 --- a/internal/reportutils/testdata/TestGetReports/golden/multiple_consequtive_windows +++ /dev/null @@ -1,3 +0,0 @@ -0: 7 -100: 107 -200: 207 diff --git a/internal/reportutils/testdata/TestGetReports/golden/multiple_non-consecutive_windows b/internal/reportutils/testdata/TestGetReports/golden/multiple_non-consecutive_windows deleted file mode 100644 index f2cb19d..0000000 --- a/internal/reportutils/testdata/TestGetReports/golden/multiple_non-consecutive_windows +++ /dev/null @@ -1,3 +0,0 @@ -0: 7 -100: 107 -250: 257 diff --git a/internal/reportutils/testdata/TestGetReports/golden/multiple_non-consequtive_windows b/internal/reportutils/testdata/TestGetReports/golden/multiple_non-consequtive_windows deleted file mode 100644 index f2cb19d..0000000 --- a/internal/reportutils/testdata/TestGetReports/golden/multiple_non-consequtive_windows +++ /dev/null @@ -1,3 +0,0 @@ -0: 7 -100: 107 -250: 257 diff --git a/internal/uploader/internal_test.go b/internal/uploader/internal_test.go index f9882d3..27be531 100644 --- a/internal/uploader/internal_test.go +++ b/internal/uploader/internal_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" "github.com/ubuntu/ubuntu-insights/internal/constants" "github.com/ubuntu/ubuntu-insights/internal/fileutils" + "github.com/ubuntu/ubuntu-insights/internal/report" ) func TestUploadBadFile(t *testing.T) { @@ -25,21 +26,22 @@ func TestUploadBadFile(t *testing.T) { url string consent bool + rNewErr bool wantErr bool }{ "Ok": {fName: "0.json", fileContents: basicContent, wantErr: false}, "Missing File": {fName: "0.json", fileContents: basicContent, missingFile: true, wantErr: true}, "File Is Dir": {fName: "0.json", fileIsDir: true, wantErr: true}, - "Non-numeric": {fName: "not-numeric.json", fileContents: basicContent, wantErr: true}, - "Bad File Ext": {fName: "0.txt", fileContents: basicContent}, + "Non-numeric": {fName: "not-numeric.json", fileContents: basicContent, rNewErr: true}, + "Bad File Ext": {fName: "0.txt", fileContents: basicContent, rNewErr: true}, "Bad Contents": {fName: "0.json", fileContents: badContent, wantErr: true}, "Bad URL": {fName: "0.json", fileContents: basicContent, url: "http://bad host:1234", wantErr: true}, "Ok Consent": {fName: "0.json", fileContents: basicContent, consent: true, wantErr: false}, "Missing File Consent": {fName: "0.json", fileContents: basicContent, missingFile: true, consent: true, wantErr: true}, "File Is Dir Consent": {fName: "0.json", fileIsDir: true, consent: true, wantErr: true}, - "Non-numeric Consent": {fName: "not-numeric.json", fileContents: basicContent, consent: true, wantErr: true}, - "Bad File Ext Consent": {fName: "0.txt", fileContents: basicContent, consent: true}, + "Non-numeric Consent": {fName: "not-numeric.json", fileContents: basicContent, consent: true, rNewErr: true}, + "Bad File Ext Consent": {fName: "0.txt", fileContents: basicContent, consent: true, rNewErr: true}, "Bad Contents Consent": {fName: "0.json", fileContents: badContent, consent: true, wantErr: true}, "Bad URL Consent": {fName: "0.json", fileContents: basicContent, url: "http://bad host:1234", consent: true, wantErr: true}, } @@ -77,8 +79,13 @@ func TestUploadBadFile(t *testing.T) { if tc.url == "" { tc.url = ts.URL } - - err := um.upload(tc.fName, tc.url, tc.consent, false) + r, err := report.New(fPath) + if tc.rNewErr { + require.Error(t, err, "Setup: failed to create report object") + return + } + require.NoError(t, err, "Setup: failed to create report object") + err = um.upload(r, tc.url, tc.consent, false) if tc.wantErr { require.Error(t, err) return diff --git a/internal/uploader/upload.go b/internal/uploader/upload.go index 2f181e3..cb04d46 100644 --- a/internal/uploader/upload.go +++ b/internal/uploader/upload.go @@ -16,7 +16,7 @@ import ( "github.com/ubuntu/ubuntu-insights/internal/constants" "github.com/ubuntu/ubuntu-insights/internal/fileutils" - "github.com/ubuntu/ubuntu-insights/internal/reportutils" + "github.com/ubuntu/ubuntu-insights/internal/report" ) var ( @@ -34,7 +34,7 @@ func (um Uploader) Upload(force bool) error { return fmt.Errorf("upload failed to get consent state: %v", err) } - reports, err := reportutils.GetAllReports(um.collectedDir) + reports, err := report.GetAll(um.collectedDir) if err != nil { return fmt.Errorf("failed to get reports: %v", err) } @@ -45,17 +45,17 @@ func (um Uploader) Upload(force bool) error { } var wg sync.WaitGroup - for _, name := range reports { + for _, r := range reports { wg.Add(1) - go func(name string) { + go func(r report.Report) { defer wg.Done() - err := um.upload(name, url, consent, force) + err := um.upload(r, url, consent, force) if errors.Is(err, ErrReportNotMature) { - slog.Debug("Skipped report upload, not mature enough", "file", name, "source", um.source) + slog.Debug("Skipped report upload, not mature enough", "file", r.Name, "source", um.source) } else if err != nil { - slog.Warn("Failed to upload report", "file", name, "source", um.source, "error", err) + slog.Warn("Failed to upload report", "file", r.Name, "source", um.source, "error", err) } - }(name) + }(r) } wg.Wait() @@ -64,21 +64,15 @@ func (um Uploader) Upload(force bool) error { // upload uploads an individual report to the server. It returns an error if the report is not mature enough to be uploaded, or if the upload fails. // It also moves the report to the uploaded directory after a successful upload. -func (um Uploader) upload(name, url string, consent, force bool) error { - slog.Debug("Uploading report", "file", name, "consent", consent, "force", force) +func (um Uploader) upload(r report.Report, url string, consent, force bool) error { + slog.Debug("Uploading report", "file", r.Name, "consent", consent, "force", force) - // TODO… pass the Report object directly. - ts, err := reportutils.GetReportTime(name) - if err != nil { - return fmt.Errorf("failed to parse report time from filename: %v", err) - } - - if um.timeProvider.Now().Add(time.Duration(-um.minAge)*time.Second).Before(time.Unix(ts, 0)) && !force { + if um.timeProvider.Now().Add(time.Duration(-um.minAge)*time.Second).Before(time.Unix(r.TimeStamp, 0)) && !force { return ErrReportNotMature } // Check for duplicate reports. - _, err = os.Stat(filepath.Join(um.uploadedDir, name)) + _, err := os.Stat(filepath.Join(um.uploadedDir, r.Name)) if err != nil && !os.IsNotExist(err) { return fmt.Errorf("failed to check if report has already been uploaded: %v", err) } @@ -88,9 +82,9 @@ func (um Uploader) upload(name, url string, consent, force bool) error { return fmt.Errorf("report has already been uploaded") } - origData, err := um.readJSON(name) + origData, err := r.ReadJSON() if err != nil { - return fmt.Errorf("failed to get payload: %v", err) + return fmt.Errorf("failed to read report: %v", err) } data := origData if !consent { @@ -108,11 +102,11 @@ func (um Uploader) upload(name, url string, consent, force bool) error { // Move report first to avoid the situation where the report is sent, but not marked as sent. // TODO: maybe a method on Reports ? - if err := um.moveReport(filepath.Join(um.uploadedDir, name), filepath.Join(um.collectedDir, name), data); err != nil { + if err := um.moveReport(filepath.Join(um.uploadedDir, r.Name), filepath.Join(um.collectedDir, r.Name), data); err != nil { return fmt.Errorf("failed to move report after uploading: %v", err) } if err := send(url, data); err != nil { - if moveErr := um.moveReport(filepath.Join(um.collectedDir, name), filepath.Join(um.uploadedDir, name), origData); moveErr != nil { + if moveErr := um.moveReport(filepath.Join(um.collectedDir, r.Name), filepath.Join(um.uploadedDir, r.Name), origData); moveErr != nil { return fmt.Errorf("failed to send data: %v, and failed to restore the original report: %v", err, moveErr) } return fmt.Errorf("failed to send data: %v", err) @@ -130,21 +124,6 @@ func (um Uploader) getURL() (string, error) { return u.String(), nil } -// readJSON reads the JSON data from the report file. -func (um Uploader) readJSON(file string) ([]byte, error) { - // Read the report file - data, err := os.ReadFile(path.Join(um.collectedDir, file)) - if err != nil { - return nil, fmt.Errorf("failed to read report file: %v", err) - } - - if !json.Valid(data) { - return nil, fmt.Errorf("invalid JSON data in report file") - } - - return data, nil -} - func (um Uploader) moveReport(writePath, removePath string, data []byte) error { // (Report).MarkAsProcessed(data) // (Report).UndoProcessed From f5a0ee471c5c1cf0c70a47ce931d7d188421a61b Mon Sep 17 00:00:00 2001 From: kat <28567881+hk21702@users.noreply.github.com> Date: Thu, 30 Jan 2025 13:38:51 -0500 Subject: [PATCH 22/26] Implement Report processed marking --- internal/fileutils/fileutils_test.go | 25 +- internal/report/report.go | 65 +++- internal/report/report_test.go | 291 +++++++++++++++++- .../TestMarkAsProcessed/golden/basic_move | 7 + .../golden/basic_move_new_data | 7 + .../golden/basic_move_overwrite | 7 + .../TestMarkAsProcessed/golden/dstperm_none | 7 + .../TestMarkAsProcessed/golden/srcperm_none | 7 + .../testdata/TestReadJSON/golden/basic_read | 1 + .../TestReadJSON/golden/multiple_files | 1 + .../TestUndoProcessed/golden/basic_move | 7 + .../golden/basic_move_new_data | 7 + .../golden/basic_move_overwrite | 7 + internal/uploader/internal_test.go | 52 ---- internal/uploader/upload.go | 32 +- 15 files changed, 417 insertions(+), 106 deletions(-) create mode 100644 internal/report/testdata/TestMarkAsProcessed/golden/basic_move create mode 100644 internal/report/testdata/TestMarkAsProcessed/golden/basic_move_new_data create mode 100644 internal/report/testdata/TestMarkAsProcessed/golden/basic_move_overwrite create mode 100644 internal/report/testdata/TestMarkAsProcessed/golden/dstperm_none create mode 100644 internal/report/testdata/TestMarkAsProcessed/golden/srcperm_none create mode 100644 internal/report/testdata/TestReadJSON/golden/basic_read create mode 100644 internal/report/testdata/TestReadJSON/golden/multiple_files create mode 100644 internal/report/testdata/TestUndoProcessed/golden/basic_move create mode 100644 internal/report/testdata/TestUndoProcessed/golden/basic_move_new_data create mode 100644 internal/report/testdata/TestUndoProcessed/golden/basic_move_overwrite diff --git a/internal/fileutils/fileutils_test.go b/internal/fileutils/fileutils_test.go index 15c2b96..9308134 100644 --- a/internal/fileutils/fileutils_test.go +++ b/internal/fileutils/fileutils_test.go @@ -3,6 +3,7 @@ package fileutils_test import ( "os" "path/filepath" + "runtime" "testing" "github.com/stretchr/testify/require" @@ -13,21 +14,24 @@ func TestAtomicWrite(t *testing.T) { t.Parallel() tests := map[string]struct { - data []byte - fileExists bool - invalidDir bool + data []byte + fileExists bool + fileExistsPerms os.FileMode + invalidDir bool wantError bool }{ "Empty file": {data: []byte{}}, "Non-empty file": {data: []byte("data")}, - "Override file": {data: []byte("data"), fileExists: true}, - "Override empty file": {data: []byte{}, fileExists: true}, + "Override file": {data: []byte("data"), fileExistsPerms: 0600, fileExists: true}, + "Override empty file": {data: []byte{}, fileExistsPerms: 0600, fileExists: true}, - "Existing empty file": {data: []byte{}, fileExists: true}, - "Existing non-empty file": {data: []byte("data"), fileExists: true}, + "Existing empty file": {data: []byte{}, fileExistsPerms: 0600, fileExists: true}, + "Existing non-empty file": {data: []byte("data"), fileExistsPerms: 0600, fileExists: true}, - "Invalid Dir": {data: []byte("data"), invalidDir: true, wantError: true}, + "Override read-only file": {data: []byte("data"), fileExistsPerms: 0400, fileExists: true, wantError: runtime.GOOS == "windows"}, + "Override No Perms file": {data: []byte("data"), fileExistsPerms: 0000, fileExists: true, wantError: runtime.GOOS == "windows"}, + "Invalid Dir": {data: []byte("data"), invalidDir: true, wantError: true}, } for name, tc := range tests { @@ -42,8 +46,9 @@ func TestAtomicWrite(t *testing.T) { } if tc.fileExists { - err := fileutils.AtomicWrite(path, oldFile) - require.NoError(t, err, "Setup: AtomicWrite should not return an error") + err := os.WriteFile(path, oldFile, tc.fileExistsPerms) + require.NoError(t, err, "Setup: WriteFile should not return an error") + t.Cleanup(func() { _ = os.Chmod(path, 0600) }) } err := fileutils.AtomicWrite(path, tc.data) diff --git a/internal/report/report.go b/internal/report/report.go index eb22687..d866fad 100644 --- a/internal/report/report.go +++ b/internal/report/report.go @@ -15,6 +15,7 @@ import ( "time" "github.com/ubuntu/ubuntu-insights/internal/constants" + "github.com/ubuntu/ubuntu-insights/internal/fileutils" ) var ( @@ -30,7 +31,7 @@ var ( // Report represents a report file. type Report struct { - Path string // Path is the path to the report. + Path string // Path is the path to the report file. Name string // Name is the name of the report file, including extension. TimeStamp int64 // TimeStamp is the timestamp of the report. @@ -73,6 +74,50 @@ func (r Report) ReadJSON() ([]byte, error) { return data, nil } +// MarkAsProcessed moves the report to a destination directory, and writes the data to the report. +// The original report is removed. +// +// The new report is returned, and the original data is stashed for use with UndoProcessed. +// Note that calling MarkAsProcessed multiple times on the same report will overwrite the stashed data. +func (r Report) MarkAsProcessed(dest string, data []byte) (Report, error) { + origData, err := r.ReadJSON() + if err != nil { + return Report{}, fmt.Errorf("failed to read original report: %v", err) + } + + newReport := Report{Path: filepath.Join(dest, r.Name), Name: r.Name, TimeStamp: r.TimeStamp, + reportStash: reportStash{Path: r.Path, Data: origData}} + + if err := fileutils.AtomicWrite(newReport.Path, data); err != nil { + return Report{}, fmt.Errorf("failed to write report: %v", err) + } + + if err := os.Remove(r.Path); err != nil { + return Report{}, fmt.Errorf("failed to remove report: %v", err) + } + + return newReport, nil +} + +// UndoProcessed moves the report back to the original directory, and writes the original data to the report. +// The new report is returned, and the original data is removed. +func (r Report) UndoProcessed() (Report, error) { + if r.reportStash.Path == "" { + return Report{}, errors.New("no stashed data to restore") + } + + if err := fileutils.AtomicWrite(r.reportStash.Path, r.reportStash.Data); err != nil { + return Report{}, fmt.Errorf("failed to write report: %v", err) + } + + if err := os.Remove(r.Path); err != nil { + return Report{}, fmt.Errorf("failed to remove report: %v", err) + } + + newReport := Report{Path: r.reportStash.Path, Name: r.Name, TimeStamp: r.TimeStamp} + return newReport, nil +} + // getReportTime returns a int64 representation of the report time from the report path. func getReportTime(path string) (int64, error) { fileName := filepath.Base(path) @@ -108,8 +153,7 @@ func GetForPeriod(dir string, time time.Time, period int) (Report, error) { var report Report err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { if err != nil { - slog.Error("Failed to access path", "path", path, "error", err) - return err + return fmt.Errorf("failed to access path: %v", err) } // Skip subdirectories. @@ -122,8 +166,7 @@ func GetForPeriod(dir string, time time.Time, period int) (Report, error) { slog.Info("Skipping non-report file", "file", d.Name(), "error", err) return nil } else if err != nil { - slog.Error("Failed to create report object", "error", err) - return err + return fmt.Errorf("failed to create report object: %v", err) } if r.TimeStamp < periodStart { @@ -156,8 +199,7 @@ func GetPerPeriod(dir string, period int) (map[int64]Report, error) { reports := make(map[int64]Report) err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { if err != nil { - slog.Error("Failed to access path", "path", path, "error", err) - return err + return fmt.Errorf("failed to access path: %v", err) } if d.IsDir() && path != dir { @@ -169,8 +211,7 @@ func GetPerPeriod(dir string, period int) (map[int64]Report, error) { slog.Info("Skipping non-report file", "file", d.Name(), "error", err) return nil } else if err != nil { - slog.Error("Failed to create report object", "error", err) - return err + return fmt.Errorf("failed to create report object: %v", err) } periodStart := r.TimeStamp - (r.TimeStamp % int64(period)) @@ -195,8 +236,7 @@ func GetAll(dir string) ([]Report, error) { reports := make([]Report, 0) err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { if err != nil { - slog.Error("Failed to access path", "path", path, "error", err) - return err + return fmt.Errorf("failed to access path: %v", err) } if d.IsDir() && path != dir { @@ -208,8 +248,7 @@ func GetAll(dir string) ([]Report, error) { slog.Info("Skipping non-report file", "file", d.Name(), "error", err) return nil } else if err != nil { - slog.Error("Failed to create report object", "error", err) - return err + return fmt.Errorf("failed to create report object: %v", err) } reports = append(reports, r) diff --git a/internal/report/report_test.go b/internal/report/report_test.go index 96b41be..b41803e 100644 --- a/internal/report/report_test.go +++ b/internal/report/report_test.go @@ -3,11 +3,12 @@ package report_test import ( "os" "path/filepath" + "runtime" "testing" "time" "github.com/stretchr/testify/require" - report "github.com/ubuntu/ubuntu-insights/internal/report" + "github.com/ubuntu/ubuntu-insights/internal/report" "github.com/ubuntu/ubuntu-insights/internal/testutils" ) @@ -109,7 +110,7 @@ func TestGetForPeriod(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - dir, err := setupTmpDir(t, tc.files, tc.subDir, tc.subDirFiles) + dir, err := setupNoDataDir(t, tc.files, tc.subDir, tc.subDirFiles) require.NoError(t, err, "Setup: failed to setup temporary directory") if tc.invalidDir { dir = filepath.Join(dir, "invalid dir") @@ -169,7 +170,7 @@ func TestGetPerPeriod(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - dir, err := setupTmpDir(t, tc.files, tc.subDir, tc.subDirFiles) + dir, err := setupNoDataDir(t, tc.files, tc.subDir, tc.subDirFiles) require.NoError(t, err, "Setup: failed to setup temporary directory") if tc.invalidDir { dir = filepath.Join(dir, "invalid dir") @@ -219,7 +220,7 @@ func TestGetAll(t *testing.T) { for name, tc := range tests { t.Run(name, func(t *testing.T) { t.Parallel() - dir, err := setupTmpDir(t, tc.files, tc.subDir, tc.subDirFiles) + dir, err := setupNoDataDir(t, tc.files, tc.subDir, tc.subDirFiles) require.NoError(t, err, "Setup: failed to setup temporary directory") if tc.invalidDir { dir = filepath.Join(dir, "invalid dir") @@ -242,11 +243,287 @@ func TestGetAll(t *testing.T) { } } -func setupTmpDir(t *testing.T, files []string, subDir string, subDirFiles []string) (string, error) { +func TestMarkAsProcessed(t *testing.T) { + t.Parallel() + + type got struct { + Report report.Report + SrcFiles map[string]string + DstFiles map[string]string + } + + tests := map[string]struct { + srcFile map[string]string + dstFile map[string]string + + fileName string + data []byte + srcFilePerms os.FileMode + dstFilePerms os.FileMode + + wantErr bool + }{ + "Basic Move": { + srcFile: map[string]string{"1.json": `{"test": true}`}, + dstFile: map[string]string{}, + fileName: "1.json", + data: []byte(`{"test": true}`), + srcFilePerms: os.FileMode(0o600), + dstFilePerms: os.FileMode(0o600), + wantErr: false, + }, + "Basic Move New Data": { + srcFile: map[string]string{"1.json": `{"test": true}`}, + dstFile: map[string]string{}, + fileName: "1.json", + data: []byte("new data"), + srcFilePerms: os.FileMode(0o600), + dstFilePerms: os.FileMode(0o600), + wantErr: false, + }, + "Basic Move Overwrite": { + srcFile: map[string]string{"1.json": `{"test": true}`}, + dstFile: map[string]string{"1.json": "old data"}, + fileName: "1.json", + data: []byte("new data"), + srcFilePerms: os.FileMode(0o600), + dstFilePerms: os.FileMode(0o600), + wantErr: false, + }, "SrcPerm None": { + srcFile: map[string]string{"1.json": `{"test": true}`}, + dstFile: map[string]string{}, + fileName: "1.json", + data: []byte(`{"test": true}`), + srcFilePerms: os.FileMode(000), + dstFilePerms: os.FileMode(0o600), + wantErr: runtime.GOOS != "windows", + }, "DstPerm None": { + srcFile: map[string]string{"1.json": `{"test": true}`}, + dstFile: map[string]string{"1.json": "old data"}, + fileName: "1.json", + data: []byte("new data"), + srcFilePerms: os.FileMode(0o600), + dstFilePerms: os.FileMode(000), + wantErr: runtime.GOOS == "windows", + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + rootDir, srcDir, dstDir := setupProcessingDirs(t) + + setupBasicDir(t, tc.srcFile, tc.srcFilePerms, srcDir) + setupBasicDir(t, tc.dstFile, tc.dstFilePerms, dstDir) + + r, err := report.New(filepath.Join(srcDir, tc.fileName)) + require.NoError(t, err, "Setup: failed to create report object") + + r, err = r.MarkAsProcessed(dstDir, tc.data) + if tc.wantErr { + require.Error(t, err, "expected an error but got none") + return + } + require.NoError(t, err, "got an unexpected error") + + dstDirContents, err := testutils.GetDirContents(t, dstDir, 2) + require.NoError(t, err, "failed to get directory contents") + + srcDirContents, err := testutils.GetDirContents(t, srcDir, 2) + require.NoError(t, err, "failed to get directory contents") + + r = sanitizeReportPath(t, r, rootDir) + got := got{Report: r, SrcFiles: srcDirContents, DstFiles: dstDirContents} + want := testutils.LoadWithUpdateFromGoldenYAML(t, got) + require.EqualExportedValues(t, want, got, "MarkAsProcessed should move the report to the processed directory") + }) + } +} + +func TestUndoProcessed(t *testing.T) { + t.Parallel() + + type got struct { + Report report.Report + SrcFiles map[string]string + DstFiles map[string]string + } + + tests := map[string]struct { + srcFile map[string]string + dstFile map[string]string + + fileName string + data []byte + + wantErr bool + }{ + "Basic Move": { + srcFile: map[string]string{"1.json": `{"test": true}`}, + dstFile: map[string]string{}, + fileName: "1.json", + data: []byte(`"new data"`), + wantErr: false, + }, + "Basic Move New Data": { + srcFile: map[string]string{"1.json": `{"test": true}`}, + dstFile: map[string]string{}, + fileName: "1.json", + data: []byte("new data"), + wantErr: false, + }, + "Basic Move Overwrite": { + srcFile: map[string]string{"1.json": `{"test": true}`}, + dstFile: map[string]string{"1.json": "old data"}, + fileName: "1.json", + data: []byte("new data"), + wantErr: false, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + rootDir, srcDir, dstDir := setupProcessingDirs(t) + + setupBasicDir(t, tc.srcFile, 0600, srcDir) + setupBasicDir(t, tc.dstFile, 0600, dstDir) + + r, err := report.New(filepath.Join(srcDir, tc.fileName)) + require.NoError(t, err, "Setup: failed to create report object") + + r, err = r.MarkAsProcessed(dstDir, tc.data) + require.NoError(t, err, "Setup: failed to mark report as processed") + + r, err = r.UndoProcessed() + if tc.wantErr { + require.Error(t, err, "expected an error but got none") + return + } + require.NoError(t, err, "got an unexpected error") + + dstDirContents, err := testutils.GetDirContents(t, dstDir, 2) + require.NoError(t, err, "failed to get directory contents") + + srcDirContents, err := testutils.GetDirContents(t, srcDir, 2) + require.NoError(t, err, "failed to get directory contents") + + r = sanitizeReportPath(t, r, rootDir) + got := got{Report: r, SrcFiles: srcDirContents, DstFiles: dstDirContents} + want := testutils.LoadWithUpdateFromGoldenYAML(t, got) + require.EqualExportedValues(t, want, got, "UndoProcessed should move the report to the processed directory") + }) + } +} + +func TestUndoProcessedNoStash(t *testing.T) { + t.Parallel() + + r, err := report.New("1.json") + require.NoError(t, err, "Setup: failed to create report object") + + _, err = r.UndoProcessed() + require.Error(t, err, "UndoProcessed should return an error if the report has not been marked as processed") +} + +func TestMarkAsProcessedNoFile(t *testing.T) { + t.Parallel() + + _, srcDir, dstDir := setupProcessingDirs(t) + r, err := report.New(filepath.Join(srcDir, "1.json")) + require.NoError(t, err, "Setup: failed to create report object") + + _, err = r.MarkAsProcessed(dstDir, []byte(`"new data"`)) + require.Error(t, err, "MarkAsProcessed should return an error if the report file does not exist") +} + +func TestUndoProcessedNoFile(t *testing.T) { + t.Parallel() + + _, srcDir, dstDir := setupProcessingDirs(t) + reportPath := filepath.Join(srcDir, "1.json") + require.NoError(t, os.WriteFile(reportPath, []byte(`{"test": true}`), 0600), "Setup: failed to write report file") + r, err := report.New(reportPath) + require.NoError(t, err, "Setup: failed to create report object") + + r, err = r.MarkAsProcessed(dstDir, []byte(`"new data"`)) + require.NoError(t, err, "Setup: failed to mark report as processed") + + require.NoError(t, os.Remove(r.Path), "Setup: failed to remove report file") + + _, err = r.UndoProcessed() + require.Error(t, err, "UndoProcessed should return an error if the report file does not exist") +} + +func TestReadJSON(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + files map[string]string + file string + + wantErr bool + }{ + "Basic Read": {files: map[string]string{"1.json": `{"test": true}`}, file: "1.json"}, + "Multiple Files": {files: map[string]string{"1.json": `{"test": true}`, "2.json": `{"test": false}`}, file: "1.json"}, + + "Empty File": {files: map[string]string{"1.json": ""}, file: "1.json", wantErr: true}, + "Invalid JSON": {files: map[string]string{"1.json": `{"test":::`}, file: "1.json", wantErr: true}, + "No File": {files: map[string]string{}, file: "1.json", wantErr: true}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + _, srcDir, _ := setupProcessingDirs(t) + + setupBasicDir(t, tc.files, 0600, srcDir) + + r, err := report.New(filepath.Join(srcDir, tc.file)) + require.NoError(t, err, "Setup: failed to create report object") + + data, err := r.ReadJSON() + if tc.wantErr { + require.Error(t, err, "expected an error but got none") + return + } + require.NoError(t, err, "got an unexpected error") + + got := string(data) + want := testutils.LoadWithUpdateFromGolden(t, got) + require.Equal(t, want, got, "ReadJSON should return the data from the report file") + }) + } +} + +func setupProcessingDirs(t *testing.T) (rootDir, srcDir, dstDir string) { t.Helper() + rootDir = t.TempDir() + srcDir = filepath.Join(rootDir, "src") + dstDir = filepath.Join(rootDir, "dst") + err := os.MkdirAll(srcDir, 0700) + require.NoError(t, err, "Setup: failed to create source directory") + err = os.MkdirAll(dstDir, 0700) + require.NoError(t, err, "Setup: failed to create destination directory") + return rootDir, srcDir, dstDir +} - dir := t.TempDir() +func setupBasicDir(t *testing.T, files map[string]string, perms os.FileMode, dir string) { + t.Helper() + for file, data := range files { + path := filepath.Join(dir, file) + err := os.WriteFile(path, []byte(data), perms) + require.NoError(t, err, "Setup: failed to write file") + t.Cleanup(func() { + _ = os.Chmod(path, os.FileMode(0600)) + }) + } +} +func setupNoDataDir(t *testing.T, files []string, subDir string, subDirFiles []string) (string, error) { + t.Helper() + + dir := t.TempDir() for _, file := range files { path := filepath.Join(dir, file) if err := os.WriteFile(path, []byte{}, 0600); err != nil { @@ -282,5 +559,5 @@ func sanitizeReportPath(t *testing.T, r report.Report, dir string) report.Report require.NoError(t, err, "failed to get relative path") return report.Report{} } - return report.Report{Path: fp, Name: r.Name, TimeStamp: r.TimeStamp} + return report.Report{Path: filepath.ToSlash(fp), Name: r.Name, TimeStamp: r.TimeStamp} } diff --git a/internal/report/testdata/TestMarkAsProcessed/golden/basic_move b/internal/report/testdata/TestMarkAsProcessed/golden/basic_move new file mode 100644 index 0000000..fa8bd51 --- /dev/null +++ b/internal/report/testdata/TestMarkAsProcessed/golden/basic_move @@ -0,0 +1,7 @@ +report: + path: dst/1.json + name: 1.json + timestamp: 1 +srcfiles: {} +dstfiles: + 1.json: '{"test": true}' diff --git a/internal/report/testdata/TestMarkAsProcessed/golden/basic_move_new_data b/internal/report/testdata/TestMarkAsProcessed/golden/basic_move_new_data new file mode 100644 index 0000000..0948d76 --- /dev/null +++ b/internal/report/testdata/TestMarkAsProcessed/golden/basic_move_new_data @@ -0,0 +1,7 @@ +report: + path: dst/1.json + name: 1.json + timestamp: 1 +srcfiles: {} +dstfiles: + 1.json: new data diff --git a/internal/report/testdata/TestMarkAsProcessed/golden/basic_move_overwrite b/internal/report/testdata/TestMarkAsProcessed/golden/basic_move_overwrite new file mode 100644 index 0000000..0948d76 --- /dev/null +++ b/internal/report/testdata/TestMarkAsProcessed/golden/basic_move_overwrite @@ -0,0 +1,7 @@ +report: + path: dst/1.json + name: 1.json + timestamp: 1 +srcfiles: {} +dstfiles: + 1.json: new data diff --git a/internal/report/testdata/TestMarkAsProcessed/golden/dstperm_none b/internal/report/testdata/TestMarkAsProcessed/golden/dstperm_none new file mode 100644 index 0000000..0948d76 --- /dev/null +++ b/internal/report/testdata/TestMarkAsProcessed/golden/dstperm_none @@ -0,0 +1,7 @@ +report: + path: dst/1.json + name: 1.json + timestamp: 1 +srcfiles: {} +dstfiles: + 1.json: new data diff --git a/internal/report/testdata/TestMarkAsProcessed/golden/srcperm_none b/internal/report/testdata/TestMarkAsProcessed/golden/srcperm_none new file mode 100644 index 0000000..fa8bd51 --- /dev/null +++ b/internal/report/testdata/TestMarkAsProcessed/golden/srcperm_none @@ -0,0 +1,7 @@ +report: + path: dst/1.json + name: 1.json + timestamp: 1 +srcfiles: {} +dstfiles: + 1.json: '{"test": true}' diff --git a/internal/report/testdata/TestReadJSON/golden/basic_read b/internal/report/testdata/TestReadJSON/golden/basic_read new file mode 100644 index 0000000..fb25aa1 --- /dev/null +++ b/internal/report/testdata/TestReadJSON/golden/basic_read @@ -0,0 +1 @@ +{"test": true} \ No newline at end of file diff --git a/internal/report/testdata/TestReadJSON/golden/multiple_files b/internal/report/testdata/TestReadJSON/golden/multiple_files new file mode 100644 index 0000000..fb25aa1 --- /dev/null +++ b/internal/report/testdata/TestReadJSON/golden/multiple_files @@ -0,0 +1 @@ +{"test": true} \ No newline at end of file diff --git a/internal/report/testdata/TestUndoProcessed/golden/basic_move b/internal/report/testdata/TestUndoProcessed/golden/basic_move new file mode 100644 index 0000000..3330bd0 --- /dev/null +++ b/internal/report/testdata/TestUndoProcessed/golden/basic_move @@ -0,0 +1,7 @@ +report: + path: src/1.json + name: 1.json + timestamp: 1 +srcfiles: + 1.json: '{"test": true}' +dstfiles: {} diff --git a/internal/report/testdata/TestUndoProcessed/golden/basic_move_new_data b/internal/report/testdata/TestUndoProcessed/golden/basic_move_new_data new file mode 100644 index 0000000..3330bd0 --- /dev/null +++ b/internal/report/testdata/TestUndoProcessed/golden/basic_move_new_data @@ -0,0 +1,7 @@ +report: + path: src/1.json + name: 1.json + timestamp: 1 +srcfiles: + 1.json: '{"test": true}' +dstfiles: {} diff --git a/internal/report/testdata/TestUndoProcessed/golden/basic_move_overwrite b/internal/report/testdata/TestUndoProcessed/golden/basic_move_overwrite new file mode 100644 index 0000000..3330bd0 --- /dev/null +++ b/internal/report/testdata/TestUndoProcessed/golden/basic_move_overwrite @@ -0,0 +1,7 @@ +report: + path: src/1.json + name: 1.json + timestamp: 1 +srcfiles: + 1.json: '{"test": true}' +dstfiles: {} diff --git a/internal/uploader/internal_test.go b/internal/uploader/internal_test.go index 27be531..6ececef 100644 --- a/internal/uploader/internal_test.go +++ b/internal/uploader/internal_test.go @@ -95,58 +95,6 @@ func TestUploadBadFile(t *testing.T) { } } -func TestMoveReport(t *testing.T) { - t.Parallel() - - tests := map[string]struct { - fileExists bool - - wantErr bool - }{ - "File Exists": {fileExists: true}, - - "File Not Found": {fileExists: false, wantErr: true}, - } - - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - t.Parallel() - dir := t.TempDir() - - um := &Uploader{ - collectedDir: filepath.Join(dir, constants.LocalFolder), - uploadedDir: filepath.Join(dir, constants.UploadedFolder), - } - - require.NoError(t, os.MkdirAll(um.collectedDir, 0750), "Setup: failed to create uploaded folder") - require.NoError(t, os.MkdirAll(um.uploadedDir, 0750), "Setup: failed to create collected folder") - - cDir := filepath.Join(um.collectedDir, "report.json") - uDir := filepath.Join(um.uploadedDir, "report.json") - if tc.fileExists { - f, err := os.Create(cDir) - require.NoError(t, err) - f.Close() - } - - err := um.moveReport(uDir, cDir, []byte("payload")) - if tc.wantErr { - require.Error(t, err) - return - } - require.NoError(t, err) - - _, err = os.Stat(uDir) - if !tc.fileExists { - require.Error(t, err, "File should not exist in the uploaded directory") - return - } - - require.NoError(t, err, "File should exist in the uploaded directory") - }) - } -} - func TestSend(t *testing.T) { t.Parallel() diff --git a/internal/uploader/upload.go b/internal/uploader/upload.go index cb04d46..9fcdec9 100644 --- a/internal/uploader/upload.go +++ b/internal/uploader/upload.go @@ -15,7 +15,6 @@ import ( "time" "github.com/ubuntu/ubuntu-insights/internal/constants" - "github.com/ubuntu/ubuntu-insights/internal/fileutils" "github.com/ubuntu/ubuntu-insights/internal/report" ) @@ -25,7 +24,9 @@ var ( ) // Upload uploads the reports corresponding to the source to the configured server. -// Does not do duplicate checks. +// +// It will only upload reports that are mature enough, and have not been uploaded before. +// If force is true, maturity and duplicate check will be skipped. func (um Uploader) Upload(force bool) error { slog.Debug("Uploading reports") @@ -101,13 +102,13 @@ func (um Uploader) upload(r report.Report, url string, consent, force bool) erro } // Move report first to avoid the situation where the report is sent, but not marked as sent. - // TODO: maybe a method on Reports ? - if err := um.moveReport(filepath.Join(um.uploadedDir, r.Name), filepath.Join(um.collectedDir, r.Name), data); err != nil { - return fmt.Errorf("failed to move report after uploading: %v", err) + r, err = r.MarkAsProcessed(um.uploadedDir, data) + if err != nil { + return fmt.Errorf("failed to mark report as processed: %v", err) } if err := send(url, data); err != nil { - if moveErr := um.moveReport(filepath.Join(um.collectedDir, r.Name), filepath.Join(um.uploadedDir, r.Name), origData); moveErr != nil { - return fmt.Errorf("failed to send data: %v, and failed to restore the original report: %v", err, moveErr) + if _, err := r.UndoProcessed(); err != nil { + return fmt.Errorf("failed to send data: %v, and failed to restore the original report: %v", err, err) } return fmt.Errorf("failed to send data: %v", err) } @@ -124,23 +125,6 @@ func (um Uploader) getURL() (string, error) { return u.String(), nil } -func (um Uploader) moveReport(writePath, removePath string, data []byte) error { - // (Report).MarkAsProcessed(data) - // (Report).UndoProcessed - // moveReport writes the data to the writePath, and removes the matching file from the removePath. - // dest, src, data - - if err := fileutils.AtomicWrite(writePath, data); err != nil { - return fmt.Errorf("failed to write report: %v", err) - } - - if err := os.Remove(removePath); err != nil { - return fmt.Errorf("failed to remove report: %v", err) - } - - return nil -} - func send(url string, data []byte) error { slog.Debug("Sending data to server", "url", url, "data", data) req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(data)) From dd85be6052072973cbe82c41c972e51f0d9f8329 Mon Sep 17 00:00:00 2001 From: hk21702 <28567881+hk21702@users.noreply.github.com> Date: Fri, 31 Jan 2025 16:53:53 -0500 Subject: [PATCH 23/26] Improve directory not found behavior --- .../testdata/TestUpload/golden/bad_content | 2 +- .../testdata/TestUpload/golden/bad_response | 2 +- .../testdata/TestUpload/golden/bad_url | 2 +- .../TestUpload/golden/consent_manager_false | 2 +- .../golden/consent_manager_global_error | 1 - .../consent_manager_global_error_with_true | 1 - .../consent_manager_global_false,_source_true | 1 - .../consent_manager_global_true,_source_false | 1 - .../golden/consent_manager_source_error | 2 +- .../consent_manager_source_error_with_true | 2 +- .../testdata/TestUpload/golden/dry_run | 2 +- .../TestUpload/golden/duplicate_upload | 4 +- .../testdata/TestUpload/golden/force_cm_false | 2 +- .../TestUpload/golden/force_duplicate | 2 +- .../testdata/TestUpload/golden/force_min_age | 4 +- .../TestUpload/golden/future_timestamp | 4 +- .../testdata/TestUpload/golden/min_age | 4 +- .../testdata/TestUpload/golden/multi_upload | 4 +- .../{invalid_directory => no_directory} | 0 .../TestUpload/golden/no_reports_with_dummy | 20 ++-- .../testdata/TestUpload/golden/offline_server | 2 +- .../TestUpload/golden/optout_payload_cm_false | 2 +- .../TestUpload/golden/optout_payload_cm_true | 2 +- .../testdata/TestUpload/golden/single_upload | 2 +- internal/uploader/upload.go | 14 +++ internal/uploader/uploader.go | 4 +- internal/uploader/uploader_test.go | 111 +++++++++++------- 27 files changed, 116 insertions(+), 83 deletions(-) delete mode 100644 internal/uploader/testdata/TestUpload/golden/consent_manager_global_error delete mode 100644 internal/uploader/testdata/TestUpload/golden/consent_manager_global_error_with_true delete mode 100644 internal/uploader/testdata/TestUpload/golden/consent_manager_global_false,_source_true delete mode 100644 internal/uploader/testdata/TestUpload/golden/consent_manager_global_true,_source_false rename internal/uploader/testdata/TestUpload/golden/{invalid_directory => no_directory} (100%) diff --git a/internal/uploader/testdata/TestUpload/golden/bad_content b/internal/uploader/testdata/TestUpload/golden/bad_content index be447b6..9f97665 100644 --- a/internal/uploader/testdata/TestUpload/golden/bad_content +++ b/internal/uploader/testdata/TestUpload/golden/bad_content @@ -1 +1 @@ -local/1.json: bad content +source/local/1.json: bad content diff --git a/internal/uploader/testdata/TestUpload/golden/bad_response b/internal/uploader/testdata/TestUpload/golden/bad_response index 0d60902..ec0b658 100644 --- a/internal/uploader/testdata/TestUpload/golden/bad_response +++ b/internal/uploader/testdata/TestUpload/golden/bad_response @@ -1 +1 @@ -local/1.json: '{"Content":"normal content"}' +source/local/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/bad_url b/internal/uploader/testdata/TestUpload/golden/bad_url index 0d60902..ec0b658 100644 --- a/internal/uploader/testdata/TestUpload/golden/bad_url +++ b/internal/uploader/testdata/TestUpload/golden/bad_url @@ -1 +1 @@ -local/1.json: '{"Content":"normal content"}' +source/local/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/consent_manager_false b/internal/uploader/testdata/TestUpload/golden/consent_manager_false index 4d2381a..587b133 100644 --- a/internal/uploader/testdata/TestUpload/golden/consent_manager_false +++ b/internal/uploader/testdata/TestUpload/golden/consent_manager_false @@ -1 +1 @@ -uploaded/1.json: '{"OptOut":true}' +source/uploaded/1.json: '{"OptOut":true}' diff --git a/internal/uploader/testdata/TestUpload/golden/consent_manager_global_error b/internal/uploader/testdata/TestUpload/golden/consent_manager_global_error deleted file mode 100644 index 0d60902..0000000 --- a/internal/uploader/testdata/TestUpload/golden/consent_manager_global_error +++ /dev/null @@ -1 +0,0 @@ -local/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/consent_manager_global_error_with_true b/internal/uploader/testdata/TestUpload/golden/consent_manager_global_error_with_true deleted file mode 100644 index 0d60902..0000000 --- a/internal/uploader/testdata/TestUpload/golden/consent_manager_global_error_with_true +++ /dev/null @@ -1 +0,0 @@ -local/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/consent_manager_global_false,_source_true b/internal/uploader/testdata/TestUpload/golden/consent_manager_global_false,_source_true deleted file mode 100644 index 4d2381a..0000000 --- a/internal/uploader/testdata/TestUpload/golden/consent_manager_global_false,_source_true +++ /dev/null @@ -1 +0,0 @@ -uploaded/1.json: '{"OptOut":true}' diff --git a/internal/uploader/testdata/TestUpload/golden/consent_manager_global_true,_source_false b/internal/uploader/testdata/TestUpload/golden/consent_manager_global_true,_source_false deleted file mode 100644 index 4d2381a..0000000 --- a/internal/uploader/testdata/TestUpload/golden/consent_manager_global_true,_source_false +++ /dev/null @@ -1 +0,0 @@ -uploaded/1.json: '{"OptOut":true}' diff --git a/internal/uploader/testdata/TestUpload/golden/consent_manager_source_error b/internal/uploader/testdata/TestUpload/golden/consent_manager_source_error index 0d60902..ec0b658 100644 --- a/internal/uploader/testdata/TestUpload/golden/consent_manager_source_error +++ b/internal/uploader/testdata/TestUpload/golden/consent_manager_source_error @@ -1 +1 @@ -local/1.json: '{"Content":"normal content"}' +source/local/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/consent_manager_source_error_with_true b/internal/uploader/testdata/TestUpload/golden/consent_manager_source_error_with_true index 0d60902..ec0b658 100644 --- a/internal/uploader/testdata/TestUpload/golden/consent_manager_source_error_with_true +++ b/internal/uploader/testdata/TestUpload/golden/consent_manager_source_error_with_true @@ -1 +1 @@ -local/1.json: '{"Content":"normal content"}' +source/local/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/dry_run b/internal/uploader/testdata/TestUpload/golden/dry_run index 0d60902..ec0b658 100644 --- a/internal/uploader/testdata/TestUpload/golden/dry_run +++ b/internal/uploader/testdata/TestUpload/golden/dry_run @@ -1 +1 @@ -local/1.json: '{"Content":"normal content"}' +source/local/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/duplicate_upload b/internal/uploader/testdata/TestUpload/golden/duplicate_upload index 382d755..e37f517 100644 --- a/internal/uploader/testdata/TestUpload/golden/duplicate_upload +++ b/internal/uploader/testdata/TestUpload/golden/duplicate_upload @@ -1,2 +1,2 @@ -local/1.json: '{"Content":"normal content"}' -uploaded/1.json: bad content +source/local/1.json: '{"Content":"normal content"}' +source/uploaded/1.json: bad content diff --git a/internal/uploader/testdata/TestUpload/golden/force_cm_false b/internal/uploader/testdata/TestUpload/golden/force_cm_false index 4d2381a..587b133 100644 --- a/internal/uploader/testdata/TestUpload/golden/force_cm_false +++ b/internal/uploader/testdata/TestUpload/golden/force_cm_false @@ -1 +1 @@ -uploaded/1.json: '{"OptOut":true}' +source/uploaded/1.json: '{"OptOut":true}' diff --git a/internal/uploader/testdata/TestUpload/golden/force_duplicate b/internal/uploader/testdata/TestUpload/golden/force_duplicate index 9dc4645..f436ae8 100644 --- a/internal/uploader/testdata/TestUpload/golden/force_duplicate +++ b/internal/uploader/testdata/TestUpload/golden/force_duplicate @@ -1 +1 @@ -uploaded/1.json: '{"Content":"normal content"}' +source/uploaded/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/force_min_age b/internal/uploader/testdata/TestUpload/golden/force_min_age index f12da56..e21264b 100644 --- a/internal/uploader/testdata/TestUpload/golden/force_min_age +++ b/internal/uploader/testdata/TestUpload/golden/force_min_age @@ -1,2 +1,2 @@ -uploaded/1.json: '{"Content":"normal content"}' -uploaded/9.json: '{"Content":"normal content"}' +source/uploaded/1.json: '{"Content":"normal content"}' +source/uploaded/9.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/future_timestamp b/internal/uploader/testdata/TestUpload/golden/future_timestamp index f8a9041..ce033c1 100644 --- a/internal/uploader/testdata/TestUpload/golden/future_timestamp +++ b/internal/uploader/testdata/TestUpload/golden/future_timestamp @@ -1,2 +1,2 @@ -local/11.json: '{"Content":"normal content"}' -uploaded/1.json: '{"Content":"normal content"}' +source/local/11.json: '{"Content":"normal content"}' +source/uploaded/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/min_age b/internal/uploader/testdata/TestUpload/golden/min_age index 463c163..85afcf0 100644 --- a/internal/uploader/testdata/TestUpload/golden/min_age +++ b/internal/uploader/testdata/TestUpload/golden/min_age @@ -1,2 +1,2 @@ -local/9.json: '{"Content":"normal content"}' -uploaded/1.json: '{"Content":"normal content"}' +source/local/9.json: '{"Content":"normal content"}' +source/uploaded/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/multi_upload b/internal/uploader/testdata/TestUpload/golden/multi_upload index 55b3ee3..91fd905 100644 --- a/internal/uploader/testdata/TestUpload/golden/multi_upload +++ b/internal/uploader/testdata/TestUpload/golden/multi_upload @@ -1,2 +1,2 @@ -uploaded/1.json: '{"Content":"normal content"}' -uploaded/5.json: '{"Content":"normal content"}' +source/uploaded/1.json: '{"Content":"normal content"}' +source/uploaded/5.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/invalid_directory b/internal/uploader/testdata/TestUpload/golden/no_directory similarity index 100% rename from internal/uploader/testdata/TestUpload/golden/invalid_directory rename to internal/uploader/testdata/TestUpload/golden/no_directory diff --git a/internal/uploader/testdata/TestUpload/golden/no_reports_with_dummy b/internal/uploader/testdata/TestUpload/golden/no_reports_with_dummy index 0a52bdc..dcd1986 100644 --- a/internal/uploader/testdata/TestUpload/golden/no_reports_with_dummy +++ b/internal/uploader/testdata/TestUpload/golden/no_reports_with_dummy @@ -9,25 +9,25 @@ dummy/dummy.json: |- } dummy/empty-dummy.json: "" empty-dummy.json: "" -local/-1.json: bad contents -local/dummy.json: |- +source/local/-1.json: bad contents +source/local/dummy.json: |- { "OS": "something" } -local/dummy/dummy.json: |- +source/local/dummy/dummy.json: |- { "OS": "something" } -local/dummy/empty-dummy.json: "" -local/empty-dummy.json: "" -uploaded/-1.json: bad contents -uploaded/dummy.json: |- +source/local/dummy/empty-dummy.json: "" +source/local/empty-dummy.json: "" +source/uploaded/-1.json: bad contents +source/uploaded/dummy.json: |- { "OS": "something" } -uploaded/dummy/dummy.json: |- +source/uploaded/dummy/dummy.json: |- { "OS": "something" } -uploaded/dummy/empty-dummy.json: "" -uploaded/empty-dummy.json: "" +source/uploaded/dummy/empty-dummy.json: "" +source/uploaded/empty-dummy.json: "" diff --git a/internal/uploader/testdata/TestUpload/golden/offline_server b/internal/uploader/testdata/TestUpload/golden/offline_server index 0d60902..ec0b658 100644 --- a/internal/uploader/testdata/TestUpload/golden/offline_server +++ b/internal/uploader/testdata/TestUpload/golden/offline_server @@ -1 +1 @@ -local/1.json: '{"Content":"normal content"}' +source/local/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/testdata/TestUpload/golden/optout_payload_cm_false b/internal/uploader/testdata/TestUpload/golden/optout_payload_cm_false index 4d2381a..587b133 100644 --- a/internal/uploader/testdata/TestUpload/golden/optout_payload_cm_false +++ b/internal/uploader/testdata/TestUpload/golden/optout_payload_cm_false @@ -1 +1 @@ -uploaded/1.json: '{"OptOut":true}' +source/uploaded/1.json: '{"OptOut":true}' diff --git a/internal/uploader/testdata/TestUpload/golden/optout_payload_cm_true b/internal/uploader/testdata/TestUpload/golden/optout_payload_cm_true index 4d2381a..587b133 100644 --- a/internal/uploader/testdata/TestUpload/golden/optout_payload_cm_true +++ b/internal/uploader/testdata/TestUpload/golden/optout_payload_cm_true @@ -1 +1 @@ -uploaded/1.json: '{"OptOut":true}' +source/uploaded/1.json: '{"OptOut":true}' diff --git a/internal/uploader/testdata/TestUpload/golden/single_upload b/internal/uploader/testdata/TestUpload/golden/single_upload index 9dc4645..f436ae8 100644 --- a/internal/uploader/testdata/TestUpload/golden/single_upload +++ b/internal/uploader/testdata/TestUpload/golden/single_upload @@ -1 +1 @@ -uploaded/1.json: '{"Content":"normal content"}' +source/uploaded/1.json: '{"Content":"normal content"}' diff --git a/internal/uploader/upload.go b/internal/uploader/upload.go index 9fcdec9..c14366a 100644 --- a/internal/uploader/upload.go +++ b/internal/uploader/upload.go @@ -29,6 +29,9 @@ var ( // If force is true, maturity and duplicate check will be skipped. func (um Uploader) Upload(force bool) error { slog.Debug("Uploading reports") + if err := um.makeDirs(); err != nil { + return err + } consent, err := um.consentM.HasConsent(um.source) if err != nil { @@ -146,3 +149,14 @@ func send(url string, data []byte) error { return nil } + +// makeDirs creates the directories for the collected and uploaded reports if they don't already exist. +func (um Uploader) makeDirs() error { + if err := os.MkdirAll(um.collectedDir, 0750); err != nil { + return fmt.Errorf("failed to create collected directory: %v", err) + } + if err := os.MkdirAll(um.uploadedDir, 0750); err != nil { + return fmt.Errorf("failed to create uploaded directory: %v", err) + } + return nil +} diff --git a/internal/uploader/uploader.go b/internal/uploader/uploader.go index 2469319..d1be313 100644 --- a/internal/uploader/uploader.go +++ b/internal/uploader/uploader.go @@ -78,7 +78,7 @@ func New(cm consentManager, source string, minAge uint, dryRun bool, args ...Opt timeProvider: opts.timeProvider, baseServerURL: opts.baseServerURL, - collectedDir: filepath.Join(opts.cachePath, constants.LocalFolder), - uploadedDir: filepath.Join(opts.cachePath, constants.UploadedFolder), + collectedDir: filepath.Join(opts.cachePath, source, constants.LocalFolder), + uploadedDir: filepath.Join(opts.cachePath, source, constants.UploadedFolder), }, nil } diff --git a/internal/uploader/uploader_test.go b/internal/uploader/uploader_test.go index 2a4d3d1..45a5757 100644 --- a/internal/uploader/uploader_test.go +++ b/internal/uploader/uploader_test.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "runtime" "testing" "github.com/stretchr/testify/require" @@ -67,57 +68,66 @@ func TestNew(t *testing.T) { func TestUpload(t *testing.T) { t.Parallel() - const mockTime = 10 + const ( + mockTime = 10 + defaultResponse = http.StatusOK + source = "source" + ) tests := map[string]struct { - localFiles, uploadedFiles map[string]reportType - dummy bool - serverResponse int - serverOffline bool - url string - invalidDir bool + lFiles, uFiles map[string]reportType + dummy bool + serverResponse int + serverOffline bool + url string + rmLocal bool + noPerms bool consent testConsentChecker minAge uint dryRun bool force bool - wantErr bool + skipContentCheck bool + wantErr bool }{ - "No Reports": {consent: cTrue, serverResponse: http.StatusOK}, - "No Reports with Dummy": {dummy: true, consent: cTrue, serverResponse: http.StatusOK}, - "Single Upload": {localFiles: map[string]reportType{"1.json": normal}, consent: cTrue, serverResponse: http.StatusOK}, - "Multi Upload": {localFiles: map[string]reportType{"1.json": normal, "5.json": normal}, consent: cTrue, serverResponse: http.StatusOK}, - "Min Age": {localFiles: map[string]reportType{"1.json": normal, "9.json": normal}, consent: cTrue, minAge: 5, serverResponse: http.StatusOK}, - "Future Timestamp": {localFiles: map[string]reportType{"1.json": normal, "11.json": normal}, consent: cTrue, serverResponse: http.StatusOK}, - "Duplicate Upload": {localFiles: map[string]reportType{"1.json": normal}, uploadedFiles: map[string]reportType{"1.json": badContent}, consent: cTrue, serverResponse: http.StatusAccepted}, - "Bad Content": {localFiles: map[string]reportType{"1.json": badContent}, consent: cTrue, serverResponse: http.StatusOK}, - - "Consent Manager Source Error": {localFiles: map[string]reportType{"1.json": normal}, consent: cErr, serverResponse: http.StatusOK, wantErr: true}, - "Consent Manager Source Error with True": {localFiles: map[string]reportType{"1.json": normal}, consent: cErrTrue, serverResponse: http.StatusOK, wantErr: true}, - "Consent Manager False": {localFiles: map[string]reportType{"1.json": normal}, consent: cFalse, serverResponse: http.StatusOK}, - - "Force CM False": {localFiles: map[string]reportType{"1.json": normal}, consent: cFalse, force: true, serverResponse: http.StatusOK}, - "Force Min Age": {localFiles: map[string]reportType{"1.json": normal, "9.json": normal}, consent: cTrue, minAge: 5, force: true, serverResponse: http.StatusOK}, - "Force Duplicate": {localFiles: map[string]reportType{"1.json": normal}, uploadedFiles: map[string]reportType{"1.json": badContent}, consent: cTrue, force: true, serverResponse: http.StatusOK}, - - "OptOut Payload CM True": {localFiles: map[string]reportType{"1.json": optOut}, consent: cTrue, serverResponse: http.StatusOK}, - "OptOut Payload CM False": {localFiles: map[string]reportType{"1.json": optOut}, consent: cFalse, serverResponse: http.StatusOK}, - - "Dry run": {localFiles: map[string]reportType{"1.json": normal}, consent: cTrue, dryRun: true}, - - "Bad URL": {localFiles: map[string]reportType{"1.json": normal}, consent: cTrue, url: "http://a b.com/", wantErr: true}, - "Bad Response": {localFiles: map[string]reportType{"1.json": normal}, consent: cTrue, serverResponse: http.StatusForbidden}, - "Offline Server": {localFiles: map[string]reportType{"1.json": normal}, consent: cTrue, serverOffline: true}, - - "Invalid Directory": {localFiles: map[string]reportType{"1.json": normal}, consent: cTrue, invalidDir: true, wantErr: true}, + "No Reports": {consent: cTrue}, + "No Reports with Dummy": {dummy: true, consent: cTrue}, + "Single Upload": {lFiles: map[string]reportType{"1.json": normal}, consent: cTrue}, + "Multi Upload": {lFiles: map[string]reportType{"1.json": normal, "5.json": normal}, consent: cTrue}, + "Min Age": {lFiles: map[string]reportType{"1.json": normal, "9.json": normal}, consent: cTrue, minAge: 5}, + "Future Timestamp": {lFiles: map[string]reportType{"1.json": normal, "11.json": normal}, consent: cTrue}, + "Duplicate Upload": {lFiles: map[string]reportType{"1.json": normal}, uFiles: map[string]reportType{"1.json": badContent}, consent: cTrue}, + "Bad Content": {lFiles: map[string]reportType{"1.json": badContent}, consent: cTrue}, + "No Directory": {lFiles: map[string]reportType{"1.json": normal}, consent: cTrue, rmLocal: true}, + + "Consent Manager Source Error": {lFiles: map[string]reportType{"1.json": normal}, consent: cErr, wantErr: true}, + "Consent Manager Source Error with True": {lFiles: map[string]reportType{"1.json": normal}, consent: cErrTrue, wantErr: true}, + "Consent Manager False": {lFiles: map[string]reportType{"1.json": normal}, consent: cFalse}, + + "Force CM False": {lFiles: map[string]reportType{"1.json": normal}, consent: cFalse, force: true}, + "Force Min Age": {lFiles: map[string]reportType{"1.json": normal, "9.json": normal}, consent: cTrue, minAge: 5, force: true}, + "Force Duplicate": {lFiles: map[string]reportType{"1.json": normal}, uFiles: map[string]reportType{"1.json": badContent}, consent: cTrue, force: true}, + + "OptOut Payload CM True": {lFiles: map[string]reportType{"1.json": optOut}, consent: cTrue}, + "OptOut Payload CM False": {lFiles: map[string]reportType{"1.json": optOut}, consent: cFalse}, + + "Dry run": {lFiles: map[string]reportType{"1.json": normal}, consent: cTrue, dryRun: true}, + + "Bad URL": {lFiles: map[string]reportType{"1.json": normal}, consent: cTrue, url: "http://a b.com/", wantErr: true}, + "Bad Response": {lFiles: map[string]reportType{"1.json": normal}, consent: cTrue, serverResponse: http.StatusForbidden}, + "Offline Server": {lFiles: map[string]reportType{"1.json": normal}, consent: cTrue, serverOffline: true}, + "No Permissions": {lFiles: map[string]reportType{"1.json": normal}, consent: cTrue, noPerms: true, wantErr: runtime.GOOS != "windows", skipContentCheck: true}, } for name, tc := range tests { t.Run(name, func(t *testing.T) { t.Parallel() + dir := setupTmpDir(t, tc.lFiles, tc.uFiles, source, tc.dummy) - dir := setupTmpDir(t, tc.localFiles, tc.uploadedFiles, tc.dummy) + if tc.serverResponse == 0 { + tc.serverResponse = defaultResponse + } if !tc.serverOffline { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -129,11 +139,15 @@ func TestUpload(t *testing.T) { } } - if tc.invalidDir { - require.NoError(t, os.RemoveAll(filepath.Join(dir, "local")), "Setup: failed to remove local directory") + localDir := filepath.Join(dir, source, constants.LocalFolder) + if tc.rmLocal { + require.NoError(t, os.RemoveAll(localDir), "Setup: failed to remove local directory") + } else if tc.noPerms { + require.NoError(t, os.Chmod(localDir, 0), "Setup: failed to remove local directory") + t.Cleanup(func() { require.NoError(t, os.Chmod(localDir, 0750), "Cleanup: failed to restore permissions") }) //nolint:gosec //0750 is fine for folders } - mgr, err := uploader.New(tc.consent, "source", tc.minAge, tc.dryRun, + mgr, err := uploader.New(tc.consent, source, tc.minAge, tc.dryRun, uploader.WithBaseServerURL(tc.url), uploader.WithCachePath(dir), uploader.WithTimeProvider(uploader.MockTimeProvider{CurrentTime: mockTime})) require.NoError(t, err, "Setup: failed to create new uploader manager") @@ -144,6 +158,15 @@ func TestUpload(t *testing.T) { require.NoError(t, err) } + if tc.noPerms { + //nolint:gosec //0750 is fine for folders + require.NoError(t, os.Chmod(localDir, 0750), "Post: failed to restore permissions") + } + + if tc.skipContentCheck { + return + } + got, err := testutils.GetDirContents(t, dir, 3) require.NoError(t, err) want := testutils.LoadWithUpdateFromGoldenYAML(t, got) @@ -152,14 +175,14 @@ func TestUpload(t *testing.T) { } } -func setupTmpDir(t *testing.T, localFiles, uploadedFiles map[string]reportType, dummy bool) string { +func setupTmpDir(t *testing.T, localFiles, uploadedFiles map[string]reportType, source string, dummy bool) string { t.Helper() dir := t.TempDir() - localDir := filepath.Join(dir, "local") - uploadedDir := filepath.Join(dir, "uploaded") - require.NoError(t, os.Mkdir(localDir, 0750), "Setup: failed to create local directory") - require.NoError(t, os.Mkdir(uploadedDir, 0750), "Setup: failed to create uploaded directory") + localDir := filepath.Join(dir, source, constants.LocalFolder) + uploadedDir := filepath.Join(dir, source, constants.UploadedFolder) + require.NoError(t, os.MkdirAll(localDir, 0750), "Setup: failed to create local directory") + require.NoError(t, os.MkdirAll(uploadedDir, 0750), "Setup: failed to create uploaded directory") if dummy { copyDummyData(t, "testdata/test_source", dir, localDir, uploadedDir) From 6b068591be1e66d9c010bb2d8f7ed39e982854d7 Mon Sep 17 00:00:00 2001 From: hk21702 <28567881+hk21702@users.noreply.github.com> Date: Fri, 31 Jan 2025 17:51:56 -0500 Subject: [PATCH 24/26] Fix uploader minAge overflow protection --- internal/uploader/upload.go | 2 +- internal/uploader/uploader.go | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/internal/uploader/upload.go b/internal/uploader/upload.go index c14366a..7666205 100644 --- a/internal/uploader/upload.go +++ b/internal/uploader/upload.go @@ -71,7 +71,7 @@ func (um Uploader) Upload(force bool) error { func (um Uploader) upload(r report.Report, url string, consent, force bool) error { slog.Debug("Uploading report", "file", r.Name, "consent", consent, "force", force) - if um.timeProvider.Now().Add(time.Duration(-um.minAge)*time.Second).Before(time.Unix(r.TimeStamp, 0)) && !force { + if um.timeProvider.Now().Add(-um.minAge).Before(time.Unix(r.TimeStamp, 0)) && !force { return ErrReportNotMature } diff --git a/internal/uploader/uploader.go b/internal/uploader/uploader.go index d1be313..b2aa83f 100644 --- a/internal/uploader/uploader.go +++ b/internal/uploader/uploader.go @@ -5,7 +5,6 @@ package uploader import ( "fmt" "log/slog" - "math" "path/filepath" "time" @@ -26,7 +25,7 @@ func (realTimeProvider) Now() time.Time { type Uploader struct { source string consentM consentManager - minAge int64 + minAge time.Duration dryRun bool baseServerURL string @@ -57,7 +56,7 @@ func New(cm consentManager, source string, minAge uint, dryRun bool, args ...Opt return Uploader{}, fmt.Errorf("source cannot be an empty string") } - if minAge > math.MaxInt64 { + if minAge > (1<<63-1)/uint(time.Second) { return Uploader{}, fmt.Errorf("min age %d is too large, would overflow", minAge) } @@ -73,7 +72,7 @@ func New(cm consentManager, source string, minAge uint, dryRun bool, args ...Opt return Uploader{ source: source, consentM: cm, - minAge: int64(minAge), + minAge: time.Duration(minAge) * time.Second, dryRun: dryRun, timeProvider: opts.timeProvider, From 87ed3d637276aca8a060cc091ae5a79b0c48cb1c Mon Sep 17 00:00:00 2001 From: kat <28567881+hk21702@users.noreply.github.com> Date: Mon, 3 Feb 2025 10:25:20 -0500 Subject: [PATCH 25/26] Remove TODO at duplicate report checking --- internal/uploader/upload.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/uploader/upload.go b/internal/uploader/upload.go index 7666205..c6e86bb 100644 --- a/internal/uploader/upload.go +++ b/internal/uploader/upload.go @@ -81,8 +81,6 @@ func (um Uploader) upload(r report.Report, url string, consent, force bool) erro return fmt.Errorf("failed to check if report has already been uploaded: %v", err) } if err == nil && !force { - // TODO: What to do with the original file? Should we clean it up? - // Should we move it elsewhere for investigation in a "tmp" and clean it afterwards? return fmt.Errorf("report has already been uploaded") } From 0378ed152d649fbd99c7622a38f8f37d6ec78193 Mon Sep 17 00:00:00 2001 From: hk21702 <28567881+hk21702@users.noreply.github.com> Date: Mon, 3 Feb 2025 14:41:36 -0500 Subject: [PATCH 26/26] Small fixes --- internal/report/report.go | 16 +++++++--------- internal/report/report_test.go | 11 ++++++++--- .../testdata/TestGetAll/golden/files_in_subdir | 6 ------ .../TestGetAll/golden/mix_of_valid_and_invalid | 9 --------- .../TestGetPeriodStart/golden/negative_time | 1 + .../TestGetPeriodStart/golden/non-multiple_time | 1 + .../TestGetPeriodStart/golden/valid_period | 1 + 7 files changed, 18 insertions(+), 27 deletions(-) create mode 100644 internal/report/testdata/TestGetPeriodStart/golden/negative_time create mode 100644 internal/report/testdata/TestGetPeriodStart/golden/non-multiple_time create mode 100644 internal/report/testdata/TestGetPeriodStart/golden/valid_period diff --git a/internal/report/report.go b/internal/report/report.go index d866fad..6116851 100644 --- a/internal/report/report.go +++ b/internal/report/report.go @@ -129,29 +129,27 @@ func getReportTime(path string) (int64, error) { } // GetPeriodStart returns the start of the period window for a given period in seconds. -func GetPeriodStart(period int) (int64, error) { +func GetPeriodStart(period int, t time.Time) (int64, error) { if period <= 0 { return 0, ErrInvalidPeriod } - utcTime := time.Now().UTC().Unix() - return utcTime - (utcTime % int64(period)), nil + return t.Unix() - (t.Unix() % int64(period)), nil } // GetForPeriod returns the most recent report within a period window for a given directory. // Not inclusive of the period end (periodStart + period). // // For example, given reports 1 and 7, with time 2 and period 7, the function will return the path for report 1. -func GetForPeriod(dir string, time time.Time, period int) (Report, error) { - if period <= 0 { - return Report{}, ErrInvalidPeriod +func GetForPeriod(dir string, t time.Time, period int) (Report, error) { + periodStart, err := GetPeriodStart(period, t) + if err != nil { + return Report{}, err } - - periodStart := time.Unix() - (time.Unix() % int64(period)) periodEnd := periodStart + int64(period) // Reports names are utc timestamps. Get the most recent report within the period window. var report Report - err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + err = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { if err != nil { return fmt.Errorf("failed to access path: %v", err) } diff --git a/internal/report/report_test.go b/internal/report/report_test.go index b41803e..0764e69 100644 --- a/internal/report/report_test.go +++ b/internal/report/report_test.go @@ -17,10 +17,13 @@ func TestGetPeriodStart(t *testing.T) { tests := map[string]struct { period int + time int64 wantErr error }{ - "Valid Period": {period: 500}, + "Valid Period": {period: 500, time: 100000}, + "Negative Time:": {period: 500, time: -100000}, + "Non-Multiple Time": {period: 500, time: 1051}, "Invalid Negative Period": {period: -500, wantErr: report.ErrInvalidPeriod}, "Invalid Zero Period": {period: 0, wantErr: report.ErrInvalidPeriod}, @@ -30,7 +33,7 @@ func TestGetPeriodStart(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - got, err := report.GetPeriodStart(tc.period) + got, err := report.GetPeriodStart(tc.period, time.Unix(tc.time, 0)) if tc.wantErr != nil { require.ErrorIs(t, err, tc.wantErr) return @@ -38,6 +41,8 @@ func TestGetPeriodStart(t *testing.T) { require.NoError(t, err, "got an unexpected error") require.IsType(t, int64(0), got) + want := testutils.LoadWithUpdateFromGoldenYAML(t, got) + require.Equal(t, want, got, "GetPeriodStart should return the expect start of the period window") }) } } @@ -233,7 +238,7 @@ func TestGetAll(t *testing.T) { } require.NoError(t, err, "got an unexpected error") - got := make([]report.Report, len(reports)) + got := make([]report.Report, 0, len(reports)) for _, r := range reports { got = append(got, sanitizeReportPath(t, r, dir)) } diff --git a/internal/report/testdata/TestGetAll/golden/files_in_subdir b/internal/report/testdata/TestGetAll/golden/files_in_subdir index ccc5761..a079365 100644 --- a/internal/report/testdata/TestGetAll/golden/files_in_subdir +++ b/internal/report/testdata/TestGetAll/golden/files_in_subdir @@ -1,9 +1,3 @@ -- path: "" - name: "" - timestamp: 0 -- path: "" - name: "" - timestamp: 0 - path: 1.json name: 1.json timestamp: 1 diff --git a/internal/report/testdata/TestGetAll/golden/mix_of_valid_and_invalid b/internal/report/testdata/TestGetAll/golden/mix_of_valid_and_invalid index e2ad51f..cd1e267 100644 --- a/internal/report/testdata/TestGetAll/golden/mix_of_valid_and_invalid +++ b/internal/report/testdata/TestGetAll/golden/mix_of_valid_and_invalid @@ -1,12 +1,3 @@ -- path: "" - name: "" - timestamp: 0 -- path: "" - name: "" - timestamp: 0 -- path: "" - name: "" - timestamp: 0 - path: 1.json name: 1.json timestamp: 1 diff --git a/internal/report/testdata/TestGetPeriodStart/golden/negative_time b/internal/report/testdata/TestGetPeriodStart/golden/negative_time new file mode 100644 index 0000000..c6730fa --- /dev/null +++ b/internal/report/testdata/TestGetPeriodStart/golden/negative_time @@ -0,0 +1 @@ +-100000 diff --git a/internal/report/testdata/TestGetPeriodStart/golden/non-multiple_time b/internal/report/testdata/TestGetPeriodStart/golden/non-multiple_time new file mode 100644 index 0000000..83b33d2 --- /dev/null +++ b/internal/report/testdata/TestGetPeriodStart/golden/non-multiple_time @@ -0,0 +1 @@ +1000 diff --git a/internal/report/testdata/TestGetPeriodStart/golden/valid_period b/internal/report/testdata/TestGetPeriodStart/golden/valid_period new file mode 100644 index 0000000..f7393e8 --- /dev/null +++ b/internal/report/testdata/TestGetPeriodStart/golden/valid_period @@ -0,0 +1 @@ +100000