Initial uploader

cmd/insights/commands/upload.go

@@ -4,6 +4,7 @@ import (
 	"log/slog"
 
 	"github.com/spf13/cobra"
+	"github.com/ubuntu/ubuntu-insights/internal/constants"
 )
 
 type uploadConfig struct {
@@ -16,7 +17,7 @@ type uploadConfig struct {
 
 var defaultUploadConfig = uploadConfig{
 	sources: []string{""},
-	server:  "https://metrics.ubuntu.com",
+	server:  constants.DefaultServerURL,
 	minAge:  604800,
 	force:   false,
 	dryRun:  false,

internal/constants/constants.go

@@ -18,6 +18,15 @@ const (
 	// DefaultLogLevel is the default log level selected without any verbosity flags.
 	DefaultLogLevel = slog.LevelInfo
 
+	// DefaultServerURL is the default base URL for the server that reports are uploaded to.
+	DefaultServerURL = "https://metrics.ubuntu.com"
+
+	// LocalFolder is the default name of the local collected reports folder.
+	LocalFolder = "local"
+
+	// UploadedFolder is the default name of the uploaded reports folder.
+	UploadedFolder = "uploaded"
+
 	// GlobalFileName is the default base name of the consent state files.
 	GlobalFileName = "consent.toml"
 
@@ -26,11 +35,11 @@ const (
 
 	// ReportExt is the default extension for the report files.
 	ReportExt = ".json"
-
-	// OptOutJSON is the data sent in case of Opt-Out choice.
-	OptOutJSON = `{"OptOut": true}`
 )
 
+// OptOutJSON is the data sent in case of Opt-Out choice.
+var OptOutJSON = struct{ OptOut bool }{OptOut: true}
+
 type options struct {
 	baseDir func() (string, error)
 }

internal/fileutils/fileutils.go

@@ -2,7 +2,6 @@
 package fileutils
 
 import (
-	"errors"
 	"fmt"
 	"log/slog"
 	"os"
@@ -13,7 +12,7 @@ import (
 // If the file already exists, then it will be overwritten.
 // Not atomic on Windows.
 func AtomicWrite(path string, data []byte) error {
-	tmp, err := os.CreateTemp(filepath.Dir(path), "consent-*.tmp")
+	tmp, err := os.CreateTemp(filepath.Dir(path), "tmp-*.tmp")
 	if err != nil {
 		return fmt.Errorf("could not create temporary file: %v", err)
 	}
@@ -37,12 +36,3 @@ func AtomicWrite(path string, data []byte) error {
 	}
 	return nil
 }
-
-// FileExists checks if a file exists at the given path.
-func FileExists(path string) (bool, error) {
-	_, err := os.Stat(path)
-	if err != nil && !errors.Is(err, os.ErrNotExist) {
-		return false, err
-	}
-	return !errors.Is(err, os.ErrNotExist), nil
-}

internal/fileutils/fileutils_test.go

@@ -15,16 +15,19 @@ func TestAtomicWrite(t *testing.T) {
 	tests := map[string]struct {
 		data       []byte
 		fileExists bool
+		invalidDir bool
 
 		wantError bool
 	}{
-		"Empty file":          {data: []byte{}, fileExists: false},
-		"Non-empty file":      {data: []byte("data"), fileExists: false},
+		"Empty file":          {data: []byte{}},
+		"Non-empty file":      {data: []byte("data")},
 		"Override file":       {data: []byte("data"), fileExists: true},
 		"Override empty file": {data: []byte{}, fileExists: true},
 
 		"Existing empty file":     {data: []byte{}, fileExists: true},
 		"Existing non-empty file": {data: []byte("data"), fileExists: true},
+
+		"Invalid Dir": {data: []byte("data"), invalidDir: true, wantError: true},
 	}
 
 	for name, tc := range tests {
@@ -34,6 +37,10 @@ func TestAtomicWrite(t *testing.T) {
 			oldFile := []byte("Old File!")
 			tempDir := t.TempDir()
 			path := filepath.Join(tempDir, "file")
+			if tc.invalidDir {
+				path = filepath.Join(path, "fake_dir")
+			}
+
 			if tc.fileExists {
 				err := fileutils.AtomicWrite(path, oldFile)
 				require.NoError(t, err, "Setup: AtomicWrite should not return an error")
@@ -44,60 +51,26 @@ func TestAtomicWrite(t *testing.T) {
 				require.Error(t, err, "AtomicWrite should return an error")
 
 				// Check that the file was not overwritten
-				data, err := os.ReadFile(path)
-				require.NoError(t, err, "ReadFile should not return an error")
-				require.Equal(t, oldFile, data, "AtomicWrite should not overwrite the file")
-			} else {
-				require.NoError(t, err, "AtomicWrite should not return an error")
+				if !tc.fileExists {
+					return
+				}
+
+				if tc.invalidDir {
+					path = filepath.Dir(path)
+				}
 
-				// Check that the file was written
 				data, err := os.ReadFile(path)
 				require.NoError(t, err, "ReadFile should not return an error")
-				require.Equal(t, tc.data, data, "AtomicWrite should write the data to the file")
-			}
-		})
-	}
-}
-
-func TestFileExists(t *testing.T) {
-	t.Parallel()
-
-	tests := map[string]struct {
-		fileExists      bool
-		parentDirIsFile bool
-
-		wantExists bool
-		wantError  bool
-	}{
-		"Returns_true_when_file_exists":                      {fileExists: true, wantExists: true},
-		"Returns_false_when_file_does_not_exist":             {fileExists: false, wantExists: false},
-		"Returns_false_when_parent_directory_does_not_exist": {fileExists: false, wantExists: false},
-
-		"Error_when_parent_directory_is_a_file": {parentDirIsFile: true, wantError: true},
-	}
-	for name, tc := range tests {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
+				require.Equal(t, oldFile, data, "AtomicWrite should not overwrite the file")
 
-			tempDir := t.TempDir()
-			path := filepath.Join(tempDir, "file")
-			if tc.fileExists {
-				err := fileutils.AtomicWrite(path, []byte{})
-				require.NoError(t, err, "AtomicWrite should not return an error")
-			}
-			if tc.parentDirIsFile {
-				path = filepath.Join(tempDir, "file", "file")
-				err := fileutils.AtomicWrite(filepath.Join(tempDir, "file"), []byte{})
-				require.NoError(t, err, "AtomicWrite should not return an error")
+				return
 			}
+			require.NoError(t, err, "AtomicWrite should not return an error")
 
-			exists, err := fileutils.FileExists(path)
-			if tc.wantError {
-				require.Error(t, err, "FileExists should return an error")
-			} else {
-				require.NoError(t, err, "FileExists should not return an error")
-			}
-			require.Equal(t, tc.wantExists, exists, "FileExists should return the expected result")
+			// Check that the file was written
+			data, err := os.ReadFile(path)
+			require.NoError(t, err, "ReadFile should not return an error")
+			require.Equal(t, tc.data, data, "AtomicWrite should write the data to the file")
 		})
 	}
 }

internal/reportutils/reportutils.go

@@ -45,69 +45,122 @@ func GetReportPath(dir string, time int64, period int) (string, error) {
 
 	// Reports names are utc timestamps. Get the most recent report within the period window.
 	var mostRecentReportPath string
-	files, err := os.ReadDir(dir)
-	if err != nil {
-		slog.Error("Failed to read directory", "directory", dir, "error", err)
-		return "", err
-	}
+	err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
+		if err != nil {
+			slog.Error("Failed to access path", "path", path, "error", err)
+			return err
+		}
 
-	for _, file := range files {
-		if filepath.Ext(file.Name()) != constants.ReportExt {
-			slog.Info("Skipping non-report file, invalid extension", "file", file.Name())
-			continue
+		// Skip subdirectories.
+		if d.IsDir() && path != dir {
+			return filepath.SkipDir
 		}
 
-		reportTime, err := GetReportTime(file.Name())
+		if filepath.Ext(d.Name()) != constants.ReportExt {
+			slog.Info("Skipping non-report file, invalid extension", "file", d.Name())
+			return nil
+		}
+
+		reportTime, err := GetReportTime(d.Name())
 		if err != nil {
-			slog.Info("Skipping non-report file, invalid file name", "file", file.Name())
-			continue
+			slog.Info("Skipping non-report file, invalid file name", "file", d.Name())
+			return nil
 		}
 
 		if reportTime < periodStart {
-			continue
+			return nil
 		}
 		if reportTime >= periodEnd {
-			break
+			return filepath.SkipDir
 		}
 
-		mostRecentReportPath = filepath.Join(dir, file.Name())
+		mostRecentReportPath = path
+		return nil
+	})
+
+	if err != nil {
+		return "", err
 	}
 
 	return mostRecentReportPath, nil
 }
 
 // GetReports returns the paths for the latest report within each period window for a given directory.
 // The key of the map is the start of the period window, and the report timestamp.
+//
+// If period is 1, then all reports in the dir are returned.
 func GetReports(dir string, period int) (map[int64]int64, error) {
 	if period <= 0 {
 		return nil, ErrInvalidPeriod
 	}
 
-	// Get the most recent report within each period window.
-	files, err := os.ReadDir(dir)
-	if err != nil {
-		slog.Error("Failed to read directory", "directory", dir, "error", err)
-		return nil, err
-	}
-
-	// Map to store the most recent report within each period window.
 	reports := make(map[int64]int64)
-	for _, file := range files {
-		if filepath.Ext(file.Name()) != constants.ReportExt {
-			slog.Info("Skipping non-report file, invalid extension", "file", file.Name())
-			continue
+	err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
+		if err != nil {
+			slog.Error("Failed to access path", "path", path, "error", err)
+			return err
 		}
 
-		reportTime, err := GetReportTime(file.Name())
+		if d.IsDir() && path != dir {
+			return filepath.SkipDir
+		}
+
+		if filepath.Ext(d.Name()) != constants.ReportExt {
+			slog.Info("Skipping non-report file, invalid extension", "file", d.Name())
+			return nil
+		}
+
+		reportTime, err := GetReportTime(d.Name())
 		if err != nil {
-			slog.Info("Skipping non-report file, invalid file name", "file", file.Name())
-			continue
+			slog.Info("Skipping non-report file, invalid file name", "file", d.Name())
+			return nil
 		}
 
 		periodStart := reportTime - (reportTime % int64(period))
 		if existingReport, ok := reports[periodStart]; !ok || existingReport < reportTime {
 			reports[periodStart] = reportTime
 		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return reports, nil
+}
+
+// GetAllReports returns the filename for all reports within a given directory, which match the expected pattern.
+// Does not traverse subdirectories.
+func GetAllReports(dir string) ([]string, error) {
+	reports := make([]string, 0)
+	err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
+		if err != nil {
+			slog.Error("Failed to access path", "path", path, "error", err)
+			return err
+		}
+
+		if d.IsDir() && path != dir {
+			return filepath.SkipDir
+		}
+
+		if filepath.Ext(d.Name()) != constants.ReportExt {
+			slog.Info("Skipping non-report file, invalid extension", "file", d.Name())
+			return nil
+		}
+
+		if _, err := GetReportTime(d.Name()); err != nil {
+			slog.Info("Skipping non-report file, invalid file name", "file", d.Name())
+			return nil
+		}
+
+		reports = append(reports, d.Name())
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
 	}
 
 	return reports, nil

internal/reportutils/reportutils_test.go

@@ -176,6 +176,7 @@ func TestGetReports(t *testing.T) {
 		"Get Newest of Period":             {files: []string{"1.json", "7.json"}, period: 100},
 		"Multiple Consecutive Windows":     {files: []string{"1.json", "7.json", "101.json", "107.json", "201.json", "207.json"}, period: 100},
 		"Multiple Non-Consecutive Windows": {files: []string{"1.json", "7.json", "101.json", "107.json", "251.json", "257.json"}, period: 50},
+		"Get All Reports":                  {files: []string{"1.json", "2.json", "3.json", "101.json", "107.json", "251.json", "257.json"}, period: 1},
 	}
 
 	for name, tc := range tests {
@@ -198,3 +199,40 @@ func TestGetReports(t *testing.T) {
 		})
 	}
 }
+
+func TestGetAllReports(t *testing.T) {
+	t.Parallel()
+
+	tests := map[string]struct {
+		files       []string
+		subDir      string
+		subDirFiles []string
+
+		wantErr error
+	}{
+		"Empty Directory":          {},
+		"Files in subDir":          {files: []string{"1.json", "2.json"}, subDir: "subdir", subDirFiles: []string{"1.json", "2.json"}},
+		"Invalid File Extension":   {files: []string{"1.txt", "2.txt"}},
+		"Invalid File Names":       {files: []string{"i-1.json", "i-2.json", "i-3.json", "test.json", "one.json"}},
+		"Mix of Valid and Invalid": {files: []string{"1.json", "2.json", "500.json", "i-1.json", "i-2.json", "i-3.json", "test.json", "five.json"}},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			dir, err := setupTmpDir(t, tc.files, tc.subDir, tc.subDirFiles)
+			require.NoError(t, err, "Setup: failed to setup temporary directory")
+			defer os.RemoveAll(dir)
+
+			got, err := reportutils.GetAllReports(dir)
+			if tc.wantErr != nil {
+				require.ErrorIs(t, err, tc.wantErr)
+				return
+			}
+			require.NoError(t, err, "got an unexpected error")
+
+			want := testutils.LoadWithUpdateFromGoldenYAML(t, got)
+			require.Equal(t, want, got, "GetAllReports should return all reports in the directory")
+		})
+	}
+}

internal/reportutils/testdata/TestGetAllReports/golden/empty_directory

@@ -0,0 +1 @@
+[]

internal/reportutils/testdata/TestGetAllReports/golden/files_in_subdir

@@ -0,0 +1,2 @@
+- 1.json
+- 2.json

internal/reportutils/testdata/TestGetAllReports/golden/invalid_file_extension

@@ -0,0 +1 @@
+[]

internal/reportutils/testdata/TestGetAllReports/golden/invalid_file_names

@@ -0,0 +1 @@
+[]