From 047617a2ed29ef88de1c8bd90a714347c06239b7 Mon Sep 17 00:00:00 2001 From: mirkobrombin Date: Mon, 8 Jul 2024 08:20:07 +0200 Subject: [PATCH] feat: Add verbose logging to file processing --- pkg/processor/cp.go | 25 +++++++++++++++++++++++++ pkg/processor/dedup.go | 36 ++++++++++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/pkg/processor/cp.go b/pkg/processor/cp.go index 29115d5..66a8257 100644 --- a/pkg/processor/cp.go +++ b/pkg/processor/cp.go @@ -2,6 +2,7 @@ package processor import ( "fmt" + "log" "os" "path/filepath" @@ -38,15 +39,25 @@ func NewCpProcessor(sourceFile, destFile string, storage *storage.Storage, hashG // Process processes the file and creates a link at the destination func (p *CpProcessor) Process(verbose bool) (err error) { + if verbose { + log.Printf("Processing file: %s", p.SourceFile) + } + // Compute file hash var finalHash string if p.Storage.Opts.WithMetadata { + if verbose { + log.Println("Computing full hash with metadata") + } finalHash, err = p.HashGen.ComputeFullHash(p.SourceFile) if err != nil { return fmt.Errorf("computing full hash: %w", err) } } else { + if verbose { + log.Println("Computing content hash without metadata") + } finalHash, err = p.HashGen.ComputeFileHash(p.SourceFile) if err != nil { return fmt.Errorf("computing content hash: %w", err) @@ -63,19 +74,33 @@ func (p *CpProcessor) Process(verbose bool) (err error) { // If the file does not exist, move it to storage if !exists { + if verbose { + log.Printf("File does not exist in storage, moving it: %s", dedupPath) + } err = p.Storage.MoveFileToStorage(p.SourceFile, finalHash) if err != nil { return fmt.Errorf("moving file to storage: %w", err) } + } else { + if verbose { + log.Printf("File already exists in storage: %s", dedupPath) + } } // Create a link at the destination pointing to the file in storage, // removing the destination if it already exists + if verbose { + log.Printf("Creating link at destination: %s", p.DestFile) + } os.Remove(p.DestFile) err = os.Link(dedupPath, p.DestFile) if err != nil { return fmt.Errorf("linking file: %w", err) } + if verbose { + log.Printf("Successfully linked file to destination: %s", p.DestFile) + } + return nil } diff --git a/pkg/processor/dedup.go b/pkg/processor/dedup.go index f47b16e..338963b 100644 --- a/pkg/processor/dedup.go +++ b/pkg/processor/dedup.go @@ -100,7 +100,7 @@ func (p *DedupProcessor) Process(verbose bool) error { go func() { defer wg.Done() for path := range jobs { - err := p.processFile(path) + err := p.processFile(path, verbose) if err != nil { if verbose { log.Printf("Error processing file %s: %v", path, err) @@ -116,6 +116,9 @@ func (p *DedupProcessor) Process(verbose bool) error { return err } if !info.IsDir() && path != p.Storage.Opts.Root { + if verbose { + log.Printf("Adding file to job queue: %s", path) + } jobs <- path } return nil @@ -129,16 +132,26 @@ func (p *DedupProcessor) Process(verbose bool) error { return nil } -func (p *DedupProcessor) processFile(path string) (err error) { +func (p *DedupProcessor) processFile(path string, verbose bool) (err error) { + if verbose { + log.Printf("Processing file: %s", path) + } + // Compute file hash var finalHash string if p.Storage.Opts.WithMetadata { + if verbose { + log.Println("Computing full hash with metadata") + } finalHash, err = p.HashGen.ComputeFullHash(path) if err != nil { return fmt.Errorf("computing full hash: %w", err) } } else { + if verbose { + log.Println("Computing content hash without metadata") + } finalHash, err = p.HashGen.ComputeFileHash(path) if err != nil { return fmt.Errorf("computing content hash: %w", err) @@ -148,6 +161,9 @@ func (p *DedupProcessor) processFile(path string) (err error) { // Check if the file is already being processed alreadyProcessing, waitChan := dedupStartProcessing(finalHash) if alreadyProcessing { + if verbose { + log.Printf("File %s is already being processed, waiting...", path) + } <-waitChan // Wait for the processing to finish } @@ -160,6 +176,9 @@ func (p *DedupProcessor) processFile(path string) (err error) { } if !exists { + if verbose { + log.Printf("File does not exist in storage, moving it: %s", dedupPath) + } // If the file does not exist in storage, move it there err = p.Storage.MoveFileToStorage(path, finalHash) if err != nil { @@ -167,6 +186,9 @@ func (p *DedupProcessor) processFile(path string) (err error) { return fmt.Errorf("moving file to storage: %w", err) } } else { + if verbose { + log.Printf("File already exists in storage: %s", dedupPath) + } // If the file already exists in storage, remove the source file err = os.Remove(path) if err != nil { @@ -181,6 +203,9 @@ func (p *DedupProcessor) processFile(path string) (err error) { p.mapMutex.Unlock() // Create a link at the original location + if verbose { + log.Printf("Creating link at original location: %s", path) + } if _, err := os.Lstat(path); os.IsNotExist(err) { err = os.Link(dedupPath, path) if err != nil { @@ -198,6 +223,9 @@ func (p *DedupProcessor) processFile(path string) (err error) { } destPath := filepath.Join(p.DestDir, relativePath) + if verbose { + log.Printf("Creating link at destination: %s", destPath) + } if _, err := os.Lstat(destPath); os.IsNotExist(err) { err = os.Link(dedupPath, destPath) if err != nil { @@ -208,5 +236,9 @@ func (p *DedupProcessor) processFile(path string) (err error) { } dedupFinishProcessing(finalHash) + + if verbose { + log.Printf("Finished processing file: %s", path) + } return nil }