From d3369bad7a3d1cbc5a80d4d75fd504643388e4d9 Mon Sep 17 00:00:00 2001 From: Thushan Fernando Date: Wed, 29 Nov 2023 18:13:51 +1100 Subject: [PATCH] Sync changes (#34) * Adds nerd stats * catch dupe collisions. * RWLock for smashing. * Fix empty files. * refactor guards. * Fix refactor renaming. * Fix struct alignment. --- internal/cli/cli.go | 1 + internal/report/nerdstats.go | 52 ++++++++++++++++++++++++++++++++++++ internal/report/smashfile.go | 30 +++++++++++++-------- internal/smash/app.go | 18 +++++++++---- internal/smash/flags.go | 1 + internal/smash/formatter.go | 15 ++++++----- pkg/slicer/slicer_test.go | 2 +- readme.md | 3 ++- 8 files changed, 97 insertions(+), 25 deletions(-) create mode 100644 internal/report/nerdstats.go diff --git a/internal/cli/cli.go b/internal/cli/cli.go index cc3329a..6af6189 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -50,6 +50,7 @@ func init() { flags.BoolVarP(&af.Silent, "silent", "q", false, "Run in silent mode") flags.BoolVarP(&af.Verbose, "verbose", "", false, "Run in verbose mode") flags.BoolVarP(&af.HideProgress, "no-progress", "", false, "Disable progress updates") + flags.BoolVarP(&af.ShowNerdStats, "nerd-stats", "", false, "Show nerd stats") flags.BoolVarP(&af.ShowVersion, "version", "v", false, "Show version information") flags.StringVarP(&af.OutputFile, "output-file", "o", "", "Export as JSON") } diff --git a/internal/report/nerdstats.go b/internal/report/nerdstats.go new file mode 100644 index 0000000..ed7b552 --- /dev/null +++ b/internal/report/nerdstats.go @@ -0,0 +1,52 @@ +package report + +import ( + "runtime" + + "github.com/dustin/go-humanize" + "github.com/thushan/smash/internal/theme" +) + +type NerdStats struct { + Allocations, + TotalAllocations, + Sys, + Mallocs, + Frees, + LiveObjects, + GcPauseTotalNs uint64 + + CompletedGcCycles uint32 + GoRoutines int +} + +func ReadNerdStats() NerdStats { + var rtm runtime.MemStats + runtime.ReadMemStats(&rtm) + + return NerdStats{ + GoRoutines: runtime.NumGoroutine(), + Allocations: rtm.Alloc, + TotalAllocations: rtm.TotalAlloc, + Sys: rtm.Sys, + Mallocs: rtm.Mallocs, + Frees: rtm.Frees, + LiveObjects: rtm.Mallocs - rtm.Frees, + GcPauseTotalNs: rtm.PauseTotalNs, + CompletedGcCycles: rtm.NumGC, + } +} +func PrintNerdStats(stats NerdStats, context string) { + theme.StyleContext.Println(context) + + bold := theme.StyleBold + theme.Println(bold.Sprint(" OSMemory:"), theme.ColourConfig(humanize.Bytes(stats.Sys), " (", stats.Sys, " Bytes)")) + theme.Println(bold.Sprint(" Allocations:"), theme.ColourConfig(humanize.Bytes(stats.Allocations), " (", stats.Allocations, " Bytes)")) + theme.Println(bold.Sprint(" Allocations (total):"), theme.ColourConfig(humanize.Bytes(stats.TotalAllocations), " (", stats.TotalAllocations, " Bytes)")) + theme.Println(bold.Sprint(" mallocs:"), theme.ColourConfig(stats.Mallocs)) + theme.Println(bold.Sprint(" frees:"), theme.ColourConfig(stats.Frees)) + theme.Println(bold.Sprint(" LiveObjects:"), theme.ColourConfig(stats.LiveObjects)) + theme.Println(bold.Sprint(" GC-Pauses (ns):"), theme.ColourConfig(stats.GcPauseTotalNs)) + theme.Println(bold.Sprint("GC-Cycles (completed):"), theme.ColourConfig(stats.CompletedGcCycles)) + theme.Println(bold.Sprint(" GoRoutines (active):"), theme.ColourConfig(stats.GoRoutines)) +} diff --git a/internal/report/smashfile.go b/internal/report/smashfile.go index 3af4138..b8bbec6 100644 --- a/internal/report/smashfile.go +++ b/internal/report/smashfile.go @@ -2,6 +2,7 @@ package report import ( "encoding/hex" + "sync" "github.com/alphadose/haxmap" "github.com/dustin/go-humanize" @@ -17,9 +18,17 @@ type SmashFile struct { FullHash bool EmptyFile bool } +type DuplicateFiles struct { + Files []SmashFile + sync.RWMutex +} +type EmptyFiles struct { + Files []SmashFile + sync.RWMutex +} -func SummariseSmashedFile(stats slicer.SlicerStats, filename string, ms int64, duplicates *haxmap.Map[string, []SmashFile], emptyFiles *[]SmashFile) { - sf := SmashFile{ +func SummariseSmashedFile(stats slicer.SlicerStats, filename string, ms int64, duplicates *haxmap.Map[string, *DuplicateFiles], empty *EmptyFiles) { + file := SmashFile{ Hash: hex.EncodeToString(stats.Hash), Filename: filename, FileSize: stats.FileSize, @@ -28,16 +37,15 @@ func SummariseSmashedFile(stats slicer.SlicerStats, filename string, ms int64, d FileSizeF: humanize.Bytes(stats.FileSize), ElapsedTime: ms, } - if sf.EmptyFile { - *emptyFiles = append(*emptyFiles, sf) + if file.EmptyFile { + empty.Lock() + empty.Files = append(empty.Files, file) + empty.Unlock() } else { - hash := sf.Hash - if v, existing := duplicates.Get(hash); existing { - v = append(v, sf) - duplicates.Set(hash, v) - } else { - duplicates.Set(hash, []SmashFile{sf}) - } + dupes, _ := duplicates.GetOrSet(file.Hash, &DuplicateFiles{}) + dupes.Lock() + dupes.Files = append(dupes.Files, file) + dupes.Unlock() } } diff --git a/internal/smash/app.go b/internal/smash/app.go index 6f0c2ef..eb8d867 100644 --- a/internal/smash/app.go +++ b/internal/smash/app.go @@ -31,9 +31,9 @@ type App struct { Locations []string } type AppSession struct { - Dupes *haxmap.Map[string, []report.SmashFile] + Dupes *haxmap.Map[string, *report.DuplicateFiles] Fails *haxmap.Map[string, error] - Empty *[]report.SmashFile + Empty *report.EmptyFiles StartTime int64 EndTime int64 } @@ -57,9 +57,9 @@ func (app *App) Run() error { } app.Session = &AppSession{ - Dupes: haxmap.New[string, []report.SmashFile](), + Dupes: haxmap.New[string, *report.DuplicateFiles](), Fails: haxmap.New[string, error](), - Empty: &[]report.SmashFile{}, + Empty: &report.EmptyFiles{}, StartTime: time.Now().UnixNano(), EndTime: -1, } @@ -89,7 +89,7 @@ func (app *App) Exec() error { if err := app.validateArgs(); err != nil { return err } - + startStats := report.ReadNerdStats() session := app.Session wk := app.Runtime.IndexerConfig @@ -172,9 +172,17 @@ func (app *App) Exec() error { pap.Stop() + endStats := report.ReadNerdStats() + app.PrintRunAnalysis(app.Flags.IgnoreEmpty) report.PrintRunSummary(*app.Summary, app.Flags.IgnoreEmpty) + if app.Flags.ShowNerdStats { + theme.StyleHeading.Println("---| Nerd Stats") + report.PrintNerdStats(startStats, "Commenced analysis") + report.PrintNerdStats(endStats, "Completed analysis") + } + return nil } diff --git a/internal/smash/flags.go b/internal/smash/flags.go index 111030d..b6467c6 100644 --- a/internal/smash/flags.go +++ b/internal/smash/flags.go @@ -17,6 +17,7 @@ type Flags struct { IgnoreHidden bool `yaml:"ignore-hidden"` IgnoreSystem bool `yaml:"ignore-system"` ShowVersion bool `yaml:"version"` + ShowNerdStats bool `yaml:"nerd-stats"` ShowDuplicates bool `yaml:"show-duplicates"` Silent bool `yaml:"silent"` HideTopList bool `yaml:"no-top-list"` diff --git a/internal/smash/formatter.go b/internal/smash/formatter.go index 425ac2e..370da3b 100644 --- a/internal/smash/formatter.go +++ b/internal/smash/formatter.go @@ -24,12 +24,12 @@ func (app *App) printVerbose(message ...any) { func (app *App) PrintRunAnalysis(ignoreEmptyFiles bool) { duplicates := app.Session.Dupes - emptyFiles := *app.Session.Empty + emptyFiles := app.Session.Empty.Files topFiles := app.Summary.TopFiles totalDuplicates := app.Summary.DuplicateFiles - theme.StyleHeading.Println("---| Duplicates (", totalDuplicates, ")") + theme.StyleHeading.Println("---| Duplicate Files (", totalDuplicates, ")") if duplicates.Len() == 0 || len(topFiles) == 0 { theme.Println(theme.ColourSuccess("No duplicates found :-)")) @@ -39,15 +39,15 @@ func (app *App) PrintRunAnalysis(ignoreEmptyFiles bool) { theme.StyleSubHeading.Println("---[ Top ", app.Flags.ShowTop, " Duplicates ]---") for _, tf := range topFiles { if files, ok := duplicates.Get(tf.Key); ok { - displayFiles(files) + displayFiles(files.Files) } } } if app.Flags.ShowDuplicates { theme.StyleSubHeading.Println("---[ All Duplicates ]---") - duplicates.ForEach(func(hash string, files []report.SmashFile) bool { - displayFiles(files) + duplicates.ForEach(func(hash string, files *report.DuplicateFiles) bool { + displayFiles(files.Files) return true }) } @@ -87,7 +87,7 @@ func printSmashHits(files []report.SmashFile) { func (app *App) generateRunSummary(totalFiles int64) { session := *app.Session duplicates := session.Dupes - emptyFiles := *session.Empty + emptyFiles := session.Empty.Files topFiles := analysis.NewSummary(app.Flags.ShowTop) @@ -97,7 +97,8 @@ func (app *App) generateRunSummary(totalFiles int64) { totalFailFileCount := int64(session.Fails.Len()) totalEmptyFileCount := int64(len(emptyFiles)) - duplicates.ForEach(func(hash string, files []report.SmashFile) bool { + duplicates.ForEach(func(hash string, df *report.DuplicateFiles) bool { + files := df.Files duplicateFiles := len(files) - 1 if duplicateFiles == 0 { // prune unique files diff --git a/pkg/slicer/slicer_test.go b/pkg/slicer/slicer_test.go index bc1cf63..c5492eb 100644 --- a/pkg/slicer/slicer_test.go +++ b/pkg/slicer/slicer_test.go @@ -74,7 +74,7 @@ func TestSlice_New_With0KbBlob(t *testing.T) { } if stats.EmptyFile != true { - t.Errorf("expected Empty to be %v, got %v", true, stats.EmptyFile) + t.Errorf("expected Files to be %v, got %v", true, stats.EmptyFile) } if stats.FileSize != 0 { diff --git a/readme.md b/readme.md index 50e337b..c9e6b45 100644 --- a/readme.md +++ b/readme.md @@ -61,10 +61,11 @@ Flags: --ignore-system Ignore system files & folders Eg. '$MFT', '.Trash' (default true) -p, --max-threads int Maximum threads to utilise (default 16) -w, --max-workers int Maximum workers to utilise when smashing (default 8) + --nerd-stats Show nerd stats --no-progress Disable progress updates --no-top-list Hides top x duplicates list -o, --output-file string Export as JSON - --progress-update int Update progress every x seconds (default 5) + --progress-update int Update progress every x seconds (default 5) --show-duplicates Show full list of duplicates --show-top int Show the top x duplicates (default 10) -q, --silent Run in silent mode