From 607d5d5a4b3d03e084d0e6c164161650831de2cb Mon Sep 17 00:00:00 2001 From: Thushan Fernando Date: Tue, 6 Feb 2024 14:29:28 +1100 Subject: [PATCH 1/3] introduces slicing options to the CLI. --- internal/cli/cli.go | 9 ++++-- internal/smash/app.go | 2 +- internal/smash/configuration.go | 8 ++---- internal/smash/flags.go | 22 ++++++++++++++- internal/smash/flags_test.go | 50 +++++++++++++++++++++++++++++++++ internal/smash/version.go | 2 +- pkg/slicer/slicer.go | 10 ++----- readme.md | 3 ++ 8 files changed, 88 insertions(+), 18 deletions(-) diff --git a/internal/cli/cli.go b/internal/cli/cli.go index cc1eec2..81f4351 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -6,13 +6,13 @@ import ( "os" "runtime" - "github.com/thushan/smash/internal/theme" - "github.com/thushan/smash/internal/algorithms" + "github.com/thushan/smash/internal/smash" + "github.com/thushan/smash/internal/theme" + "github.com/thushan/smash/pkg/slicer" "github.com/spf13/cobra" "github.com/thediveo/enumflag/v2" - "github.com/thushan/smash/internal/smash" ) var ( @@ -60,6 +60,9 @@ func init() { flags.BoolVarP(&af.ShowNerdStats, "nerd-stats", "", false, "Show nerd stats") flags.BoolVarP(&af.ShowVersion, "version", "v", false, "Show version information") flags.StringVarP(&af.OutputFile, "output-file", "o", "", "Export analysis as JSON (generated automatically like ./report-*.json)") + flags.IntVarP(&af.Slices, "slices", "", slicer.DefaultSlices, "Number of Slices to use") + flags.Int64VarP(&af.SliceSize, "slice-size", "", slicer.DefaultSliceSize, "Size of a Slice (in bytes)") + flags.Int64VarP(&af.SliceThreshold, "slice-threshold", "", slicer.DefaultThreshold, "Threshold to use for slicing (in bytes) - if file is smaller than this, it won't be sliced") } func Main() { diff --git a/internal/smash/app.go b/internal/smash/app.go index 776200c..0168752 100644 --- a/internal/smash/app.go +++ b/internal/smash/app.go @@ -73,7 +73,7 @@ func (app *App) Run() error { EndTime: -1, } - sl := slicer.New(algorithms.Algorithm(af.Algorithm)) + sl := slicer.NewConfigured(algorithms.Algorithm(af.Algorithm), af.Slices, uint64(af.SliceSize), uint64(af.SliceThreshold)) wk := indexer.NewConfigured(af.ExcludeDir, af.ExcludeFile, af.IgnoreHidden, af.IgnoreSystem) slo := slicer.Options{ DisableSlicing: af.DisableSlicing, diff --git a/internal/smash/configuration.go b/internal/smash/configuration.go index f13598b..a4a9d67 100644 --- a/internal/smash/configuration.go +++ b/internal/smash/configuration.go @@ -5,8 +5,6 @@ import ( "strings" "github.com/dustin/go-humanize" - "github.com/thushan/smash/pkg/slicer" - "github.com/thushan/smash/internal/theme" "github.com/thushan/smash/internal/algorithms" @@ -20,9 +18,9 @@ func (app *App) printConfiguration() { theme.StyleHeading.Println("---| Configuration") if app.Flags.Verbose { - slices := theme.ColourConfig(slicer.DefaultSlices) - size := theme.ColourConfig(humanize.Bytes(slicer.DefaultSliceSize)) - threshold := theme.ColourConfig(humanize.Bytes(slicer.DefaultThreshold)) + slices := theme.ColourConfig(f.Slices) + size := theme.ColourConfig(humanize.Bytes(uint64(f.SliceSize))) + threshold := theme.ColourConfig(humanize.Bytes(uint64(f.SliceThreshold))) config = "(Slices: " + slices + " | Size: " + size + " | Threshold: " + threshold + ")" diff --git a/internal/smash/flags.go b/internal/smash/flags.go index 7bf1ce5..0f1c2a1 100644 --- a/internal/smash/flags.go +++ b/internal/smash/flags.go @@ -1,6 +1,11 @@ package smash -import "errors" +import ( + "errors" + "fmt" + + "github.com/thushan/smash/pkg/slicer" +) type Flags struct { OutputFile string `yaml:"output"` @@ -9,6 +14,9 @@ type Flags struct { ExcludeFile []string `yaml:"exclude-file"` MinSize int64 `yaml:"min-size"` MaxSize int64 `yaml:"max-size"` + SliceThreshold int64 `yaml:"slice-threshold"` + SliceSize int64 `yaml:"slice-size"` + Slices int `yaml:"slices"` Algorithm int `yaml:"algorithm"` MaxThreads int `yaml:"max-threads"` MaxWorkers int `yaml:"max-workers"` @@ -61,6 +69,18 @@ func (app *App) validateArgs() error { if f.ProgressUpdate < 1 { return errors.New("updateseconds cannot be less than 1") } + if f.Slices < slicer.DefaultSlices { + return fmt.Errorf("defaultSlices cannot be less than %q", slicer.DefaultSlices) + } + if f.Slices > slicer.MaxSlices { + return fmt.Errorf("defaultSlices cannot be greater than %q", slicer.MaxSlices) + } + if f.SliceSize < slicer.DefaultSliceSize { + return fmt.Errorf("slicesize cannot be less than %q bytes ", slicer.DefaultSliceSize) + } + if f.SliceThreshold < slicer.DefaultThreshold { + return fmt.Errorf("slicethreshold cannot be less than %q bytes ", slicer.DefaultThreshold) + } return nil } diff --git a/internal/smash/flags_test.go b/internal/smash/flags_test.go index 12142ae..c101163 100644 --- a/internal/smash/flags_test.go +++ b/internal/smash/flags_test.go @@ -2,6 +2,8 @@ package smash import ( "testing" + + "github.com/thushan/smash/pkg/slicer" ) func TestApp_ValidateArgs(t *testing.T) { @@ -62,6 +64,9 @@ func TestApp_ValidateArgs(t *testing.T) { MaxWorkers: 5, ShowTop: 10, ProgressUpdate: 2, + SliceSize: slicer.DefaultSliceSize, + SliceThreshold: slicer.DefaultThreshold, + Slices: slicer.DefaultSlices, }, wantErr: false, }, @@ -74,6 +79,9 @@ func TestApp_ValidateArgs(t *testing.T) { MaxWorkers: 5, ShowTop: 10, ProgressUpdate: 2, + SliceSize: slicer.DefaultSliceSize, + SliceThreshold: slicer.DefaultThreshold, + Slices: slicer.DefaultSlices, }, wantErr: false, }, @@ -86,6 +94,9 @@ func TestApp_ValidateArgs(t *testing.T) { MaxWorkers: 5, ShowTop: 10, ProgressUpdate: 2, + SliceSize: slicer.DefaultSliceSize, + SliceThreshold: slicer.DefaultThreshold, + Slices: slicer.DefaultSlices, }, wantErr: false, }, @@ -99,6 +110,9 @@ func TestApp_ValidateArgs(t *testing.T) { MaxWorkers: 5, ShowTop: 10, ProgressUpdate: 2, + SliceSize: slicer.DefaultSliceSize, + SliceThreshold: slicer.DefaultThreshold, + Slices: slicer.DefaultSlices, }, wantErr: false, }, @@ -132,6 +146,42 @@ func TestApp_ValidateArgs(t *testing.T) { }, wantErr: true, }, + { + name: "Should fail when Slices is below DefaultSlices", + flags: &Flags{ + Slices: 0, + SliceSize: 1000, + SliceThreshold: 500, + }, + wantErr: true, + }, + { + name: "Should fail when Slices is above MaxSlices", + flags: &Flags{ + Slices: 100, + SliceSize: 1000, + SliceThreshold: 500, + }, + wantErr: true, + }, + { + name: "Should fail when SliceSize is below DefaultSliceSize", + flags: &Flags{ + Slices: 10, + SliceSize: 0, + SliceThreshold: 500, + }, + wantErr: true, + }, + { + name: "Should fail when SliceThreshold is below DefaultThreshold", + flags: &Flags{ + Slices: 10, + SliceSize: 1000, + SliceThreshold: 0, + }, + wantErr: true, + }, } for _, tt := range tests { diff --git a/internal/smash/version.go b/internal/smash/version.go index cba143c..acb3ce5 100644 --- a/internal/smash/version.go +++ b/internal/smash/version.go @@ -8,7 +8,7 @@ import ( ) var ( - Version = "v0.7.0" + Version = "v0.9.0" Commit = "none" Date = "unknown" Time = "nowish" diff --git a/pkg/slicer/slicer.go b/pkg/slicer/slicer.go index f655416..a327427 100644 --- a/pkg/slicer/slicer.go +++ b/pkg/slicer/slicer.go @@ -43,6 +43,7 @@ type Options struct { DisableAutoText bool } +const MaxSlices = 128 const DefaultSlices = 4 const DefaultSliceSize = 8 * 1024 const DefaultThreshold = 100 * 1024 @@ -53,11 +54,11 @@ const DefaultMaxSize = 0 func New(algorithm algorithms.Algorithm) Slicer { return NewConfigured(algorithm, DefaultSlices, DefaultSliceSize, DefaultThreshold) } -func NewConfigured(algorithm algorithms.Algorithm, slices int, size, maxSlice uint64) Slicer { +func NewConfigured(algorithm algorithms.Algorithm, slices int, size, threshold uint64) Slicer { return Slicer{ slices: slices, sliceSize: size, - threshold: maxSlice, + threshold: threshold, algorithm: algorithm, defaultBytes: []byte{}, } @@ -72,11 +73,6 @@ func (slicer *Slicer) SliceFS(fs fs.FS, name string, options *Options) (SlicerSt return } _ = fs.Close() - /* - if ferr := fs.Close(); ferr != nil { - theme.Error.Println(theme.ColourFilename(name), ferr) - } - */ }(f) if err != nil { diff --git a/readme.md b/readme.md index bb03ae1..5aa2896 100644 --- a/readme.md +++ b/readme.md @@ -89,6 +89,9 @@ Flags: --show-duplicates Show full list of duplicates --show-top int Show the top x duplicates (default 10) -q, --silent Run in silent mode + --slice-size int Size of a Slice (in bytes) (default 8192) + --slice-threshold int Threshold to use for slicing (in bytes) - if file is smaller than this, it won't be sliced (default 102400) + --slices int Number of Slices to use (default 4) --verbose Run in verbose mode -v, --version Show version information ``` From 665833c29a6a1d45985f35acce4b43828f756581 Mon Sep 17 00:00:00 2001 From: Thushan Fernando Date: Tue, 6 Feb 2024 15:42:12 +1100 Subject: [PATCH 2/3] how is this not already in the repo :( realised when switching back to windows. --- makefile | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 makefile diff --git a/makefile b/makefile new file mode 100644 index 0000000..de023de --- /dev/null +++ b/makefile @@ -0,0 +1,40 @@ +##################### +# smash make +# make smash +##################### +BINARY_NAME=smash +VERSION=$(shell git describe --tags --always --dirty) +GCOMMIT=$(shell git rev-parse --short HEAD) +TODAY=$(shell date --iso-8601) + +.PHONY: all +all: lint test build + +.PHONY: lint +lint: + golangci-lint run -c .golangci.yml ./... + +.PHONY: test +test: + go test -v ./... + +.PHONY: build +build: + go build -ldflags " -X github.com/thushan/smash/internal/smash.Date=$(TODAY) \ + -X github.com/thushan/smash/internal/smash.User=make \ + -X github.com/thushan/smash/internal/smash.Version=$(VERSION) \ + -X github.com/thushan/smash/internal/smash.Commit=$(GCOMMIT)" \ + -o dist/$(BINARY_NAME) . + +.PHONY: release +release: + MSYS_NO_PATHCONV=1 docker run -ti -v "$(PWD):/app" -w "/app" goreleaser/goreleaser:latest release --snapshot --clean + +.PHONY: clean +clean: + go clean + rm -rf dist/ + +.PHONY: clean-reports +clean-reports: + rm -rf report-*.json From 23c84dc8409c7f87b06ab067cd0386842c5dcb89 Mon Sep 17 00:00:00 2001 From: Thushan Fernando Date: Tue, 6 Feb 2024 16:23:00 +1100 Subject: [PATCH 3/3] missing args. --- makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/makefile b/makefile index de023de..c2cf1e1 100644 --- a/makefile +++ b/makefile @@ -23,8 +23,10 @@ build: go build -ldflags " -X github.com/thushan/smash/internal/smash.Date=$(TODAY) \ -X github.com/thushan/smash/internal/smash.User=make \ -X github.com/thushan/smash/internal/smash.Version=$(VERSION) \ - -X github.com/thushan/smash/internal/smash.Commit=$(GCOMMIT)" \ - -o dist/$(BINARY_NAME) . + -X github.com/thushan/smash/internal/smash.Commit=$(GCOMMIT) \ + -s -w" \ + -trimpath \ + -o dist/$(BINARY_NAME) . .PHONY: release release: