Upgrade binary test fixtures management

.github/scripts/check_binary_fixture_size.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# current limit for fixture size
+size=1000
+
+if [ $# -eq 0 ]; then
+  echo "Usage: $0 <directory>"
+  exit 1
+fi
+
+directory="$1"
+
+# Remove trailing slash using parameter expansion
+directory="${directory%/}"
+
+if [ ! -d "$directory" ]; then
+  echo "Directory not found: $directory"
+  exit 1
+fi
+
+found_large_files=0
+while IFS= read -r -d '' file; do
+  if [ $(wc -c < "$file") -gt $size ]; then
+    echo "File $file is greater than ${size} bytes."
+    found_large_files=1
+  fi
+done < <(find "$directory" -type f -print0)
+
+if [ "$found_large_files" -eq 1 ]; then
+  echo "Script failed: Some files are greater than ${size} bytes."
+  exit 1
+else
+  echo "All files in $directory and its subdirectories are ${size} bytes or smaller. Check passed."
+  exit 0
+fi
+

.github/workflows/validations.yaml

@@ -57,7 +57,7 @@ jobs:
       - name: Restore binary cataloger test-fixture cache
         uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 #v3.3.2
         with:
-          path: syft/pkg/cataloger/binary/test-fixtures/classifiers/dynamic
+          path: syft/pkg/cataloger/binary/test-fixtures/classifiers/bin
           key: ${{ runner.os }}-unit-binary-cataloger-cache-${{ hashFiles( 'syft/pkg/cataloger/binary/test-fixtures/cache.fingerprint' ) }}
 
       - name: Restore Kernel test-fixture cache

Taskfile.yaml

@@ -54,6 +54,7 @@ tasks:
       - task: check-licenses
       - task: lint
       - task: check-json-schema-drift
+      - task: check-binary-fixture-size
 
   test:
     desc: Run all levels of test
@@ -169,6 +170,11 @@ tasks:
     cmds:
       - .github/scripts/json-schema-drift-check.sh
 
+  check-binary-fixture-size:
+    desc: Ensure that the binary test fixtures are not too large
+    cmds:
+      - .github/scripts/check_binary_fixture_size.sh syft/pkg/cataloger/binary/test-fixtures/classifiers/snippets
+
 
   ## Testing tasks #################################
 

go.mod

@@ -76,6 +76,12 @@ require (
 	modernc.org/sqlite v1.28.0
 )
 
+require (
+	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
+	github.com/charmbracelet/bubbles v0.16.1
+	github.com/jedib0t/go-pretty/v6 v6.4.9
+)
+
 require (
 	dario.cat/mergo v1.0.0 // indirect
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
@@ -91,9 +97,9 @@ require (
 	github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 // indirect
 	github.com/andybalholm/brotli v1.0.4 // indirect
 	github.com/aquasecurity/go-version v0.0.0-20210121072130-637058cfe492 // indirect
+	github.com/atotto/clipboard v0.1.4 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/becheran/wildmatch-go v1.0.0 // indirect
-	github.com/charmbracelet/bubbles v0.16.1 // indirect
 	github.com/charmbracelet/harmonica v0.2.0 // indirect
 	github.com/cloudflare/circl v1.3.3 // indirect
 	github.com/containerd/cgroups v1.1.0 // indirect
@@ -180,6 +186,7 @@ require (
 	github.com/rogpeppe/go-internal v1.11.0 // indirect
 	github.com/sagikazarmark/locafero v0.3.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sahilm/fuzzy v0.1.0 // indirect
 	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/skeema/knownhosts v1.2.1 // indirect

go.sum

@@ -127,6 +127,8 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
+github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
 github.com/becheran/wildmatch-go v1.0.0 h1:mE3dGGkTmpKtT4Z+88t8RStG40yN9T+kFEGj2PZFSzA=
@@ -464,6 +466,8 @@ github.com/invopop/jsonschema v0.7.0 h1:2vgQcBz1n256N+FpX3Jq7Y17AjYt46Ig3zIWyy77
 github.com/invopop/jsonschema v0.7.0/go.mod h1:O9uiLokuu0+MGFlyiaqtWxwqJm41/+8Nj0lD7A36YH0=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
+github.com/jedib0t/go-pretty/v6 v6.4.9 h1:vZ6bjGg2eBSrJn365qlxGcaWu09Id+LHtrfDWlB2Usc=
+github.com/jedib0t/go-pretty/v6 v6.4.9/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs=
 github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
 github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -503,6 +507,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs=
 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
@@ -533,6 +539,7 @@ github.com/mattn/go-localereader v0.0.2-0.20220822084749-2491eb6c1c75 h1:P8UmIzZ
 github.com/mattn/go-localereader v0.0.2-0.20220822084749-2491eb6c1c75/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
+github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
@@ -629,6 +636,7 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.6.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18=
 github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA=
 github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo=
 github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
@@ -673,6 +681,8 @@ github.com/sagikazarmark/locafero v0.3.0 h1:zT7VEGWC2DTflmccN/5T1etyKvxSxpHsjb9c
 github.com/sagikazarmark/locafero v0.3.0/go.mod h1:w+v7UsPNFwzF1cHuOajOOzoq4U7v/ig1mpRjqV+Bu1U=
 github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
 github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
+github.com/sahilm/fuzzy v0.1.0 h1:FzWGaw2Opqyu+794ZQ9SYifWv2EIXpwP4q8dY1kDAwI=
+github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y=
 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
 github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo=
@@ -738,6 +748,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=

syft/pkg/cataloger/binary/README.md

@@ -0,0 +1,122 @@
+# Adding tests for the Binary cataloger
+
+> [!TIP]
+> **TL;DR** to add a test for a new classifier:
+>  1. head to the correct directory: `cd test-fixtures`
+>  2. add a new entry to `config.yaml` to track where to get the binary from (verify the entry with `make list`)
+>  3. run `make download` to get the binary
+>  4. run `make add-snippet` and follow the prompts (use `/` to search)
+>  5. add a new test case to `Test_Cataloger_PositiveCases` in `../cataloger_test.go`
+
+
+To test the binary cataloger we run it against a set of files ("test fixtures"). There are two kinds of test fixtures:
+
+- **Full binaries**: files downloaded and cached at test runtime
+- **Snippets**: ~100 byte files checked into the repo
+
+The upside with snippets is that they live with the test, don't necessarily require network access or hosting concerns, and are easy to add. The downside is that they are not the entire real binary so modifications may require recreating the snippet entirely.
+
+The upside with full binaries is that they are the "Real McCoy" and allows the business logic to change without needing to update the fixture. The downside is that they require network access and take up a lot of space. For instance, downloading all binaries for testing today requires downloading ~15GB of container images and ends up being ~500MB of disk space.
+
+You can find the test fixtures at the following locations:
+```
+syft/pkg/cataloger/binary/test-fixtures/
+└── classifiers/
+    ├── bin/        # full binaries
+    ├── ...
+    └── snippets/   # snippets
+```
+
+And use tooling to list and manage the fixtures:
+
+- `make list` - list all fixtures
+- `make download` - download binaries that are not covered by a snippet
+- `make download-all` - download all binaries
+- `go run ./manager add-snippet` - add a new snippet based off of a configured binary
+- `capture-snippet.sh` - add a new snippet based off of a binary on your local machine (not recommended, but allowed)
+
+There is a `config.yaml` that tracks all binaries that the tests can use. This makes it possible to download it at any time from a hosted source. Today the only method allowed is to download a container image and extract files out.
+
+Here is an example entry in `config.yaml` for a binary reference from a container image:
+
+```yaml
+from-images:
+
+  # note the group-name is assumed from the single binary name extracted: "redis-server"
+  - version: 7.2.3
+    images:
+      # note we're pulling the same binary from multiple images (representing different architectures)
+      - ref: redis:7.2.3@sha256:d4c84914b872521e215f77d8845914c2268a96b0e35bacd5691e1f5e1f88b500
+        platform: linux/amd64
+      - ref: redis:7.2.3@sha256:a0a0c38b31011b813cddf78d997f8ccba13019c27efd386984b0cfc1e4b618ff
+        platform: linux/arm64
+    # the paths to extract from the binary...
+    paths:
+      - /usr/local/bin/redis-server
+
+  # since there are multiple binaries in the image, we need to specify the group-name
+  - name: ruby-bullseye-shared-libs
+    version: 2.7.7
+    images:
+      - ref: ruby:2.7.7-bullseye@sha256:055191740a063f33fef1f09423e5ed8f91143aae62a3772a90910118464c5120
+        platform: linux/amd64
+    paths:
+      - /usr/local/bin/ruby
+      - /usr/local/lib/libruby.so.2.7.7
+      - /usr/local/lib/libruby.so.2.7
+```
+
+
+> [!NOTE]  
+> You will need a system with `go`, `bash`, `strings`, and `xxd` installed to capture test snippets.
+
+
+## Testing
+
+The test cases have been setup to allow testing against full binaries or a mix of both (default).
+To force running only against full binaries run with:
+
+```bash
+go test -must-use-full-binaries ./syft/pkg/cataloger/binary/test-fixtures/...
+```
+
+## Adding a new test fixture
+
+### Adding a snippet (recommended)
+
+Even if you are adding a snippet, it is best practice to:
+
+- create that snippet from a full binary (not craft a snippet by hand)
+- track where the binary is from and how to download it in `config.yaml`
+
+1. Follow the steps above to [add a full binary](#adding-a-full-binary)
+
+2. Run `go run ./manager add-snippet` and follow the prompts to create a new snippet
+   - you should see your binary in the list of binaries to choose from. If not, check step 2
+   - if the search results in no matching snippets, you can specify your own search with `--search-for <grep-pattern>`
+   - you should see a new snippet file created in `snippets/`
+
+3. Write a test that references your new snippet by `<name>/<version>/<architecture>`
+   - `<name>` is the name of the binary (e.g. `curl`) or the name in `config.yaml` if specified
+   - note that your test does not know about if it's running against a snippet or a full binary
+
+
+### Adding a full binary
+
+1. Add a new entry to `config.yaml` with the following fields
+  - if you are adding a single binary, the `name` field does not need to be specified
+  - the `name` field is useful for distinguishing a quality about the binary (e.g. `java` vs `java-jre-ibm`)
+
+2. Run `make download` and ensure your new binary is downloaded
+
+
+### Adding a custom snippet
+
+If you need to add a snippet that is not based off of a full binary, you can use the `capture-snippet.sh` script.
+
+```bash
+./capture-snippet.sh <binary-path> <version> [--search-for <pattern>] [--length <length>] [--prefix-length <prefix_length>] [--group <name>]
+```
+
+
+This is **not** recommended because it is not reproducible and does not allow for the test to be run against a full binary.