diff --git a/.binny.yaml b/.binny.yaml index a24ad308f0c..3d7fab25b92 100644 --- a/.binny.yaml +++ b/.binny.yaml @@ -90,7 +90,7 @@ tools: - name: gh version: - want: v2.39.2 + want: v2.40.1 method: github-release with: repo: cli/cli diff --git a/.github/workflows/benchmark-testing.yaml b/.github/workflows/benchmark-testing.yaml index a87fa1f9d18..5119fcf80a1 100644 --- a/.github/workflows/benchmark-testing.yaml +++ b/.github/workflows/benchmark-testing.yaml @@ -39,7 +39,7 @@ jobs: OUTPUT="${OUTPUT//$'\r'/'%0D'}" # URL encode all '\r' characters echo "result=$OUTPUT" >> $GITHUB_OUTPUT - - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 with: name: benchmark-test-results path: test/results/**/* diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 75cd2f5bace..023fd3d499d 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,13 +39,13 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1 - name: Install Go - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe #v4.1.0 + uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 #v5.0.0 with: go-version-file: go.mod # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@407ffafae6a767df3e0230c3df91b6443ae8df75 #v2.22.8 + uses: github/codeql-action/init@b374143c1149a9115d881581d29b8390bbcbb59c #v3.22.11 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -56,7 +56,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@407ffafae6a767df3e0230c3df91b6443ae8df75 #v2.22.8 + uses: github/codeql-action/autobuild@b374143c1149a9115d881581d29b8390bbcbb59c #v3.22.11 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -70,4 +70,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@407ffafae6a767df3e0230c3df91b6443ae8df75 #v2.22.8 + uses: github/codeql-action/analyze@b374143c1149a9115d881581d29b8390bbcbb59c #v3.22.11 diff --git a/.github/workflows/release-homebrew.yaml b/.github/workflows/release-homebrew.yaml index 0be645a635e..8ac7c848d81 100644 --- a/.github/workflows/release-homebrew.yaml +++ b/.github/workflows/release-homebrew.yaml @@ -30,7 +30,7 @@ jobs: echo -n "commit=$(git rev-parse HEAD)" | tee -a $GITHUB_OUTPUT - name: Update Homebrew formula - uses: dawidd6/action-homebrew-bump-formula@d3667e5ae14df19579e4414897498e3e88f2f458 # v3.10.0 + uses: dawidd6/action-homebrew-bump-formula@75ed025ff3ad1d617862838b342b06d613a0ddf3 # v3.10.1 with: token: ${{ secrets.HOMEBREW_TOKEN }} org: anchore diff --git a/.github/workflows/update-stereoscope-release.yml b/.github/workflows/update-stereoscope-release.yml index ed5d00a631c..20a4416f4fd 100644 --- a/.github/workflows/update-stereoscope-release.yml +++ b/.github/workflows/update-stereoscope-release.yml @@ -19,7 +19,7 @@ jobs: steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1 - - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe #v4.1.0 + - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 #v5.0.0 with: go-version: ${{ env.GO_VERSION }} stable: ${{ env.GO_STABLE_VERSION }} diff --git a/README.md b/README.md index 1919c357dab..d75f248bee4 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ syft --scope all-layers ### Supported sources -Syft can generate a SBOM from a variety of sources: +Syft can generate an SBOM from a variety of sources: ``` # catalog a container image archive (from the result of `docker image save ...`, `podman save ...`, or `skopeo copy` commands) diff --git a/go.mod b/go.mod index 7a57f097efc..68e28f4455a 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/anchore/syft go 1.21.0 require ( - github.com/CycloneDX/cyclonedx-go v0.7.2 + github.com/CycloneDX/cyclonedx-go v0.8.0 github.com/Masterminds/semver v1.5.0 github.com/Masterminds/sprig/v3 v3.2.3 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d @@ -16,12 +16,12 @@ require ( github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b github.com/anchore/packageurl-go v0.1.1-0.20230104203445-02e0a6721501 - github.com/anchore/stereoscope v0.0.0-20231117203853-3610f4ef3e83 + github.com/anchore/stereoscope v0.0.0-20231215220732-4b999b76ca89 // we are hinting brotli to latest due to warning when installing archiver v3: // go: warning: github.com/andybalholm/brotli@v1.0.1: retracted by module author: occasional panics and data corruption github.com/aquasecurity/go-pep440-version v0.0.0-20210121094942-22b2f8951d46 github.com/bmatcuk/doublestar/v4 v4.6.1 - github.com/charmbracelet/bubbletea v0.24.2 + github.com/charmbracelet/bubbletea v0.25.0 github.com/charmbracelet/lipgloss v0.9.1 github.com/dave/jennifer v1.7.0 github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da @@ -32,12 +32,12 @@ require ( github.com/github/go-spdx/v2 v2.2.0 github.com/gkampitakis/go-snaps v0.4.12 github.com/go-git/go-billy/v5 v5.5.0 - github.com/go-git/go-git/v5 v5.10.1 + github.com/go-git/go-git/v5 v5.11.0 github.com/go-test/deep v1.1.0 github.com/google/go-cmp v0.6.0 github.com/google/go-containerregistry v0.17.0 github.com/google/licensecheck v0.3.1 - github.com/google/uuid v1.4.0 + github.com/google/uuid v1.5.0 github.com/gookit/color v1.5.4 github.com/hashicorp/go-multierror v1.1.1 github.com/iancoleman/strcase v0.3.0 @@ -53,7 +53,7 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/opencontainers/go-digest v1.0.0 github.com/pelletier/go-toml v1.9.5 - github.com/saferwall/pe v1.4.7 + github.com/saferwall/pe v1.4.8 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d github.com/sanity-io/litter v1.5.5 github.com/sassoftware/go-rpmutils v0.2.0 @@ -73,7 +73,7 @@ require ( golang.org/x/mod v0.14.0 golang.org/x/net v0.19.0 gopkg.in/yaml.v3 v3.0.1 - modernc.org/sqlite v1.27.0 + modernc.org/sqlite v1.28.0 ) require ( diff --git a/go.sum b/go.sum index 4bb8ed4adb0..a0a5878abfe 100644 --- a/go.sum +++ b/go.sum @@ -58,8 +58,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/CycloneDX/cyclonedx-go v0.7.2 h1:kKQ0t1dPOlugSIYVOMiMtFqeXI2wp/f5DBIdfux8gnQ= -github.com/CycloneDX/cyclonedx-go v0.7.2/go.mod h1:K2bA+324+Og0X84fA8HhN2X066K7Bxz4rpMQ4ZhjtSk= +github.com/CycloneDX/cyclonedx-go v0.8.0 h1:FyWVj6x6hoJrui5uRQdYZcSievw3Z32Z88uYzG/0D6M= +github.com/CycloneDX/cyclonedx-go v0.8.0/go.mod h1:K2bA+324+Og0X84fA8HhN2X066K7Bxz4rpMQ4ZhjtSk= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= @@ -107,8 +107,8 @@ github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZV github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E= github.com/anchore/packageurl-go v0.1.1-0.20230104203445-02e0a6721501 h1:AV7qjwMcM4r8wFhJq3jLRztew3ywIyPTRapl2T1s9o8= github.com/anchore/packageurl-go v0.1.1-0.20230104203445-02e0a6721501/go.mod h1:Blo6OgJNiYF41ufcgHKkbCKF2MDOMlrqhXv/ij6ocR4= -github.com/anchore/stereoscope v0.0.0-20231117203853-3610f4ef3e83 h1:mxGIOmj+asEm8LUkPTG3/v0hi27WIlDVjiEVsUB9eqY= -github.com/anchore/stereoscope v0.0.0-20231117203853-3610f4ef3e83/go.mod h1:GKAnytSVV1hoqB5r5Gd9M5Ph3Rzqq0zPdEJesewjC2w= +github.com/anchore/stereoscope v0.0.0-20231215220732-4b999b76ca89 h1:dymFMCwnENqLr74KQppq8zHKwOPL0M1ToYAU+KVfTew= +github.com/anchore/stereoscope v0.0.0-20231215220732-4b999b76ca89/go.mod h1:GKAnytSVV1hoqB5r5Gd9M5Ph3Rzqq0zPdEJesewjC2w= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= @@ -147,8 +147,8 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbles v0.16.1 h1:6uzpAAaT9ZqKssntbvZMlksWHruQLNxg49H5WdeuYSY= github.com/charmbracelet/bubbles v0.16.1/go.mod h1:2QCp9LFlEsBQMvIYERr7Ww2H2bA7xen1idUDIzm/+Xc= -github.com/charmbracelet/bubbletea v0.24.2 h1:uaQIKx9Ai6Gdh5zpTbGiWpytMU+CfsPp06RaW2cx/SY= -github.com/charmbracelet/bubbletea v0.24.2/go.mod h1:XdrNrV4J8GiyshTtx3DNuYkR1FDaJmO3l2nejekbsgg= +github.com/charmbracelet/bubbletea v0.25.0 h1:bAfwk7jRz7FKFl9RzlIULPkStffg5k6pNt5dywy4TcM= +github.com/charmbracelet/bubbletea v0.25.0/go.mod h1:EN3QDR1T5ZdWmdfDzYcqOCAps45+QIJbLOBxmVNWNNg= github.com/charmbracelet/harmonica v0.2.0 h1:8NxJWRWg/bzKqqEaaeFNipOu77YR5t8aSwG4pgaUBiQ= github.com/charmbracelet/harmonica v0.2.0/go.mod h1:KSri/1RMQOZLbw7AHqgcBycp8pgJnQMYYT8QZRqZ1Ao= github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1p/u1KWg= @@ -288,8 +288,8 @@ github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+ github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= -github.com/go-git/go-git/v5 v5.10.1 h1:tu8/D8i+TWxgKpzQ3Vc43e+kkhXqtsZCKI/egajKnxk= -github.com/go-git/go-git/v5 v5.10.1/go.mod h1:uEuHjxkHap8kAl//V5F/nNWwqIYtP/402ddd05mp0wg= +github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3cA4= +github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lKqXmCUiUCY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -397,8 +397,8 @@ github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S3 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= -github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -664,8 +664,8 @@ github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUz github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/saferwall/pe v1.4.7 h1:A+G3DxX49paJ5OsxBfHKskhyDtmTjShlDmBd81IsHlQ= -github.com/saferwall/pe v1.4.7/go.mod h1:SNzv3cdgk8SBI0UwHfyTcdjawfdnN+nbydnEL7GZ25s= +github.com/saferwall/pe v1.4.8 h1:ey/L8FGBMrJ1Xh+Rltj1MAFPZ4LOQYGJqNa5B1Na6B0= +github.com/saferwall/pe v1.4.8/go.mod h1:SNzv3cdgk8SBI0UwHfyTcdjawfdnN+nbydnEL7GZ25s= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= github.com/sagikazarmark/locafero v0.3.0 h1:zT7VEGWC2DTflmccN/5T1etyKvxSxpHsjb9cJvm4SvQ= github.com/sagikazarmark/locafero v0.3.0/go.mod h1:w+v7UsPNFwzF1cHuOajOOzoq4U7v/ig1mpRjqV+Bu1U= @@ -1353,8 +1353,8 @@ modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sqlite v1.27.0 h1:MpKAHoyYB7xqcwnUwkuD+npwEa0fojF0B5QRbN+auJ8= -modernc.org/sqlite v1.27.0/go.mod h1:Qxpazz0zH8Z1xCFyi5GSL3FzbtZ3fvbjmywNogldEW0= +modernc.org/sqlite v1.28.0 h1:Zx+LyDDmXczNnEQdvPuEfcFVA2ZPyaD7UCZDjef3BHQ= +modernc.org/sqlite v1.28.0/go.mod h1:Qxpazz0zH8Z1xCFyi5GSL3FzbtZ3fvbjmywNogldEW0= modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY= diff --git a/syft/cataloging/config.go b/syft/cataloging/config.go index ba9b7e4a452..d8e4f397391 100644 --- a/syft/cataloging/config.go +++ b/syft/cataloging/config.go @@ -4,3 +4,10 @@ type ArchiveSearchConfig struct { IncludeIndexedArchives bool `yaml:"include-indexed-archives" json:"include-indexed-archives" mapstructure:"include-indexed-archives"` IncludeUnindexedArchives bool `yaml:"include-unindexed-archives" json:"include-unindexed-archives" mapstructure:"include-unindexed-archives"` } + +func DefaultArchiveSearchConfig() ArchiveSearchConfig { + return ArchiveSearchConfig{ + IncludeIndexedArchives: true, + IncludeUnindexedArchives: false, + } +} diff --git a/syft/file/cataloger/filedigest/cataloger.go b/syft/file/cataloger/filedigest/cataloger.go index 91f719c01e2..00f193b2787 100644 --- a/syft/file/cataloger/filedigest/cataloger.go +++ b/syft/file/cataloger/filedigest/cataloger.go @@ -37,7 +37,11 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina locations = intCataloger.AllRegularFiles(resolver) } else { for _, c := range coordinates { - locations = append(locations, file.NewLocationFromCoordinates(c)) + locs, err := resolver.FilesByPath(c.RealPath) + if err != nil { + return nil, fmt.Errorf("unable to get file locations for path %q: %w", c.RealPath, err) + } + locations = append(locations, locs...) } } diff --git a/syft/file/cataloger/filedigest/cataloger_test.go b/syft/file/cataloger/filedigest/cataloger_test.go index 9ebaceed007..417e4b7398e 100644 --- a/syft/file/cataloger/filedigest/cataloger_test.go +++ b/syft/file/cataloger/filedigest/cataloger_test.go @@ -160,3 +160,47 @@ func TestDigestsCataloger_MixFileTypes(t *testing.T) { }) } } + +func TestFileDigestCataloger_GivenCoordinates(t *testing.T) { + testImage := "image-file-type-mix" + + img := imagetest.GetFixtureImage(t, "docker-archive", testImage) + + c := NewCataloger([]crypto.Hash{crypto.SHA256}) + + src, err := source.NewFromStereoscopeImageObject(img, testImage, nil) + require.NoError(t, err) + + resolver, err := src.FileResolver(source.SquashedScope) + require.NoError(t, err) + + tests := []struct { + path string + exists bool + expected string + }{ + { + path: "/file-1.txt", + exists: true, + expected: "b089629781f05ef805b4511e93717f2ffa4c9d991771d5cbfa4b7242b4ef5fff", + }, + } + + for _, test := range tests { + t.Run(test.path, func(t *testing.T) { + _, ref, err := img.SquashedTree().File(stereoscopeFile.Path(test.path)) + require.NoError(t, err) + + l := file.NewLocationFromImage(test.path, *ref.Reference, img) + + // note: an important difference between this test and the previous is that this test is using a list + // of specific coordinates to catalog + actual, err := c.Catalog(resolver, l.Coordinates) + require.NoError(t, err) + require.Len(t, actual, 1) + + assert.Equal(t, test.expected, actual[l.Coordinates][0].Value, "mismatched digests") + }) + } + +} diff --git a/syft/file/cataloger/filemetadata/cataloger_test.go b/syft/file/cataloger/filemetadata/cataloger_test.go index 8c646e5704d..48f6b6c5fa6 100644 --- a/syft/file/cataloger/filemetadata/cataloger_test.go +++ b/syft/file/cataloger/filemetadata/cataloger_test.go @@ -168,7 +168,6 @@ func TestFileMetadataCataloger_GivenCoordinates(t *testing.T) { path string exists bool expected file.Metadata - err bool }{ { path: "/file-1.txt", diff --git a/syft/format/common/cyclonedxhelpers/decoder.go b/syft/format/common/cyclonedxhelpers/decoder.go index 37de22a9ad8..6af03df16f6 100644 --- a/syft/format/common/cyclonedxhelpers/decoder.go +++ b/syft/format/common/cyclonedxhelpers/decoder.go @@ -249,9 +249,22 @@ func extractDescriptor(meta *cyclonedx.Metadata) (desc sbom.Descriptor) { return } - for _, t := range *meta.Tools { - desc.Name = t.Name - desc.Version = t.Version + // handle 1.5 component element + if meta.Tools.Components != nil { + for _, t := range *meta.Tools.Components { + desc.Name = t.Name + desc.Version = t.Version + return + } + } + + // handle pre-1.5 tool element + if meta.Tools.Tools != nil { + for _, t := range *meta.Tools.Tools { + desc.Name = t.Name + desc.Version = t.Version + return + } } return diff --git a/syft/format/common/cyclonedxhelpers/format.go b/syft/format/common/cyclonedxhelpers/format.go index b5d96487741..548a65c24ef 100644 --- a/syft/format/common/cyclonedxhelpers/format.go +++ b/syft/format/common/cyclonedxhelpers/format.go @@ -114,11 +114,14 @@ func formatCPE(cpeString string) string { func toBomDescriptor(name, version string, srcMetadata source.Description) *cyclonedx.Metadata { return &cyclonedx.Metadata{ Timestamp: time.Now().Format(time.RFC3339), - Tools: &[]cyclonedx.Tool{ - { - Vendor: "anchore", - Name: name, - Version: version, + Tools: &cyclonedx.ToolsChoice{ + Components: &[]cyclonedx.Component{ + { + Type: cyclonedx.ComponentTypeApplication, + Author: "anchore", + Name: name, + Version: version, + }, }, }, Properties: toBomProperties(srcMetadata), diff --git a/syft/format/common/cyclonedxhelpers/format_test.go b/syft/format/common/cyclonedxhelpers/format_test.go index b792dea1b83..25428823bcb 100644 --- a/syft/format/common/cyclonedxhelpers/format_test.go +++ b/syft/format/common/cyclonedxhelpers/format_test.go @@ -168,13 +168,14 @@ func Test_toBomDescriptor(t *testing.T) { want: &cyclonedx.Metadata{ Timestamp: "", Lifecycles: nil, - Tools: &[]cyclonedx.Tool{ - { - Vendor: "anchore", - Name: "test-image", - Version: "1.0.0", - Hashes: nil, - ExternalReferences: nil, + Tools: &cyclonedx.ToolsChoice{ + Components: &[]cyclonedx.Component{ + { + Type: cyclonedx.ComponentTypeApplication, + Author: "anchore", + Name: "test-image", + Version: "1.0.0", + }, }, }, Authors: nil, diff --git a/syft/format/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxDirectoryEncoder.golden b/syft/format/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxDirectoryEncoder.golden index 8328af48678..4f1190eceab 100644 --- a/syft/format/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxDirectoryEncoder.golden +++ b/syft/format/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxDirectoryEncoder.golden @@ -6,13 +6,16 @@ "version": 1, "metadata": { "timestamp": "timestamp:redacted", - "tools": [ - { - "vendor": "anchore", - "name": "syft", - "version": "v0.42.0-bogus" - } - ], + "tools": { + "components": [ + { + "type": "application", + "author": "anchore", + "name": "syft", + "version": "v0.42.0-bogus" + } + ] + }, "component": { "bom-ref":"redacted", "type": "file", diff --git a/syft/format/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden b/syft/format/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden index d8aef04cfd6..3b4844b0231 100644 --- a/syft/format/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden +++ b/syft/format/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden @@ -6,13 +6,16 @@ "version": 1, "metadata": { "timestamp": "timestamp:redacted", - "tools": [ - { - "vendor": "anchore", - "name": "syft", - "version": "v0.42.0-bogus" - } - ], + "tools": { + "components": [ + { + "type": "application", + "author": "anchore", + "name": "syft", + "version": "v0.42.0-bogus" + } + ] + }, "component": { "bom-ref":"redacted", "type": "container", diff --git a/syft/format/cyclonedxxml/test-fixtures/snapshot/TestCycloneDxDirectoryEncoder.golden b/syft/format/cyclonedxxml/test-fixtures/snapshot/TestCycloneDxDirectoryEncoder.golden index 85f455aeadb..c56f6724b02 100644 --- a/syft/format/cyclonedxxml/test-fixtures/snapshot/TestCycloneDxDirectoryEncoder.golden +++ b/syft/format/cyclonedxxml/test-fixtures/snapshot/TestCycloneDxDirectoryEncoder.golden @@ -3,11 +3,13 @@ redacted - - anchore - syft - v0.42.0-bogus - + + + anchore + syft + v0.42.0-bogus + + some/path diff --git a/syft/format/cyclonedxxml/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden b/syft/format/cyclonedxxml/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden index 750bf863085..8e0c8800fa6 100644 --- a/syft/format/cyclonedxxml/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden +++ b/syft/format/cyclonedxxml/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden @@ -3,11 +3,13 @@ redacted - - anchore - syft - v0.42.0-bogus - + + + anchore + syft + v0.42.0-bogus + + user-image-input diff --git a/syft/format/syftjson/encoder_test.go b/syft/format/syftjson/encoder_test.go index f11a9e79af3..64346217395 100644 --- a/syft/format/syftjson/encoder_test.go +++ b/syft/format/syftjson/encoder_test.go @@ -297,10 +297,15 @@ func TestEncodeFullJSONDocument(t *testing.T) { }, } + cfg := DefaultEncoderConfig() + cfg.Pretty = true + enc, err := NewFormatEncoderWithConfig(cfg) + require.NoError(t, err) + testutil.AssertEncoderAgainstGoldenSnapshot(t, testutil.EncoderSnapshotTestConfig{ Subject: s, - Format: NewFormatEncoder(), + Format: enc, UpdateSnapshot: *updateSnapshot, PersistRedactionsInSnapshot: true, IsJSON: true, diff --git a/syft/pkg/cataloger/binary/cataloger_test.go b/syft/pkg/cataloger/binary/cataloger_test.go index c71c11aedae..d38de2e41b1 100644 --- a/syft/pkg/cataloger/binary/cataloger_test.go +++ b/syft/pkg/cataloger/binary/cataloger_test.go @@ -715,6 +715,18 @@ func Test_Cataloger_DefaultClassifiers_PositiveCases(t *testing.T) { Metadata: metadata("consul-binary"), }, }, + { + name: "positive-erlang-25.3.2.7", + fixtureDir: "test-fixtures/classifiers/positive/erlang-25.3.2.7", + expected: pkg.Package{ + Name: "erlang", + Version: "25.3.2.7", + Type: "binary", + PURL: "pkg:generic/erlang@25.3.2.7", + Locations: locations("erlexec"), + Metadata: metadata("erlang-binary"), + }, + }, { name: "positive-nginx-1.25.1", fixtureDir: "test-fixtures/classifiers/positive/nginx-1.25.1", diff --git a/syft/pkg/cataloger/binary/default_classifiers.go b/syft/pkg/cataloger/binary/default_classifiers.go index 3843e9c5568..95376052fcd 100644 --- a/syft/pkg/cataloger/binary/default_classifiers.go +++ b/syft/pkg/cataloger/binary/default_classifiers.go @@ -46,6 +46,15 @@ var defaultClassifiers = []classifier{ PURL: mustPURL("pkg:generic/go@version"), CPEs: singleCPE("cpe:2.3:a:golang:go:*:*:*:*:*:*:*:*"), }, + { + Class: "julia-binary", + FileGlob: "**/libjulia-internal.so", + EvidenceMatcher: fileContentsVersionMatcher( + `(?m)__init__\x00(?P[0-9]+\.[0-9]+\.[0-9]+)\x00verify`), + Package: "julia", + PURL: mustPURL("pkg:generic/julia@version"), + CPEs: singleCPE("cpe:2.3:a:julialang:julia:*:*:*:*:*:*:*:*"), + }, { Class: "helm", FileGlob: "**/helm", @@ -172,6 +181,15 @@ var defaultClassifiers = []classifier{ PURL: mustPURL("pkg:generic/php@version"), CPEs: singleCPE("cpe:2.3:a:php:php:*:*:*:*:*:*:*:*"), }, + { + Class: "php-composer-binary", + FileGlob: "**/composer*", + EvidenceMatcher: fileContentsVersionMatcher( + `(?m)'pretty_version'\s*=>\s*'(?P[0-9]+\.[0-9]+\.[0-9]+(beta[0-9]+|alpha[0-9]+|RC[0-9]+)?)'`), + Package: "composer", + PURL: mustPURL("pkg:generic/composer@version"), + CPEs: singleCPE("cpe:2.3:a:getcomposer:composer:*:*:*:*:*:*:*:*"), + }, { Class: "httpd-binary", FileGlob: "**/httpd", @@ -265,6 +283,17 @@ var defaultClassifiers = []classifier{ PURL: mustPURL("pkg:generic/ruby@version"), CPEs: singleCPE("cpe:2.3:a:ruby-lang:ruby:*:*:*:*:*:*:*:*"), }, + { + Class: "erlang-binary", + FileGlob: "**/erlexec", + EvidenceMatcher: fileContentsVersionMatcher( + // [NUL]/usr/local/src/otp-25.3.2.7/erts/ + `(?m)\\x00/usr/local/src/otp-(?P[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+?)/erts/`, + ), + Package: "erlang", + PURL: mustPURL("pkg:generic/erlang@version"), + CPEs: singleCPE("cpe:2.3:a:erlang:erlang\\/otp:*:*:*:*:*:*:*:*"), + }, { Class: "consul-binary", FileGlob: "**/consul", diff --git a/syft/pkg/cataloger/binary/test-fixtures/classifiers/positive/erlang-25.3.2.7/erlexec b/syft/pkg/cataloger/binary/test-fixtures/classifiers/positive/erlang-25.3.2.7/erlexec new file mode 100755 index 00000000000..9ad677967f9 Binary files /dev/null and b/syft/pkg/cataloger/binary/test-fixtures/classifiers/positive/erlang-25.3.2.7/erlexec differ diff --git a/syft/pkg/cataloger/cataloger.go b/syft/pkg/cataloger/cataloger.go index b9c4867cda5..ca3d4086176 100644 --- a/syft/pkg/cataloger/cataloger.go +++ b/syft/pkg/cataloger/cataloger.go @@ -83,7 +83,7 @@ func DirectoryCatalogers(cfg Config) []pkg.Cataloger { haskell.NewHackageCataloger(), java.NewArchiveCataloger(cfg.JavaConfig()), java.NewGradleLockfileCataloger(), - java.NewPomCataloger(), + java.NewPomCataloger(cfg.JavaConfig()), java.NewNativeImageCataloger(), javascript.NewLockCataloger(cfg.Javascript), nix.NewStoreCataloger(), @@ -122,7 +122,7 @@ func AllCatalogers(cfg Config) []pkg.Cataloger { haskell.NewHackageCataloger(), java.NewArchiveCataloger(cfg.JavaConfig()), java.NewGradleLockfileCataloger(), - java.NewPomCataloger(), + java.NewPomCataloger(cfg.JavaConfig()), java.NewNativeImageCataloger(), javascript.NewLockCataloger(cfg.Javascript), javascript.NewPackageCataloger(), diff --git a/syft/pkg/cataloger/common/cpe/dictionary/data/cpe-index.json b/syft/pkg/cataloger/common/cpe/dictionary/data/cpe-index.json index e9eee86f4a9..bec345ac432 100644 --- a/syft/pkg/cataloger/common/cpe/dictionary/data/cpe-index.json +++ b/syft/pkg/cataloger/common/cpe/dictionary/data/cpe-index.json @@ -115,6 +115,7 @@ "google-kubernetes-engine": "cpe:2.3:a:jenkins:google_kubernetes_engine:*:*:*:*:*:jenkins:*:*", "google-oauth": "cpe:2.3:a:jenkins:google_oauth_credentials:*:*:*:*:*:jenkins:*:*", "google-play-android-publisher": "cpe:2.3:a:jenkins:google-play-android-publisher:*:*:*:*:*:jenkins:*:*", + "gradle": "cpe:2.3:a:jenkins:gradle:*:*:*:*:*:jenkins:*:*", "groovy": "cpe:2.3:a:jenkins:groovy:*:*:*:*:*:jenkins:*:*", "harvest": "cpe:2.3:a:jenkins:harvest_scm:*:*:*:*:*:jenkins:*:*", "hipchat": "cpe:2.3:a:atlassian:hipchat:*:*:*:*:*:jenkins:*:*", @@ -288,6 +289,7 @@ "@awsui/components-react": "cpe:2.3:a:amazon:awsui\\/components-react:*:*:*:*:*:node.js:*:*", "@azure/ms-rest-nodeauth": "cpe:2.3:a:microsoft:ms-rest-nodeauth:*:*:*:*:*:node.js:*:*", "@backstage/plugin-auth-backend": "cpe:2.3:a:linuxfoundation:auth_backend:*:*:*:*:*:node.js:*:*", + "@evershop/evershop": "cpe:2.3:a:evershop:evershop:*:*:*:*:*:node.js:*:*", "@fastly/js-compute": "cpe:2.3:a:fastly:js-compute:*:*:*:*:*:node.js:*:*", "@firebase/util": "cpe:2.3:a:google:firebase\\/util:*:*:*:*:*:node.js:*:*", "@github/paste-markdown": "cpe:2.3:a:paste-markdown_project:paste-markdown:*:*:*:*:*:node.js:*:*", @@ -295,9 +297,11 @@ "@nuxt/devalue": "cpe:2.3:a:nuxtjs:\\@nuxt\\/devalue:*:*:*:*:*:node.js:*:*", "@progfay/scrapbox-parser": "cpe:2.3:a:scrapbox-parser_project:scrapbox-parser:*:*:*:*:*:node.js:*:*", "@rkesters/gnuplot": "cpe:2.3:a:gnuplot_project:gnuplot:*:*:*:*:*:node.js:*:*", + "@sap/xssec": "cpe:2.3:a:sap:\\@sap\\/xssec:*:*:*:*:*:node.js:*:*", "@solana/pay": "cpe:2.3:a:solanalabs:pay:*:*:*:*:*:*:*:*", "@strikeentco/set": "cpe:2.3:a:set_project:set:*:*:*:*:*:node.js:*:*", "@thi.ng/egf": "cpe:2.3:a:\\@thi.ng\\/egf_project:\\@thi.ng\\/egf:*:*:*:*:*:node.js:*:*", + "@vue/devtools": "cpe:2.3:a:vuejs:devtools:*:*:*:*:*:node.js:*:*", "@zxcvbn-ts/core": "cpe:2.3:a:zxcvbn-ts_project:zxcvbn-ts:*:*:*:*:*:node.js:*:*", "Proto": "cpe:2.3:a:proto_project:proto:*:*:*:*:*:node.js:*:*", "Templ8": "cpe:2.3:a:templ8_project:templ8:*:*:*:*:*:node.js:*:*", @@ -389,6 +393,7 @@ "conf-cfg-ini": "cpe:2.3:a:conf-cfg-ini_project:conf-cfg-ini:*:*:*:*:*:node.js:*:*", "confinit": "cpe:2.3:a:confinit_project:confinit:*:*:*:*:*:node.js:*:*", "confucious": "cpe:2.3:a:realseriousgames:confucious:*:*:*:*:*:node.js:*:*", + "connect-multiparty": "cpe:2.3:a:connect-multiparty_project:connect-multiparty:*:*:*:*:*:*:*:*", "connect-pg-simple": "cpe:2.3:a:connect-pg-simple_project:connect-pg-simple:*:*:*:*:*:node.js:*:*", "connection-tester": "cpe:2.3:a:connection-tester_project:connection-tester:*:*:*:*:*:node.js:*:*", "console-io": "cpe:2.3:a:console-io_project:console-io:*:*:*:*:*:node.js:*:*", @@ -1084,11 +1089,15 @@ "reqmgr2": "cpe:2.3:a:reqmgr2_project:reqmgr2:*:*:*:*:*:python:*:*", "reqmon": "cpe:2.3:a:reqmon_project:reqmon:*:*:*:*:*:python:*:*", "requests-xml": "cpe:2.3:a:requests-xml_project:requests-xml:*:*:*:*:*:python:*:*", + "rondolu-yt-concate": "cpe:2.3:a:rondolu-yt-concate_project:rondolu-yt-concate:*:*:*:*:*:pypi:*:*", "rope": "cpe:2.3:a:rope_project:rope:*:*:*:*:*:python:*:*", "rply": "cpe:2.3:a:rply_project:rply:*:*:*:*:*:*:*:*", "rsa": "cpe:2.3:a:python:rsa:*:*:*:*:*:python:*:*", "ruamel.yaml": "cpe:2.3:a:ruamel.yaml_project:ruamel.yaml:*:*:*:*:*:*:*:*", + "sap-xssec": "cpe:2.3:a:sap:sap-xssec:*:*:*:*:*:python:*:*", + "scoptrial": "cpe:2.3:a:scoptrial_project:scoptrial:*:*:*:*:*:pypi:*:*", "simiki": "cpe:2.3:a:simiki_project:simiki:*:*:*:*:*:*:*:*", + "sixfab-power-python-api": "cpe:2.3:a:sixfab-tool_project:sixfab-tool:*:*:*:*:*:pypi:*:*", "slashify": "cpe:2.3:a:google:slashify:*:*:*:*:*:node.js:*:*", "sopel-plugins.channelmgnt": "cpe:2.3:a:mirahezebots:channelmgnt:*:*:*:*:*:sopel:*:*", "spacy": "cpe:2.3:a:explosion:spacy:*:*:*:*:*:python:*:*", @@ -1219,6 +1228,7 @@ "bzip2": "cpe:2.3:a:bzip2_project:bzip2:*:*:*:*:*:rust:*:*", "cache": "cpe:2.3:a:cache_project:cache:*:*:*:*:*:rust:*:*", "cached": "cpe:2.3:a:cached_project:cached:*:*:*:*:*:rust:*:*", + "candid": "cpe:2.3:a:dfinity:candid:*:*:*:*:*:rust:*:*", "cbox": "cpe:2.3:a:cbox_project:cbox:*:*:*:*:*:rust:*:*", "ckb": "cpe:2.3:a:nervos:ckb:*:*:*:*:*:rust:*:*", "conduit-hyper": "cpe:2.3:a:conduit-hyper_project:conduit-hyper:*:*:*:*:*:rust:*:*", diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index 1dee214efee..c423be7694e 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -3,15 +3,9 @@ package java import ( "crypto" "fmt" - "io" - "net/http" - "net/url" "os" "path" "strings" - "time" - - "github.com/vifraa/gopom" intFile "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/licenses" @@ -359,98 +353,6 @@ func findPomLicenses(pomProjectObject *parsedPomProject, cfg ArchiveCatalogerCon } } -func formatMavenPomURL(groupID, artifactID, version, mavenBaseURL string) (requestURL string, err error) { - // groupID needs to go from maven.org -> maven/org - urlPath := strings.Split(groupID, ".") - artifactPom := fmt.Sprintf("%s-%s.pom", artifactID, version) - urlPath = append(urlPath, artifactID, version, artifactPom) - - // ex:"https://repo1.maven.org/maven2/groupID/artifactID/artifactPom - requestURL, err = url.JoinPath(mavenBaseURL, urlPath...) - if err != nil { - return requestURL, fmt.Errorf("could not construct maven url: %w", err) - } - return requestURL, err -} - -func recursivelyFindLicensesFromParentPom(groupID, artifactID, version string, cfg ArchiveCatalogerConfig) []string { - var licenses []string - // As there can be nested parent poms, we'll recursively check for licenses until we reach the max depth - for i := 0; i < cfg.MaxParentRecursiveDepth; i++ { - parentPom, err := getPomFromMavenRepo(groupID, artifactID, version, cfg.MavenBaseURL) - if err != nil { - // We don't want to abort here as the parent pom might not exist in Maven Central, we'll just log the error - log.Tracef("unable to get parent pom from Maven central: %v", err) - return []string{} - } - parentLicenses := parseLicensesFromPom(parentPom) - if len(parentLicenses) > 0 || parentPom == nil || parentPom.Parent == nil { - licenses = parentLicenses - break - } - - groupID = *parentPom.Parent.GroupID - artifactID = *parentPom.Parent.ArtifactID - version = *parentPom.Parent.Version - } - - return licenses -} - -func getPomFromMavenRepo(groupID, artifactID, version, mavenBaseURL string) (*gopom.Project, error) { - requestURL, err := formatMavenPomURL(groupID, artifactID, version, mavenBaseURL) - if err != nil { - return nil, err - } - log.Tracef("trying to fetch parent pom from Maven central %s", requestURL) - - mavenRequest, err := http.NewRequest(http.MethodGet, requestURL, nil) - if err != nil { - return nil, fmt.Errorf("unable to format request for Maven central: %w", err) - } - - httpClient := &http.Client{ - Timeout: time.Second * 10, - } - - resp, err := httpClient.Do(mavenRequest) - if err != nil { - return nil, fmt.Errorf("unable to get pom from Maven central: %w", err) - } - defer func() { - if err := resp.Body.Close(); err != nil { - log.Errorf("unable to close body: %+v", err) - } - }() - - bytes, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("unable to parse pom from Maven central: %w", err) - } - - pom, err := decodePomXML(strings.NewReader(string(bytes))) - if err != nil { - return nil, fmt.Errorf("unable to parse pom from Maven central: %w", err) - } - - return &pom, nil -} - -func parseLicensesFromPom(pom *gopom.Project) []string { - var licenses []string - if pom != nil && pom.Licenses != nil { - for _, license := range *pom.Licenses { - if license.Name != nil { - licenses = append(licenses, *license.Name) - } else if license.URL != nil { - licenses = append(licenses, *license.URL) - } - } - } - - return licenses -} - // discoverPkgsFromAllMavenFiles parses Maven POM properties/xml for a given // parent package, returning all listed Java packages found for each pom // properties discovered and potentially updating the given parentPkg with new diff --git a/syft/pkg/cataloger/java/cataloger.go b/syft/pkg/cataloger/java/cataloger.go index 6764109b559..fa8560ec45e 100644 --- a/syft/pkg/cataloger/java/cataloger.go +++ b/syft/pkg/cataloger/java/cataloger.go @@ -31,9 +31,11 @@ func NewArchiveCataloger(cfg ArchiveCatalogerConfig) *generic.Cataloger { // NewPomCataloger returns a cataloger capable of parsing dependencies from a pom.xml file. // Pom files list dependencies that maybe not be locally installed yet. -func NewPomCataloger() pkg.Cataloger { +func NewPomCataloger(cfg ArchiveCatalogerConfig) pkg.Cataloger { + gap := newGenericArchiveParserAdapter(cfg) + return generic.NewCataloger("java-pom-cataloger"). - WithParserByGlobs(parserPomXML, "**/pom.xml") + WithParserByGlobs(gap.parserPomXML, "**/pom.xml") } // NewGradleLockfileCataloger returns a cataloger capable of parsing dependencies from a gradle.lockfile file. diff --git a/syft/pkg/cataloger/java/cataloger_test.go b/syft/pkg/cataloger/java/cataloger_test.go index c339e1e7a0f..da524fc5ba3 100644 --- a/syft/pkg/cataloger/java/cataloger_test.go +++ b/syft/pkg/cataloger/java/cataloger_test.go @@ -89,7 +89,15 @@ func Test_POMCataloger_Globs(t *testing.T) { pkgtest.NewCatalogTester(). FromDirectory(t, test.fixture). ExpectsResolverContentQueries(test.expected). - TestCataloger(t, NewPomCataloger()) + TestCataloger(t, + NewPomCataloger( + ArchiveCatalogerConfig{ + ArchiveSearchConfig: cataloging.ArchiveSearchConfig{ + IncludeIndexedArchives: true, + IncludeUnindexedArchives: true, + }, + }, + )) }) } } diff --git a/syft/pkg/cataloger/java/config.go b/syft/pkg/cataloger/java/config.go index 132f602478e..14c31d33426 100644 --- a/syft/pkg/cataloger/java/config.go +++ b/syft/pkg/cataloger/java/config.go @@ -13,10 +13,7 @@ type ArchiveCatalogerConfig struct { func DefaultArchiveCatalogerConfig() ArchiveCatalogerConfig { return ArchiveCatalogerConfig{ - ArchiveSearchConfig: cataloging.ArchiveSearchConfig{ - IncludeIndexedArchives: true, - IncludeUnindexedArchives: false, - }, + ArchiveSearchConfig: cataloging.DefaultArchiveSearchConfig(), UseNetwork: false, MavenBaseURL: mavenBaseURL, MaxParentRecursiveDepth: 5, diff --git a/syft/pkg/cataloger/java/maven_repo_utils.go b/syft/pkg/cataloger/java/maven_repo_utils.go new file mode 100644 index 00000000000..126037c26e8 --- /dev/null +++ b/syft/pkg/cataloger/java/maven_repo_utils.go @@ -0,0 +1,133 @@ +package java + +import ( + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/vifraa/gopom" + + "github.com/anchore/syft/internal/log" +) + +func formatMavenPomURL(groupID, artifactID, version, mavenBaseURL string) (requestURL string, err error) { + // groupID needs to go from maven.org -> maven/org + urlPath := strings.Split(groupID, ".") + artifactPom := fmt.Sprintf("%s-%s.pom", artifactID, version) + urlPath = append(urlPath, artifactID, version, artifactPom) + + // ex:"https://repo1.maven.org/maven2/groupID/artifactID/artifactPom + requestURL, err = url.JoinPath(mavenBaseURL, urlPath...) + if err != nil { + return requestURL, fmt.Errorf("could not construct maven url: %w", err) + } + return requestURL, err +} + +// An artifact can have its version defined in a parent's DependencyManagement section +func recursivelyFindVersionFromParentPom(groupID, artifactID, parentGroupID, parentArtifactID, parentVersion string, cfg ArchiveCatalogerConfig) string { + // As there can be nested parent poms, we'll recursively check for the version until we reach the max depth + for i := 0; i < cfg.MaxParentRecursiveDepth; i++ { + parentPom, err := getPomFromMavenRepo(parentGroupID, parentArtifactID, parentVersion, cfg.MavenBaseURL) + if err != nil { + // We don't want to abort here as the parent pom might not exist in Maven Central, we'll just log the error + log.Tracef("unable to get parent pom from Maven central: %v", err) + break + } + if parentPom != nil && parentPom.DependencyManagement != nil { + for _, dependency := range *parentPom.DependencyManagement.Dependencies { + if groupID == *dependency.GroupID && artifactID == *dependency.ArtifactID && dependency.Version != nil { + return *dependency.Version + } + } + } + if parentPom == nil || parentPom.Parent == nil { + break + } + parentGroupID = *parentPom.Parent.GroupID + parentArtifactID = *parentPom.Parent.ArtifactID + parentVersion = *parentPom.Parent.Version + } + return "" +} + +func recursivelyFindLicensesFromParentPom(groupID, artifactID, version string, cfg ArchiveCatalogerConfig) []string { + var licenses []string + // As there can be nested parent poms, we'll recursively check for licenses until we reach the max depth + for i := 0; i < cfg.MaxParentRecursiveDepth; i++ { + parentPom, err := getPomFromMavenRepo(groupID, artifactID, version, cfg.MavenBaseURL) + if err != nil { + // We don't want to abort here as the parent pom might not exist in Maven Central, we'll just log the error + log.Tracef("unable to get parent pom from Maven central: %v", err) + return []string{} + } + parentLicenses := parseLicensesFromPom(parentPom) + if len(parentLicenses) > 0 || parentPom == nil || parentPom.Parent == nil { + licenses = parentLicenses + break + } + + groupID = *parentPom.Parent.GroupID + artifactID = *parentPom.Parent.ArtifactID + version = *parentPom.Parent.Version + } + + return licenses +} + +func getPomFromMavenRepo(groupID, artifactID, version, mavenBaseURL string) (*gopom.Project, error) { + requestURL, err := formatMavenPomURL(groupID, artifactID, version, mavenBaseURL) + if err != nil { + return nil, err + } + log.Tracef("trying to fetch parent pom from Maven central %s", requestURL) + + mavenRequest, err := http.NewRequest(http.MethodGet, requestURL, nil) + if err != nil { + return nil, fmt.Errorf("unable to format request for Maven central: %w", err) + } + + httpClient := &http.Client{ + Timeout: time.Second * 10, + } + + resp, err := httpClient.Do(mavenRequest) + if err != nil { + return nil, fmt.Errorf("unable to get pom from Maven central: %w", err) + } + defer func() { + if err := resp.Body.Close(); err != nil { + log.Errorf("unable to close body: %+v", err) + } + }() + + bytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("unable to parse pom from Maven central: %w", err) + } + + pom, err := decodePomXML(strings.NewReader(string(bytes))) + if err != nil { + return nil, fmt.Errorf("unable to parse pom from Maven central: %w", err) + } + + return &pom, nil +} + +func parseLicensesFromPom(pom *gopom.Project) []string { + var licenses []string + if pom != nil && pom.Licenses != nil { + for _, license := range *pom.Licenses { + if license.Name != nil { + licenses = append(licenses, *license.Name) + } else if license.URL != nil { + licenses = append(licenses, *license.URL) + } + } + } + + return licenses +} diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index 75376521fc9..b3c6f2b6f78 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -24,7 +24,7 @@ const pomXMLGlob = "*pom.xml" var propertyMatcher = regexp.MustCompile("[$][{][^}]+[}]") -func parserPomXML(_ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { +func (gap genericArchiveParserAdapter) parserPomXML(_ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { pom, err := decodePomXML(reader) if err != nil { return nil, nil, err @@ -36,6 +36,7 @@ func parserPomXML(_ file.Resolver, _ *generic.Environment, reader file.LocationR p := newPackageFromPom( pom, dep, + gap.cfg, reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ) if p.Name == "" { @@ -97,7 +98,7 @@ func newPomProject(path string, p gopom.Project, location file.Location) *parsed } } -func newPackageFromPom(pom gopom.Project, dep gopom.Dependency, locations ...file.Location) pkg.Package { +func newPackageFromPom(pom gopom.Project, dep gopom.Dependency, cfg ArchiveCatalogerConfig, locations ...file.Location) pkg.Package { m := pkg.JavaArchive{ PomProperties: &pkg.JavaPomProperties{ GroupID: resolveProperty(pom, dep.GroupID, "groupId"), @@ -109,10 +110,32 @@ func newPackageFromPom(pom gopom.Project, dep gopom.Dependency, locations ...fil name := safeString(dep.ArtifactID) version := resolveProperty(pom, dep.Version, "version") + licenses := make([]pkg.License, 0) + if cfg.UseNetwork { + if version == "" { + // If we have no version then let's try to get it from a parent pom DependencyManagement section + version = recursivelyFindVersionFromParentPom(*dep.GroupID, *dep.ArtifactID, *pom.Parent.GroupID, *pom.Parent.ArtifactID, *pom.Parent.Version, cfg) + } + if version != "" { + parentLicenses := recursivelyFindLicensesFromParentPom( + m.PomProperties.GroupID, + m.PomProperties.ArtifactID, + version, + cfg) + + if len(parentLicenses) > 0 { + for _, licenseName := range parentLicenses { + licenses = append(licenses, pkg.NewLicenseFromFields(licenseName, "", nil)) + } + } + } + } + p := pkg.Package{ Name: name, Version: version, Locations: file.NewLocationSet(locations...), + Licenses: pkg.NewLicenseSet(licenses...), PURL: packageURL(name, version, m), Language: pkg.Java, Type: pkg.JavaPkg, // TODO: should we differentiate between packages from jar/war/zip versus packages from a pom.xml that were not installed yet? diff --git a/syft/pkg/cataloger/java/parse_pom_xml_test.go b/syft/pkg/cataloger/java/parse_pom_xml_test.go index b794d28e72e..c845233b125 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml_test.go +++ b/syft/pkg/cataloger/java/parse_pom_xml_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/require" "github.com/vifraa/gopom" + "github.com/anchore/syft/syft/cataloging" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/license" "github.com/anchore/syft/syft/pkg" @@ -61,7 +62,15 @@ func Test_parserPomXML(t *testing.T) { for i := range test.expected { test.expected[i].Locations.Add(file.NewLocation(test.input)) } - pkgtest.TestFileParser(t, test.input, parserPomXML, test.expected, nil) + + gap := newGenericArchiveParserAdapter(ArchiveCatalogerConfig{ + ArchiveSearchConfig: cataloging.ArchiveSearchConfig{ + IncludeIndexedArchives: true, + IncludeUnindexedArchives: true, + }, + }) + + pkgtest.TestFileParser(t, test.input, gap.parserPomXML, test.expected, nil) }) } } @@ -276,7 +285,14 @@ func Test_parseCommonsTextPomXMLProject(t *testing.T) { for i := range test.expected { test.expected[i].Locations.Add(file.NewLocation(test.input)) } - pkgtest.TestFileParser(t, test.input, parserPomXML, test.expected, nil) + + gap := newGenericArchiveParserAdapter(ArchiveCatalogerConfig{ + ArchiveSearchConfig: cataloging.ArchiveSearchConfig{ + IncludeIndexedArchives: true, + IncludeUnindexedArchives: true, + }, + }) + pkgtest.TestFileParser(t, test.input, gap.parserPomXML, test.expected, nil) }) } } diff --git a/syft/pkg/cataloger/javascript/config.go b/syft/pkg/cataloger/javascript/config.go index 9c6117970cd..f73e1899b53 100644 --- a/syft/pkg/cataloger/javascript/config.go +++ b/syft/pkg/cataloger/javascript/config.go @@ -3,25 +3,25 @@ package javascript const npmBaseURL = "https://registry.npmjs.org" type CatalogerConfig struct { - searchRemoteLicenses bool - npmBaseURL string + SearchRemoteLicenses bool `json:"search-remote-licenses" yaml:"search-remote-licenses" mapstructure:"search-remote-licenses"` + NPMBaseURL string `json:"npm-base-url" yaml:"npm-base-url" mapstructure:"npm-base-url"` } func DefaultCatalogerConfig() CatalogerConfig { return CatalogerConfig{ - searchRemoteLicenses: false, - npmBaseURL: npmBaseURL, + SearchRemoteLicenses: false, + NPMBaseURL: npmBaseURL, } } func (j CatalogerConfig) WithSearchRemoteLicenses(input bool) CatalogerConfig { - j.searchRemoteLicenses = input + j.SearchRemoteLicenses = input return j } func (j CatalogerConfig) WithNpmBaseURL(input string) CatalogerConfig { if input != "" { - j.npmBaseURL = input + j.NPMBaseURL = input } return j } diff --git a/syft/pkg/cataloger/javascript/package.go b/syft/pkg/cataloger/javascript/package.go index 127a44ccf17..990f5b01021 100644 --- a/syft/pkg/cataloger/javascript/package.go +++ b/syft/pkg/cataloger/javascript/package.go @@ -113,8 +113,8 @@ func newPnpmPackage(resolver file.Resolver, location file.Location, name, versio func newYarnLockPackage(cfg CatalogerConfig, resolver file.Resolver, location file.Location, name, version string) pkg.Package { var licenseSet pkg.LicenseSet - if cfg.searchRemoteLicenses { - license, err := getLicenseFromNpmRegistry(cfg.npmBaseURL, name, version) + if cfg.SearchRemoteLicenses { + license, err := getLicenseFromNpmRegistry(cfg.NPMBaseURL, name, version) if err == nil && license != "" { licenses := pkg.NewLicensesFromValues(license) licenseSet = pkg.NewLicenseSet(licenses...) diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go index a2fa7c5a34f..8a0bdee6d9c 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go @@ -204,7 +204,7 @@ func TestSearchYarnForLicenses(t *testing.T) { }{ { name: "search remote licenses returns the expected licenses when search is set to true", - config: CatalogerConfig{searchRemoteLicenses: true}, + config: CatalogerConfig{SearchRemoteLicenses: true}, requestHandlers: []handlerPath{ { // https://registry.yarnpkg.com/@babel/code-frame/7.10.4 @@ -232,7 +232,7 @@ func TestSearchYarnForLicenses(t *testing.T) { for _, handler := range tc.requestHandlers { mux.HandleFunc(handler.path, handler.handler) } - tc.config.npmBaseURL = url + tc.config.NPMBaseURL = url adapter := newGenericYarnLockAdapter(tc.config) pkgtest.TestFileParser(t, fixture, adapter.parseYarnLock, tc.expectedPackages, nil) }) diff --git a/syft/pkg/cataloger/python/package.go b/syft/pkg/cataloger/python/package.go index 59fd9f6b594..7e6f39a22fa 100644 --- a/syft/pkg/cataloger/python/package.go +++ b/syft/pkg/cataloger/python/package.go @@ -4,6 +4,8 @@ import ( "fmt" "github.com/anchore/packageurl-go" + "github.com/anchore/syft/internal/licenses" + "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" ) @@ -55,13 +57,42 @@ func newPackageForRequirementsWithMetadata(name, version string, metadata pkg.Py return p } -func newPackageForPackage(m parsedData, sources ...file.Location) pkg.Package { +func newPackageForPackage(resolver file.Resolver, m parsedData, sources ...file.Location) pkg.Package { + var licenseSet pkg.LicenseSet + + switch { + case m.LicenseExpression != "": + licenseSet = pkg.NewLicenseSet(pkg.NewLicensesFromLocation(m.LicenseLocation, m.LicenseExpression)...) + case m.Licenses != "": + licenseSet = pkg.NewLicenseSet(pkg.NewLicensesFromLocation(m.LicenseLocation, m.Licenses)...) + case m.LicenseLocation.Path() != "": + // If we have a license file then resolve and parse it + found, err := resolver.FilesByPath(m.LicenseLocation.Path()) + if err != nil { + log.WithFields("error", err).Tracef("unable to resolve python license path %s", m.LicenseLocation.Path()) + } + if len(found) > 0 { + metadataContents, err := resolver.FileContentsByLocation(found[0]) + if err == nil { + parsed, err := licenses.Parse(metadataContents, m.LicenseLocation) + if err != nil { + log.WithFields("error", err).Tracef("unable to parse a license from the file in %s", m.LicenseLocation.Path()) + } + if len(parsed) > 0 { + licenseSet = pkg.NewLicenseSet(parsed...) + } + } else { + log.WithFields("error", err).Tracef("unable to read file contents at %s", m.LicenseLocation.Path()) + } + } + } + p := pkg.Package{ Name: m.Name, Version: m.Version, PURL: packageURL(m.Name, m.Version, &m.PythonPackage), Locations: file.NewLocationSet(sources...), - Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromLocation(m.LicenseLocation, m.Licenses)...), + Licenses: licenseSet, Language: pkg.Python, Type: pkg.PythonPkg, Metadata: m.PythonPackage, diff --git a/syft/pkg/cataloger/python/parse_wheel_egg.go b/syft/pkg/cataloger/python/parse_wheel_egg.go index e98fe9de898..a50f9d55802 100644 --- a/syft/pkg/cataloger/python/parse_wheel_egg.go +++ b/syft/pkg/cataloger/python/parse_wheel_egg.go @@ -32,7 +32,7 @@ func parseWheelOrEgg(resolver file.Resolver, _ *generic.Environment, reader file return nil, nil, nil } - pkgs := []pkg.Package{newPackageForPackage(*pd, sources...)} + pkgs := []pkg.Package{newPackageForPackage(resolver, *pd, sources...)} return pkgs, nil, nil } diff --git a/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go b/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go index a52119995a4..4522dd45296 100644 --- a/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go +++ b/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go @@ -17,6 +17,8 @@ import ( type parsedData struct { Licenses string `mapstructure:"License"` + LicenseFile string `mapstructure:"LicenseFile"` + LicenseExpression string `mapstructure:"LicenseExpression"` LicenseLocation file.Location pkg.PythonPackage `mapstructure:",squash"` } @@ -80,8 +82,10 @@ func parseWheelOrEggMetadata(path string, reader io.Reader) (parsedData, error) // add additional metadata not stored in the egg/wheel metadata file pd.SitePackagesRootPath = determineSitePackagesRootPath(path) - if pd.Licenses != "" { + if pd.Licenses != "" || pd.LicenseExpression != "" { pd.LicenseLocation = file.NewLocation(path) + } else if pd.LicenseFile != "" { + pd.LicenseLocation = file.NewLocation(filepath.Join(filepath.Dir(path), pd.LicenseFile)) } return pd, nil diff --git a/syft/pkg/cataloger/python/parse_wheel_egg_metadata_test.go b/syft/pkg/cataloger/python/parse_wheel_egg_metadata_test.go index d3a9275122f..0c048b3a327 100644 --- a/syft/pkg/cataloger/python/parse_wheel_egg_metadata_test.go +++ b/syft/pkg/cataloger/python/parse_wheel_egg_metadata_test.go @@ -19,6 +19,8 @@ func TestParseWheelEggMetadata(t *testing.T) { Fixture: "test-fixtures/egg-info/PKG-INFO", ExpectedMetadata: parsedData{ "Apache 2.0", + "", + "", file.NewLocation("test-fixtures/egg-info/PKG-INFO"), pkg.PythonPackage{ Name: "requests", @@ -34,6 +36,8 @@ func TestParseWheelEggMetadata(t *testing.T) { Fixture: "test-fixtures/dist-info/METADATA", ExpectedMetadata: parsedData{ "BSD License", + "", + "", file.NewLocation("test-fixtures/dist-info/METADATA"), pkg.PythonPackage{ Name: "Pygments", @@ -134,6 +138,8 @@ func TestParseWheelEggMetadataInvalid(t *testing.T) { { Fixture: "test-fixtures/egg-info/PKG-INFO-INVALID", ExpectedMetadata: parsedData{ + "", + "", "", file.Location{}, pkg.PythonPackage{ diff --git a/syft/pkg/collection.go b/syft/pkg/collection.go index 10a3ed644b9..7ed028b2805 100644 --- a/syft/pkg/collection.go +++ b/syft/pkg/collection.go @@ -21,7 +21,7 @@ type Collection struct { // NewCollection returns a new empty Collection func NewCollection(pkgs ...Package) *Collection { - catalog := Collection{ + c := Collection{ byID: make(map[artifact.ID]Package), idsByName: make(map[string]orderedIDSet), idsByType: make(map[Type]orderedIDSet), @@ -29,10 +29,10 @@ func NewCollection(pkgs ...Package) *Collection { } for _, p := range pkgs { - catalog.Add(p) + c.Add(p) } - return &catalog + return &c } // PackageCount returns the total number of packages that have been added. @@ -97,32 +97,37 @@ func (c *Collection) packages(ids []artifact.ID) (result []Package) { return result } -// Add n packages to the catalog. +// Add n packages to the collection. func (c *Collection) Add(pkgs ...Package) { + for _, p := range pkgs { + c.add(p) + } +} + +// Add a package to the Collection. +func (c *Collection) add(p Package) { c.lock.Lock() defer c.lock.Unlock() - for _, p := range pkgs { - id := p.ID() - if id == "" { - log.Warnf("found package with empty ID while adding to the catalog: %+v", p) - p.SetID() - id = p.ID() - } + id := p.ID() + if id == "" { + log.Warnf("found package with empty ID while adding to the collection: %+v", p) + p.SetID() + id = p.ID() + } - if existing, exists := c.byID[id]; exists { - // there is already a package with this fingerprint merge the existing record with the new one - if err := existing.merge(p); err != nil { - log.Warnf("failed to merge packages: %+v", err) - } else { - c.byID[id] = existing - c.addPathsToIndex(p) - } - return + if existing, exists := c.byID[id]; exists { + // there is already a package with this fingerprint merge the existing record with the new one + if err := existing.merge(p); err != nil { + log.Warnf("failed to merge packages: %+v", err) + } else { + c.byID[id] = existing + c.addPathsToIndex(p) } - - c.addToIndex(p) + return } + + c.addToIndex(p) } func (c *Collection) addToIndex(p Package) { @@ -242,7 +247,7 @@ func (c *Collection) Enumerate(types ...Type) <-chan Package { defer close(channel) if c == nil { - // we should allow enumerating from a catalog that was never created (which will result in no packages enumerated) + // we should allow enumerating from a collection that was never created (which will result in no packages enumerated) return } diff --git a/syft/source/file_source.go b/syft/source/file_source.go index fcccd592914..c3a89668c86 100644 --- a/syft/source/file_source.go +++ b/syft/source/file_source.go @@ -189,7 +189,7 @@ func (s FileSource) FileResolver(_ Scope) (file.Resolver, error) { return fs.SkipDir } - if path.Base(p) != path.Base(s.config.Path) { + if filepath.Base(p) != filepath.Base(s.config.Path) { // we're in the root directory, but this is not the file we want to scan... // we should selectively skip this file (not the directory we're in). return fileresolver.ErrSkipPath