Skip to content

Commit

Permalink
Merge branch 'main' into feat/bitnami-cataloger
Browse files Browse the repository at this point in the history
  • Loading branch information
juan131 committed Jan 31, 2025
2 parents 3c2b313 + bdf6804 commit b917f5b
Show file tree
Hide file tree
Showing 29 changed files with 3,366 additions and 90 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/codeql-analysis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ jobs:

# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@dd196fa9ce80b6bacc74ca1c32bd5b0ba22efca7 #v3.28.3
uses: github/codeql-action/init@dd746615b3b9d728a6a37ca2045b68ca76d4841a #v3.28.8
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
Expand All @@ -56,7 +56,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@dd196fa9ce80b6bacc74ca1c32bd5b0ba22efca7 #v3.28.3
uses: github/codeql-action/autobuild@dd746615b3b9d728a6a37ca2045b68ca76d4841a #v3.28.8

# ℹ️ Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
Expand All @@ -70,4 +70,4 @@ jobs:
# make release

- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@dd196fa9ce80b6bacc74ca1c32bd5b0ba22efca7 #v3.28.3
uses: github/codeql-action/analyze@dd746615b3b9d728a6a37ca2045b68ca76d4841a #v3.28.8
2 changes: 1 addition & 1 deletion .github/workflows/release.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ jobs:
# for updating brew formula in anchore/homebrew-syft
GITHUB_BREW_TOKEN: ${{ secrets.ANCHOREOPS_GITHUB_OSS_WRITE_TOKEN }}

- uses: anchore/sbom-action@df80a981bc6edbc4e220a492d3cbe9f5547a6e75 #v0.17.9
- uses: anchore/sbom-action@f325610c9f50a54015d37c8d16cb3b0e2c8f4de0 #v0.18.0
continue-on-error: true
with:
file: go.mod
Expand Down
4 changes: 2 additions & 2 deletions cmd/syft/internal/options/pkg.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@ var _ interface {
} = (*packageConfig)(nil)

func (o *packageConfig) DescribeFields(descriptions clio.FieldDescriptionSet) {
descriptions.Add(&o.SearchUnindexedArchives, `search within archives that do contain a file index to search against (zip)
descriptions.Add(&o.SearchIndexedArchives, `search within archives that do contain a file index to search against (zip)
note: for now this only applies to the java package cataloger`)
descriptions.Add(&o.SearchIndexedArchives, `search within archives that do not contain a file index to search against (tar, tar.gz, tar.bz2, etc)
descriptions.Add(&o.SearchUnindexedArchives, `search within archives that do not contain a file index to search against (tar, tar.gz, tar.bz2, etc)
note: enabling this may result in a performance impact since all discovered compressed tars will be decompressed
note: for now this only applies to the java package cataloger`)
descriptions.Add(&o.ExcludeBinaryOverlapByOwnership, `allows users to exclude synthetic binary packages from the sbom
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ require (
// we are hinting brotli to latest due to warning when installing archiver v3:
// go: warning: github.com/andybalholm/[email protected]: retracted by module author: occasional panics and data corruption
github.com/aquasecurity/go-pep440-version v0.0.1
github.com/bmatcuk/doublestar/v4 v4.8.0
github.com/bmatcuk/doublestar/v4 v4.8.1
github.com/charmbracelet/bubbles v0.20.0
github.com/charmbracelet/bubbletea v1.2.4
github.com/charmbracelet/lipgloss v1.0.0
Expand All @@ -34,7 +34,7 @@ require (
github.com/elliotchance/phpserialize v1.4.0
github.com/facebookincubator/nvdtools v0.1.5
github.com/github/go-spdx/v2 v2.3.2
github.com/gkampitakis/go-snaps v0.5.8
github.com/gkampitakis/go-snaps v0.5.9
github.com/go-git/go-billy/v5 v5.6.2
github.com/go-git/go-git/v5 v5.13.2
github.com/go-test/deep v1.1.1
Expand Down
8 changes: 4 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -151,8 +151,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bitnami/go-version v0.0.0-20240404145124-6814fce176da h1:UWjloedS0Zb6li44Xtg5gxIpuSnl8jQDhZtybhTYGHo=
github.com/bitnami/go-version v0.0.0-20240404145124-6814fce176da/go.mod h1:9iglf1GG4oNRJ39bZ5AZrjgAFD2RwQbXw6Qf7Cs47wo=
github.com/bmatcuk/doublestar/v4 v4.8.0 h1:DSXtrypQddoug1459viM9X9D3dp1Z7993fw36I2kNcQ=
github.com/bmatcuk/doublestar/v4 v4.8.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/bmatcuk/doublestar/v4 v4.8.1 h1:54Bopc5c2cAvhLRAzqOGCYHYyhcDHsFF4wWIR5wKP38=
github.com/bmatcuk/doublestar/v4 v4.8.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M=
github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0=
github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
Expand Down Expand Up @@ -299,8 +299,8 @@ github.com/gkampitakis/ciinfo v0.3.1 h1:lzjbemlGI4Q+XimPg64ss89x8Mf3xihJqy/0Mgag
github.com/gkampitakis/ciinfo v0.3.1/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo=
github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M=
github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk=
github.com/gkampitakis/go-snaps v0.5.8 h1:BB4ihcyXgJEVO/Pj/P+4bs7pFzsLcEjsfU2+mFdJh1c=
github.com/gkampitakis/go-snaps v0.5.8/go.mod h1:PcKmy8q5Se7p48ywpogN5Td13reipz1Iivah4wrTIvY=
github.com/gkampitakis/go-snaps v0.5.9 h1:jO2Fe2q2fhLNcMQjYh/EAGUzrZiIdwD/CkM8+QSs5cE=
github.com/gkampitakis/go-snaps v0.5.9/go.mod h1:PcKmy8q5Se7p48ywpogN5Td13reipz1Iivah4wrTIvY=
github.com/glebarez/go-sqlite v1.20.3 h1:89BkqGOXR9oRmG58ZrzgoY/Fhy5x0M+/WV48U5zVrZ4=
github.com/glebarez/go-sqlite v1.20.3/go.mod h1:u3N6D/wftiAzIOJtZl6BmedqxmmkDfH3q+ihjqxC9u0=
github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
Expand Down
2 changes: 1 addition & 1 deletion internal/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,5 @@ package internal
const (
// JSONSchemaVersion is the current schema version output by the JSON encoder
// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
JSONSchemaVersion = "16.0.21"
JSONSchemaVersion = "16.0.22"
)
23 changes: 11 additions & 12 deletions internal/licenses/scanner.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
const coverageThreshold = 75 // determined by experimentation

type Scanner interface {
IdentifyLicenseIDs(context.Context, io.Reader) ([]string, error)
IdentifyLicenseIDs(context.Context, io.Reader) ([]string, []byte, error)
}

var _ Scanner = (*scanner)(nil)
Expand All @@ -35,34 +35,33 @@ func NewDefaultScanner() Scanner {
}
}

// TestingOnlyScanner returns a scanner that uses the built-in license scanner from the licensecheck package.
// THIS IS ONLY MEANT FOR TEST CODE, NOT PRODUCTION CODE.
func TestingOnlyScanner() Scanner {
return &scanner{
coverageThreshold: coverageThreshold,
scanner: licensecheck.Scan,
func NewScanner(scan func([]byte) licensecheck.Coverage, coverage float64) Scanner {
return scanner{
coverageThreshold: coverage,
scanner: scan,
}
}

func (s scanner) IdentifyLicenseIDs(_ context.Context, reader io.Reader) ([]string, error) {
func (s scanner) IdentifyLicenseIDs(_ context.Context, reader io.Reader) ([]string, []byte, error) {
if s.scanner == nil {
return nil, nil
return nil, nil, nil
}

content, err := io.ReadAll(reader)
if err != nil {
return nil, err
return nil, nil, err
}

cov := s.scanner(content)
if cov.Percent < s.coverageThreshold {
// unknown or no licenses here?
return nil, nil
// => return binary content
return nil, content, nil
}

var ids []string
for _, m := range cov.Match {
ids = append(ids, m.ID)
}
return ids, nil
return ids, nil, nil
}
83 changes: 83 additions & 0 deletions internal/licenses/scanner_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package licenses

import (
"bytes"
"context"
"os"
"testing"

"github.com/google/licensecheck"
"github.com/stretchr/testify/require"
)

func TestIdentifyLicenseIDs(t *testing.T) {
type expectation struct {
yieldError bool
ids []string
content []byte
}
tests := []struct {
name string
in string
expected expectation
}{
{
name: "apache license 2.0",
in: `test-fixtures/apache-license-2.0`,
expected: expectation{
yieldError: false,
ids: []string{"Apache-2.0"},
content: []byte{},
},
},
{
name: "custom license",
in: "test-fixtures/nvidia-software-and-cuda-supplement",
expected: expectation{
yieldError: false,
ids: []string{},
content: mustOpen("test-fixtures/nvidia-software-and-cuda-supplement"),
},
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
content, err := os.ReadFile(test.in)
require.NoError(t, err)
ids, content, err := testScanner().IdentifyLicenseIDs(context.TODO(), bytes.NewReader(content))
if test.expected.yieldError {
require.Error(t, err)
} else {
require.NoError(t, err)

require.Len(t, ids, len(test.expected.ids))
require.Len(t, content, len(test.expected.content))

if len(test.expected.ids) > 0 {
require.Equal(t, ids, test.expected.ids)
}

if len(test.expected.content) > 0 {
require.Equal(t, content, test.expected.content)
}
}
})
}
}

func testScanner() Scanner {
return &scanner{
coverageThreshold: coverageThreshold,
scanner: licensecheck.Scan,
}
}

func mustOpen(fixture string) []byte {
content, err := os.ReadFile(fixture)
if err != nil {
panic(err)
}

return content
}
38 changes: 34 additions & 4 deletions internal/licenses/search.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,54 @@ package licenses

import (
"context"
"crypto/sha256"
"fmt"
"strings"

"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/license"
"github.com/anchore/syft/syft/pkg"
)

const (
unknownLicenseType = "UNKNOWN"
UnknownLicensePrefix = unknownLicenseType + "_"
)

func getCustomLicenseContentHash(contents []byte) string {
hash := sha256.Sum256(contents)
return fmt.Sprintf("%x", hash[:])
}

// Search scans the contents of a license file to attempt to determine the type of license it is
func Search(ctx context.Context, scanner Scanner, reader file.LocationReadCloser) (licenses []pkg.License, err error) {
licenses = make([]pkg.License, 0)

ids, err := scanner.IdentifyLicenseIDs(ctx, reader)
ids, content, err := scanner.IdentifyLicenseIDs(ctx, reader)
if err != nil {
return nil, err
}

for _, id := range ids {
lic := pkg.NewLicenseFromLocations(id, reader.Location)
lic.Type = license.Concluded
// IdentifyLicenseIDs can only return a list of ID or content
// These return values are mutually exclusive.
// If the scanner threshold for matching scores < 75% then we return the license full content
if len(ids) > 0 {
for _, id := range ids {
lic := pkg.NewLicenseFromLocations(id, reader.Location)
lic.Type = license.Concluded

licenses = append(licenses, lic)
}
} else if len(content) > 0 {
// harmonize line endings to unix compatible first:
// 1. \r\n => \n (Windows => UNIX)
// 2. \r => \n (Macintosh => UNIX)
content = []byte(strings.ReplaceAll(strings.ReplaceAll(string(content), "\r\n", "\n"), "\r", "\n"))

lic := pkg.NewLicenseFromLocations(unknownLicenseType, reader.Location)
lic.SPDXExpression = UnknownLicensePrefix + getCustomLicenseContentHash(content)
lic.Contents = string(content)
lic.Type = license.Declared

licenses = append(licenses, lic)
}
Expand Down
95 changes: 95 additions & 0 deletions internal/licenses/search_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
package licenses

import (
"bytes"
"context"
"io"
"os"
"testing"

"github.com/stretchr/testify/require"

"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
)

type bytesReadCloser struct {
bytes.Buffer
}

func (brc *bytesReadCloser) Close() error {
return nil
}

func newBytesReadCloser(data []byte) *bytesReadCloser {
return &bytesReadCloser{
Buffer: *bytes.NewBuffer(data),
}
}

func TestSearch(t *testing.T) {
type expectation struct {
yieldError bool
licenses []pkg.License
}
testLocation := file.NewLocation("LICENSE")
tests := []struct {
name string
in string
expected expectation
}{
{
name: "apache license 2.0",
in: "test-fixtures/apache-license-2.0",
expected: expectation{
yieldError: false,
licenses: []pkg.License{
{
Value: "Apache-2.0",
SPDXExpression: "Apache-2.0",
Type: "concluded",
URLs: nil,
Locations: file.NewLocationSet(testLocation),
Contents: "",
},
},
},
},
{
name: "custom license",
in: "test-fixtures/nvidia-software-and-cuda-supplement",
expected: expectation{
yieldError: false,
licenses: []pkg.License{
{
Value: "UNKNOWN",
SPDXExpression: "UNKNOWN_eebcea3ab1d1a28e671de90119ffcfb35fe86951e4af1b17af52b7a82fcf7d0a",
Type: "declared",
URLs: nil,
Locations: file.NewLocationSet(testLocation),
Contents: string(mustOpen("test-fixtures/nvidia-software-and-cuda-supplement")),
},
},
},
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
content, err := os.ReadFile(test.in)
require.NoError(t, err)
result, err := Search(context.TODO(), testScanner(), file.NewLocationReadCloser(file.NewLocation("LICENSE"), io.NopCloser(bytes.NewReader(content))))
if test.expected.yieldError {
require.Error(t, err)
} else {
require.NoError(t, err)

require.Len(t, result, len(test.expected.licenses))

if len(test.expected.licenses) > 0 {
require.Equal(t, test.expected.licenses, result)
}
}
})
}
}
Loading

0 comments on commit b917f5b

Please sign in to comment.