From 7223239c83f42fb6b59502056e4273b17d8d6c0b Mon Sep 17 00:00:00 2001 From: wdower <57142072+wdower@users.noreply.github.com> Date: Mon, 17 Feb 2025 16:54:31 -0500 Subject: [PATCH] allowing trufflehog mapper to handle duplicate findings and ndjson format (#6626) * allowing trufflehog mapper to handle duplicate findings and ndjson format Signed-off-by: Will * lint Signed-off-by: Will * fixing fingerprints to handle ndjson Signed-off-by: Will --------- Signed-off-by: Will Co-authored-by: Amndeep Singh Mann --- .../sample_input_report/trufflehog_dup.ndjson | 6 ++ .../trufflehog-ndjson-dup-hdf.json | 94 +++++++++++++++++++ libs/hdf-converters/src/trufflehog-mapper.ts | 16 +++- .../src/utils/fingerprinting.ts | 7 ++ .../mappers/forward/trufflehog_mapper.spec.ts | 30 ++++++ 5 files changed, 149 insertions(+), 4 deletions(-) create mode 100644 libs/hdf-converters/sample_jsons/trufflehog_mapper/sample_input_report/trufflehog_dup.ndjson create mode 100644 libs/hdf-converters/sample_jsons/trufflehog_mapper/trufflehog-ndjson-dup-hdf.json diff --git a/libs/hdf-converters/sample_jsons/trufflehog_mapper/sample_input_report/trufflehog_dup.ndjson b/libs/hdf-converters/sample_jsons/trufflehog_mapper/sample_input_report/trufflehog_dup.ndjson new file mode 100644 index 0000000000..213c073b75 --- /dev/null +++ b/libs/hdf-converters/sample_jsons/trufflehog_mapper/sample_input_report/trufflehog_dup.ndjson @@ -0,0 +1,6 @@ +{"SourceMetadata":{"Data":{"Filesystem":{"file":".git/config","line":13}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":17,"DetectorName":"URI","DetectorDescription":"This detector identifies URLs with embedded credentials, which can be used to access web resources without explicit user interaction.","DecoderName":"PLAIN","Verified":false,"VerificationError":"dialing local IP addresses is not allowed","VerificationFromCache":false,"Raw":"https://gitlab-ci-token:>@gitlab.my_domain.dev","RawV2":"https://gitlab-ci-token:@gitlab.my_domain.dev/foo/bar.git","Redacted":"https://gitlab-ci-token:********@gitlab.my_domain.dev","ExtraData":null,"StructuredData":null} +{"SourceMetadata":{"Data":{"Filesystem":{"file":"github.com/jackc/pgx/v5/pgxpool/pool.go","line":297}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":968,"DetectorName":"Postgres","DetectorDescription":"Postgres connection string containing credentials","DecoderName":"PLAIN","Verified":false,"VerificationError":"lookup pg.example.com on 10.96.0.10:53: server misbehaving","VerificationFromCache":false,"Raw":"postgres://jack:secret@pg.example.com:5432","RawV2":"postgres://jack:secret@pg.example.com:5432","Redacted":"","ExtraData":{"sslmode":"verify-ca"},"StructuredData":null} +{"SourceMetadata":{"Data":{"Filesystem":{"file":"github.com/jackc/pgx/v5/pgconn/config.go","line":1}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":968,"DetectorName":"Postgres","DetectorDescription":"Postgres connection string containing credentials","DecoderName":"PLAIN","Verified":false,"VerificationError":"lookup foo.example.com:5432,bar.example.com:5432: no such host","VerificationFromCache":false,"Raw":"postgres://jack:secret@foo.example.com:5432,bar.example.com:5432:5432","RawV2":"postgres://jack:secret@foo.example.com:5432,bar.example.com:5432:5432","Redacted":"","ExtraData":{"sslmode":"\u003cunset\u003e"},"StructuredData":null} +{"SourceMetadata":{"Data":{"Filesystem":{"file":"github.com/jackc/pgx/v5/pgconn/config.go","line":171}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":968,"DetectorName":"Postgres","DetectorDescription":"Postgres connection string containing credentials","DecoderName":"PLAIN","Verified":false,"VerificationError":"lookup pg.example.com on 10.96.0.10:53: server misbehaving","VerificationFromCache":false,"Raw":"postgres://jack:secret@pg.example.com:5432","RawV2":"postgres://jack:secret@pg.example.com:5432","Redacted":"","ExtraData":{"sslmode":"verify-ca"},"StructuredData":null} +{"SourceMetadata":{"Data":{"Filesystem":{"file":"github.com/pressly/goose/v3/README.md","line":93}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":968,"DetectorName":"Postgres","DetectorDescription":"Postgres connection string containing credentials","DecoderName":"PLAIN","Verified":false,"VerificationError":"lookup qwerty.us-east-1.redshift.amazonaws.com on 10.96.0.10:53: server misbehaving","VerificationFromCache":false,"Raw":"postgres://user:password@qwerty.us-east-1.redshift.amazonaws.com:5439","RawV2":"postgres://user:password@qwerty.us-east-1.redshift.amazonaws.com:5439","Redacted":"","ExtraData":{"sslmode":"\u003cunset\u003e"},"StructuredData":null} +{"SourceMetadata":{"Data":{"Filesystem":{"file":"github.com/pressly/goose/v3/README.md","line":93}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":968,"DetectorName":"Postgres","DetectorDescription":"Postgres connection string containing credentials","DecoderName":"PLAIN","Verified":false,"VerificationError":"lookup qwerty.us-east-1.redshift.amazonaws.com on 10.96.0.10:53: server misbehaving","VerificationFromCache":false,"Raw":"postgres://user:password@qwerty.us-east-1.redshift.amazonaws.com:5439","RawV2":"postgres://user:password@qwerty.us-east-1.redshift.amazonaws.com:5439","Redacted":"","ExtraData":{"sslmode":"\u003cunset\u003e"},"StructuredData":null} diff --git a/libs/hdf-converters/sample_jsons/trufflehog_mapper/trufflehog-ndjson-dup-hdf.json b/libs/hdf-converters/sample_jsons/trufflehog_mapper/trufflehog-ndjson-dup-hdf.json new file mode 100644 index 0000000000..380228502d --- /dev/null +++ b/libs/hdf-converters/sample_jsons/trufflehog_mapper/trufflehog-ndjson-dup-hdf.json @@ -0,0 +1,94 @@ +{ + "platform": { + "name": "Heimdall Tools", + "release": "2.11.2" + }, + "version": "2.11.2", + "statistics": {}, + "profiles": [ + { + "name": "Source ID: 1, Source Name: trufflehog - filesystem", + "title": "trufflehog - filesystem", + "supports": [], + "attributes": [], + "groups": [], + "status": "loaded", + "controls": [ + { + "tags": { + "nist": [ + "IA-5(7)" + ], + "cci": [ + "CCI-004069", + "CCI-000202", + "CCI-000203", + "CCI-002367" + ], + "severity": "medium" + }, + "refs": [], + "source_location": {}, + "title": "Found URI secret using PLAIN decoder", + "id": "URI PLAIN", + "impact": 0.5, + "results": [ + { + "status": "failed", + "code_desc": "{\n \"Data\": {\n \"Filesystem\": {\n \"file\": \".git/config\",\n \"line\": 13\n }\n }\n}", + "message": "{\n \"Verified\": false,\n \"VerificationError\": \"dialing local IP addresses is not allowed\",\n \"Raw\": \"https://gitlab-ci-token:>@gitlab.my_domain.dev\",\n \"RawV2\": \"https://gitlab-ci-token:@gitlab.my_domain.dev/foo/bar.git\",\n \"Redacted\": \"https://gitlab-ci-token:********@gitlab.my_domain.dev\"\n}", + "start_time": "" + } + ] + }, + { + "tags": { + "nist": [ + "IA-5(7)" + ], + "cci": [ + "CCI-004069", + "CCI-000202", + "CCI-000203", + "CCI-002367" + ], + "severity": "medium" + }, + "refs": [], + "source_location": {}, + "title": "Found Postgres secret using PLAIN decoder", + "id": "Postgres PLAIN", + "impact": 0.5, + "results": [ + { + "status": "failed", + "code_desc": "{\n \"Data\": {\n \"Filesystem\": {\n \"file\": \"github.com/jackc/pgx/v5/pgxpool/pool.go\",\n \"line\": 297\n }\n }\n}", + "message": "{\n \"Verified\": false,\n \"VerificationError\": \"lookup pg.example.com on 10.96.0.10:53: server misbehaving\",\n \"Raw\": \"postgres://jack:secret@pg.example.com:5432\",\n \"RawV2\": \"postgres://jack:secret@pg.example.com:5432\",\n \"ExtraData\": {\n \"sslmode\": \"verify-ca\"\n }\n}", + "start_time": "" + }, + { + "status": "failed", + "code_desc": "{\n \"Data\": {\n \"Filesystem\": {\n \"file\": \"github.com/jackc/pgx/v5/pgconn/config.go\",\n \"line\": 1\n }\n }\n}", + "message": "{\n \"Verified\": false,\n \"VerificationError\": \"lookup foo.example.com:5432,bar.example.com:5432: no such host\",\n \"Raw\": \"postgres://jack:secret@foo.example.com:5432,bar.example.com:5432:5432\",\n \"RawV2\": \"postgres://jack:secret@foo.example.com:5432,bar.example.com:5432:5432\",\n \"ExtraData\": {\n \"sslmode\": \"\"\n }\n}", + "start_time": "" + }, + { + "status": "failed", + "code_desc": "{\n \"Data\": {\n \"Filesystem\": {\n \"file\": \"github.com/jackc/pgx/v5/pgconn/config.go\",\n \"line\": 171\n }\n }\n}", + "message": "{\n \"Verified\": false,\n \"VerificationError\": \"lookup pg.example.com on 10.96.0.10:53: server misbehaving\",\n \"Raw\": \"postgres://jack:secret@pg.example.com:5432\",\n \"RawV2\": \"postgres://jack:secret@pg.example.com:5432\",\n \"ExtraData\": {\n \"sslmode\": \"verify-ca\"\n }\n}", + "start_time": "" + }, + { + "status": "failed", + "code_desc": "{\n \"Data\": {\n \"Filesystem\": {\n \"file\": \"github.com/pressly/goose/v3/README.md\",\n \"line\": 93\n }\n }\n}", + "message": "{\n \"Verified\": false,\n \"VerificationError\": \"lookup qwerty.us-east-1.redshift.amazonaws.com on 10.96.0.10:53: server misbehaving\",\n \"Raw\": \"postgres://user:password@qwerty.us-east-1.redshift.amazonaws.com:5439\",\n \"RawV2\": \"postgres://user:password@qwerty.us-east-1.redshift.amazonaws.com:5439\",\n \"ExtraData\": {\n \"sslmode\": \"\"\n }\n}", + "start_time": "" + } + ] + } + ], + "sha256": "ba8c9ae68b0c4d07e06e0821811f716812ea379467c1d57d3117741d471bd0af" + } + ], + "passthrough": {} +} \ No newline at end of file diff --git a/libs/hdf-converters/src/trufflehog-mapper.ts b/libs/hdf-converters/src/trufflehog-mapper.ts index 88099434d7..4871d96a58 100644 --- a/libs/hdf-converters/src/trufflehog-mapper.ts +++ b/libs/hdf-converters/src/trufflehog-mapper.ts @@ -7,12 +7,20 @@ export class TrufflehogResults { data: Record; withRaw: boolean; constructor(trufflehogJson: string, withRaw = false) { - this.data = JSON.parse(trufflehogJson); + let parsedData = {}; + try { + parsedData = JSON.parse(trufflehogJson.trim()); + } catch (e) { + parsedData = trufflehogJson + .trim() + .split('\n') + .map((line) => JSON.parse(line.trim())); + } this.withRaw = withRaw; - if (_.isArray(this.data)) { - this.data = {wrapper: this.data}; + if (_.isArray(parsedData)) { + this.data = {wrapper: parsedData}; } else { - this.data = {wrapper: [this.data]}; + this.data = {wrapper: [parsedData]}; } } diff --git a/libs/hdf-converters/src/utils/fingerprinting.ts b/libs/hdf-converters/src/utils/fingerprinting.ts index a380bf20de..edbf722f33 100644 --- a/libs/hdf-converters/src/utils/fingerprinting.ts +++ b/libs/hdf-converters/src/utils/fingerprinting.ts @@ -166,6 +166,13 @@ export function fingerprint(guessOptions: { splitLines[0].includes('Severity') ) { return INPUT_TYPES.PRISMA; + } else if ( + splitLines[0].includes('SourceName') && + splitLines[0].includes('DetectorType') && + splitLines[0].includes('DetectorName') && + splitLines[0].includes('DecoderName') + ) { + return INPUT_TYPES.TRUFFLEHOG; } else if ( guessOptions.data.indexOf('veracode') !== -1 && guessOptions.data.indexOf('detailedreport') !== -1 diff --git a/libs/hdf-converters/test/mappers/forward/trufflehog_mapper.spec.ts b/libs/hdf-converters/test/mappers/forward/trufflehog_mapper.spec.ts index 8fef6f7e32..b31bd92452 100644 --- a/libs/hdf-converters/test/mappers/forward/trufflehog_mapper.spec.ts +++ b/libs/hdf-converters/test/mappers/forward/trufflehog_mapper.spec.ts @@ -238,3 +238,33 @@ describe('trufflehog_example_mapper', () => { ); }); }); + +describe('trufflehog_dup_ndjson', () => { + it('Successfully converts trufflehog in ndjson format with duplicate findings', () => { + const mapper = new TrufflehogResults( + fs.readFileSync( + 'sample_jsons/trufflehog_mapper/sample_input_report/trufflehog_dup.ndjson', + {encoding: 'utf-8'} + ), + false + ); + + // fs.writeFileSync( + // 'sample_jsons/trufflehog_mapper/trufflehog-ndjson-dup-hdf.json', + // JSON.stringify(mapper.toHdf(), null, 2) + // ); + + expect(omitVersions(mapper.toHdf())).toEqual( + omitVersions( + JSON.parse( + fs.readFileSync( + 'sample_jsons/trufflehog_mapper/trufflehog-ndjson-dup-hdf.json', + { + encoding: 'utf-8' + } + ) + ) + ) + ); + }); +});