allowing trufflehog mapper to handle duplicate findings and ndjson fo…

…rmat

Signed-off-by: Will <[email protected]>

libs/hdf-converters/sample_jsons/trufflehog_mapper/sample_input_report/trufflehog_dup.ndjson

@@ -0,0 +1,6 @@
+{"SourceMetadata":{"Data":{"Filesystem":{"file":".git/config","line":13}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":17,"DetectorName":"URI","DetectorDescription":"This detector identifies URLs with embedded credentials, which can be used to access web resources without explicit user interaction.","DecoderName":"PLAIN","Verified":false,"VerificationError":"dialing local IP addresses is not allowed","VerificationFromCache":false,"Raw":"https://gitlab-ci-token:<token>>@gitlab.my_domain.dev","RawV2":"https://gitlab-ci-token:<token>@gitlab.my_domain.dev/foo/bar.git","Redacted":"https://gitlab-ci-token:********@gitlab.my_domain.dev","ExtraData":null,"StructuredData":null}
+{"SourceMetadata":{"Data":{"Filesystem":{"file":"github.com/jackc/pgx/v5/pgxpool/pool.go","line":297}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":968,"DetectorName":"Postgres","DetectorDescription":"Postgres connection string containing credentials","DecoderName":"PLAIN","Verified":false,"VerificationError":"lookup pg.example.com on 10.96.0.10:53: server misbehaving","VerificationFromCache":false,"Raw":"postgres://jack:[email protected]:5432","RawV2":"postgres://jack:[email protected]:5432","Redacted":"","ExtraData":{"sslmode":"verify-ca"},"StructuredData":null}
+{"SourceMetadata":{"Data":{"Filesystem":{"file":"github.com/jackc/pgx/v5/pgconn/config.go","line":1}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":968,"DetectorName":"Postgres","DetectorDescription":"Postgres connection string containing credentials","DecoderName":"PLAIN","Verified":false,"VerificationError":"lookup foo.example.com:5432,bar.example.com:5432: no such host","VerificationFromCache":false,"Raw":"postgres://jack:[email protected]:5432,bar.example.com:5432:5432","RawV2":"postgres://jack:[email protected]:5432,bar.example.com:5432:5432","Redacted":"","ExtraData":{"sslmode":"\u003cunset\u003e"},"StructuredData":null}
+{"SourceMetadata":{"Data":{"Filesystem":{"file":"github.com/jackc/pgx/v5/pgconn/config.go","line":171}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":968,"DetectorName":"Postgres","DetectorDescription":"Postgres connection string containing credentials","DecoderName":"PLAIN","Verified":false,"VerificationError":"lookup pg.example.com on 10.96.0.10:53: server misbehaving","VerificationFromCache":false,"Raw":"postgres://jack:[email protected]:5432","RawV2":"postgres://jack:[email protected]:5432","Redacted":"","ExtraData":{"sslmode":"verify-ca"},"StructuredData":null}
+{"SourceMetadata":{"Data":{"Filesystem":{"file":"github.com/pressly/goose/v3/README.md","line":93}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":968,"DetectorName":"Postgres","DetectorDescription":"Postgres connection string containing credentials","DecoderName":"PLAIN","Verified":false,"VerificationError":"lookup qwerty.us-east-1.redshift.amazonaws.com on 10.96.0.10:53: server misbehaving","VerificationFromCache":false,"Raw":"postgres://user:[email protected]:5439","RawV2":"postgres://user:[email protected]:5439","Redacted":"","ExtraData":{"sslmode":"\u003cunset\u003e"},"StructuredData":null}
+{"SourceMetadata":{"Data":{"Filesystem":{"file":"github.com/pressly/goose/v3/README.md","line":93}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":968,"DetectorName":"Postgres","DetectorDescription":"Postgres connection string containing credentials","DecoderName":"PLAIN","Verified":false,"VerificationError":"lookup qwerty.us-east-1.redshift.amazonaws.com on 10.96.0.10:53: server misbehaving","VerificationFromCache":false,"Raw":"postgres://user:[email protected]:5439","RawV2":"postgres://user:[email protected]:5439","Redacted":"","ExtraData":{"sslmode":"\u003cunset\u003e"},"StructuredData":null}

libs/hdf-converters/sample_jsons/trufflehog_mapper/trufflehog-ndjson-dup-hdf.json

@@ -0,0 +1,94 @@
+{
+  "platform": {
+    "name": "Heimdall Tools",
+    "release": "2.11.2"
+  },
+  "version": "2.11.2",
+  "statistics": {},
+  "profiles": [
+    {
+      "name": "Source ID: 1, Source Name: trufflehog - filesystem",
+      "title": "trufflehog - filesystem",
+      "supports": [],
+      "attributes": [],
+      "groups": [],
+      "status": "loaded",
+      "controls": [
+        {
+          "tags": {
+            "nist": [
+              "IA-5(7)"
+            ],
+            "cci": [
+              "CCI-004069",
+              "CCI-000202",
+              "CCI-000203",
+              "CCI-002367"
+            ],
+            "severity": "medium"
+          },
+          "refs": [],
+          "source_location": {},
+          "title": "Found URI secret using PLAIN decoder",
+          "id": "URI PLAIN",
+          "impact": 0.5,
+          "results": [
+            {
+              "status": "failed",
+              "code_desc": "{\n  \"Data\": {\n    \"Filesystem\": {\n      \"file\": \".git/config\",\n      \"line\": 13\n    }\n  }\n}",
+              "message": "{\n  \"Verified\": false,\n  \"VerificationError\": \"dialing local IP addresses is not allowed\",\n  \"Raw\": \"https://gitlab-ci-token:<token>>@gitlab.my_domain.dev\",\n  \"RawV2\": \"https://gitlab-ci-token:<token>@gitlab.my_domain.dev/foo/bar.git\",\n  \"Redacted\": \"https://gitlab-ci-token:********@gitlab.my_domain.dev\"\n}",
+              "start_time": ""
+            }
+          ]
+        },
+        {
+          "tags": {
+            "nist": [
+              "IA-5(7)"
+            ],
+            "cci": [
+              "CCI-004069",
+              "CCI-000202",
+              "CCI-000203",
+              "CCI-002367"
+            ],
+            "severity": "medium"
+          },
+          "refs": [],
+          "source_location": {},
+          "title": "Found Postgres secret using PLAIN decoder",
+          "id": "Postgres PLAIN",
+          "impact": 0.5,
+          "results": [
+            {
+              "status": "failed",
+              "code_desc": "{\n  \"Data\": {\n    \"Filesystem\": {\n      \"file\": \"github.com/jackc/pgx/v5/pgxpool/pool.go\",\n      \"line\": 297\n    }\n  }\n}",
+              "message": "{\n  \"Verified\": false,\n  \"VerificationError\": \"lookup pg.example.com on 10.96.0.10:53: server misbehaving\",\n  \"Raw\": \"postgres://jack:[email protected]:5432\",\n  \"RawV2\": \"postgres://jack:[email protected]:5432\",\n  \"ExtraData\": {\n    \"sslmode\": \"verify-ca\"\n  }\n}",
+              "start_time": ""
+            },
+            {
+              "status": "failed",
+              "code_desc": "{\n  \"Data\": {\n    \"Filesystem\": {\n      \"file\": \"github.com/jackc/pgx/v5/pgconn/config.go\",\n      \"line\": 1\n    }\n  }\n}",
+              "message": "{\n  \"Verified\": false,\n  \"VerificationError\": \"lookup foo.example.com:5432,bar.example.com:5432: no such host\",\n  \"Raw\": \"postgres://jack:[email protected]:5432,bar.example.com:5432:5432\",\n  \"RawV2\": \"postgres://jack:[email protected]:5432,bar.example.com:5432:5432\",\n  \"ExtraData\": {\n    \"sslmode\": \"<unset>\"\n  }\n}",
+              "start_time": ""
+            },
+            {
+              "status": "failed",
+              "code_desc": "{\n  \"Data\": {\n    \"Filesystem\": {\n      \"file\": \"github.com/jackc/pgx/v5/pgconn/config.go\",\n      \"line\": 171\n    }\n  }\n}",
+              "message": "{\n  \"Verified\": false,\n  \"VerificationError\": \"lookup pg.example.com on 10.96.0.10:53: server misbehaving\",\n  \"Raw\": \"postgres://jack:[email protected]:5432\",\n  \"RawV2\": \"postgres://jack:[email protected]:5432\",\n  \"ExtraData\": {\n    \"sslmode\": \"verify-ca\"\n  }\n}",
+              "start_time": ""
+            },
+            {
+              "status": "failed",
+              "code_desc": "{\n  \"Data\": {\n    \"Filesystem\": {\n      \"file\": \"github.com/pressly/goose/v3/README.md\",\n      \"line\": 93\n    }\n  }\n}",
+              "message": "{\n  \"Verified\": false,\n  \"VerificationError\": \"lookup qwerty.us-east-1.redshift.amazonaws.com on 10.96.0.10:53: server misbehaving\",\n  \"Raw\": \"postgres://user:[email protected]:5439\",\n  \"RawV2\": \"postgres://user:[email protected]:5439\",\n  \"ExtraData\": {\n    \"sslmode\": \"<unset>\"\n  }\n}",
+              "start_time": ""
+            }
+          ]
+        }
+      ],
+      "sha256": "ba8c9ae68b0c4d07e06e0821811f716812ea379467c1d57d3117741d471bd0af"
+    }
+  ],
+  "passthrough": {}
+}

libs/hdf-converters/src/trufflehog-mapper.ts

@@ -7,12 +7,18 @@ export class TrufflehogResults {
   data: Record<string, unknown>;
   withRaw: boolean;
   constructor(trufflehogJson: string, withRaw = false) {
-    this.data = JSON.parse(trufflehogJson);
+    let parsedData = {};
+    try {
+      parsedData = JSON.parse(trufflehogJson.trim());
+    }
+    catch (e) {
+      parsedData = trufflehogJson.trim().split("\n").map((line) => JSON.parse(line.trim()));
+    }
     this.withRaw = withRaw;
-    if (_.isArray(this.data)) {
-      this.data = {wrapper: this.data};
+    if (_.isArray(parsedData)) {
+      this.data = {wrapper: parsedData};
     } else {
-      this.data = {wrapper: [this.data]};
+      this.data = {wrapper: [parsedData]};
     }
   }
 

libs/hdf-converters/test/mappers/forward/trufflehog_mapper.spec.ts

@@ -238,3 +238,33 @@ describe('trufflehog_example_mapper', () => {
     );
   });
 });
+
+describe('trufflehog_dup_ndjson', () => {
+  it('Successfully converts trufflehog in ndjson format with duplicate findings', () => {
+    const mapper = new TrufflehogResults(
+      fs.readFileSync(
+        'sample_jsons/trufflehog_mapper/sample_input_report/trufflehog_dup.ndjson',
+        {encoding: 'utf-8'}
+      ),
+      false
+    );
+
+    // fs.writeFileSync(
+    //   'sample_jsons/trufflehog_mapper/trufflehog-ndjson-dup-hdf.json',
+    //   JSON.stringify(mapper.toHdf(), null, 2)
+    // );
+
+    expect(omitVersions(mapper.toHdf())).toEqual(
+      omitVersions(
+        JSON.parse(
+          fs.readFileSync(
+            'sample_jsons/trufflehog_mapper/trufflehog-ndjson-dup-hdf.json',
+            {
+              encoding: 'utf-8'
+            }
+          )
+        )
+      )
+    );
+  });
+});