fix: fixed header line parsing issue with getHeaders, bump deps, fixe…

…d tests

helpers/get-attributes.js

@@ -46,6 +46,7 @@ async function getAttributes(headers, session, resolver, isAligned = false) {
     session.originalFromAddressRootDomain
   ];
 
+  // TODO: this is null it seems
   const replyTo = [
     // check the Reply-To header
     ...replyToAddresses.map((addr) => checkSRS(addr.address).toLowerCase()),

helpers/get-headers.js

@@ -3,6 +3,48 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
+// TODO: should we use messageSplitter.rawHeaders instead
+
+/*
+const ENCODED_HEADERS = new Set([
+  'bcc',
+  'cc',
+  'content-disposition',
+  'content-type',
+  'delivered-to',
+  'dkim-signature',
+  'from',
+  'in-reply-to',
+  'message-id',
+  'references',
+  'reply-to',
+  'return-path',
+  'sender',
+  'subject',
+  'to'
+]);
+*/
+
+const SINGLE_KEYS = new Set([
+  'content-description',
+  'content-disposition',
+  'content-id',
+  'content-transfer-encoding',
+  'content-type',
+  'date',
+  // NOTE: Errors-To header is deprecated
+  'errors-to',
+  'from',
+  'in-reply-to',
+  'message-id',
+  'mime-version',
+  'precedence',
+  'priority',
+  'reply-to',
+  'sender',
+  'subject'
+]);
+
 function getHeaders(headers, specificKey = null) {
   const _headers = {};
 
@@ -18,32 +60,66 @@ function getHeaders(headers, specificKey = null) {
   //       > 🎉 beep
   //       (e.g. so a user could search for 🎉)
   //
-  //       we can't use mailsplit b/c unicode characters get rewritten
-  //       <https://github.com/andris9/mailsplit/issues/21>
-  //
-  let lines = headers.headers.toString();
-  try {
-    lines = headers.libmime.decodeWords(lines);
-  } catch {
-    // ignore, keep as is
-  }
-
-  lines = lines
-    // <https://github.com/andris9/mailsplit/issues/22>
-    .replace(/\r?\n?\t/g, ' ')
+  // <https://github.com/andris9/mailsplit/blob/37b0d216d84bae0b2efacd65a3c2d89d8452b323/lib/headers.js#L190-L231>
+  // NOTE: we don't want toString('binary') like above mailsplit code has otherwise we lose unicode chars
+  const lines = headers.headers
+    .toString()
     .replace(/[\r\n]+$/, '')
     .split(/\r?\n/);
 
-  for (const line of lines) {
+  for (let i = lines.length - 1; i >= 0; i--) {
+    const chr = lines[i].charAt(0);
+    if (i && (chr === ' ' || chr === '\t')) {
+      lines[i - 1] += '\r\n' + lines[i];
+      lines.splice(i, 1);
+    } else {
+      const line = lines[i];
+      if (!i && /^from /i.test(line)) {
+        lines.splice(i, 1);
+        continue;
+      } else if (!i && /^post /i.test(line)) {
+        lines.splice(i, 1);
+        continue;
+      }
+
+      const key = headers._normalizeHeader(
+        line.slice(0, Math.max(0, line.indexOf(':')))
+      );
+      lines[i] = {
+        key,
+        line
+      };
+    }
+  }
+
+  for (const { line } of lines) {
     const index = line.indexOf(':');
     const key = line.slice(0, index);
+    const lc = key.toLowerCase();
+
+    // only keep the first value for certain keys
+    // <https://github.com/nodemailer/mailparser/blob/ac11f78429cf13da42162e996a05b875030ae1c1/lib/mail-parser.js#L395-L413>
+    if (_headers[key] && SINGLE_KEYS.has(lc)) continue;
+
+    _headers[key] = line
+      .slice(index + 1)
+      .trim()
+      .replace(/\s*\r?\n\s*/g, ' ');
 
-    _headers[key] = line.slice(index + 1).trim();
+    //
+    // NOTE: it's recommended but we don't decode certain keys, we decode all
+    //       (e.g. a List-Unsubscribe header with "<" could get encoded as
+    //        =?us-ascii?Q?<https://example.com/unsubscribe?= =?us-ascii?Q?.php=3Fc=3D123>?=\n' +
+    //
+    //       https://github.com/nodemailer/wildduck/blob/7daa0e35d5462c46ff4228638f2e9e5f30ed880d/lib/message-handler.js#L1271-L1274
+    //
+    // if (ENCODED_HEADERS.has(lc)) {
+    try {
+      _headers[key] = headers.libmime.decodeWords(_headers[key]);
+    } catch {}
+    // }
 
-    if (
-      typeof specificKey === 'string' &&
-      key.toLowerCase() === specificKey.toLowerCase()
-    )
+    if (typeof specificKey === 'string' && lc === specificKey.toLowerCase())
       return _headers[key];
   }
 

helpers/get-transporter.js

@@ -112,7 +112,7 @@ async function getTransporter(options = {}, err) {
   //
 
   // this is required since custom port forwarding would be recursive otherwise
-  if (port === 25) {
+  if (env.NODE_ENV === 'test' || port === 25) {
     // <https://github.com/zone-eu/mx-connect#configuration-options>
     mx = await asyncMxConnect({
       ignoreMXHosts,

helpers/on-data-mx.js

@@ -120,7 +120,7 @@ async function sendBounce(bounce, headers, session, sealedMessage) {
         messageId: headers.getFirst('message-id'),
         date: session.arrivalDate
       },
-      bounce.error,
+      bounce.err,
       sealedMessage
     );
 

package.json

@@ -23,9 +23,9 @@
   ],
   "dependencies": {
     "@ava/get-port": "2.0.0",
-    "@aws-sdk/client-s3": "3.700.0",
-    "@aws-sdk/lib-storage": "3.700.0",
-    "@aws-sdk/s3-request-presigner": "3.700.0",
+    "@aws-sdk/client-s3": "3.705.0",
+    "@aws-sdk/lib-storage": "3.705.0",
+    "@aws-sdk/s3-request-presigner": "3.705.0",
     "@forevolve/bootstrap-dark": "4.0.2",
     "@fortawesome/fontawesome-free": "5.15.4",
     "@forwardemail/bytes": "4.0.0",
@@ -62,7 +62,7 @@
     "@shopify/semaphore": "3.1.0",
     "@sidoshi/random-string": "1.0.0",
     "@tkrotoff/bootstrap-floating-label": "0.8",
-    "@ungap/structured-clone": "1.2.0",
+    "@ungap/structured-clone": "1.2.1",
     "@zainundin/mongoose-factory": "1.1.1",
     "@zxcvbn-ts/core": "3.0.4",
     "@zxcvbn-ts/language-ar": "3.1.0",
@@ -144,7 +144,7 @@
     "humanize-string": "2",
     "ical.js": "2.1.0",
     "iconv": "3.0.1",
-    "imapflow": "1.0.169",
+    "imapflow": "1.0.171",
     "into-stream": "6.0.0",
     "ip": "2.0.1",
     "ipaddr.js": "2.2.0",
@@ -170,11 +170,11 @@
     "koa-views-render": "0.0.1",
     "lazyframe": "2.2.7",
     "lazyload": "2.0.0-rc.2",
-    "libmime": "5.3.5",
+    "libmime": "5.3.6",
     "lodash": "4.17.21",
     "mailauth": "4.8.1",
-    "mailparser": "3.7.1",
-    "mailsplit": "5.4.0",
+    "mailparser": "3.7.2",
+    "mailsplit": "5.4.2",
     "mandarin": "5.0.6",
     "manifest-rev": "1.0.3",
     "markdown-it": "13.0.2",
@@ -226,7 +226,7 @@
     "piscina": "4.7.0",
     "plist": "3.1.0",
     "pluralize": "8.0.0",
-    "pm2": "5.4.2",
+    "pm2": "5.4.3",
     "popper.js": "1.16.1",
     "postcss-100vh-fix": "1.0.2",
     "postcss-css-variables": "0.19.0",
@@ -243,7 +243,7 @@
     "pug": "3.0.3",
     "puppeteer": "23.7.0",
     "qrcode": "1.5.4",
-    "qs": "6.13.0",
+    "qs": "6.13.1",
     "re2": "1.21.4",
     "regex-parser": "2.3.0",
     "reserved-email-addresses-list": "2.0.14",