From c3fc9801adfe83f18a10c14f1408cd3adffc96b7 Mon Sep 17 00:00:00 2001 From: titanism <101466223+titanism@users.noreply.github.com> Date: Sat, 7 Dec 2024 18:16:32 -0600 Subject: [PATCH] fix: fixed header line parsing issue with getHeaders, bump deps, fixed tests --- helpers/get-attributes.js | 1 + helpers/get-headers.js | 114 ++++++++++++++++++++++++++++++------- helpers/get-transporter.js | 2 +- helpers/on-data-mx.js | 2 +- package.json | 20 +++---- 5 files changed, 108 insertions(+), 31 deletions(-) diff --git a/helpers/get-attributes.js b/helpers/get-attributes.js index da0c56f661..62ddefde0d 100644 --- a/helpers/get-attributes.js +++ b/helpers/get-attributes.js @@ -46,6 +46,7 @@ async function getAttributes(headers, session, resolver, isAligned = false) { session.originalFromAddressRootDomain ]; + // TODO: this is null it seems const replyTo = [ // check the Reply-To header ...replyToAddresses.map((addr) => checkSRS(addr.address).toLowerCase()), diff --git a/helpers/get-headers.js b/helpers/get-headers.js index 8e7f60644b..f5cae1bc51 100644 --- a/helpers/get-headers.js +++ b/helpers/get-headers.js @@ -3,6 +3,48 @@ * SPDX-License-Identifier: BUSL-1.1 */ +// TODO: should we use messageSplitter.rawHeaders instead + +/* +const ENCODED_HEADERS = new Set([ + 'bcc', + 'cc', + 'content-disposition', + 'content-type', + 'delivered-to', + 'dkim-signature', + 'from', + 'in-reply-to', + 'message-id', + 'references', + 'reply-to', + 'return-path', + 'sender', + 'subject', + 'to' +]); +*/ + +const SINGLE_KEYS = new Set([ + 'content-description', + 'content-disposition', + 'content-id', + 'content-transfer-encoding', + 'content-type', + 'date', + // NOTE: Errors-To header is deprecated + 'errors-to', + 'from', + 'in-reply-to', + 'message-id', + 'mime-version', + 'precedence', + 'priority', + 'reply-to', + 'sender', + 'subject' +]); + function getHeaders(headers, specificKey = null) { const _headers = {}; @@ -18,32 +60,66 @@ function getHeaders(headers, specificKey = null) { // > 🎉 beep // (e.g. so a user could search for 🎉) // - // we can't use mailsplit b/c unicode characters get rewritten - // - // - let lines = headers.headers.toString(); - try { - lines = headers.libmime.decodeWords(lines); - } catch { - // ignore, keep as is - } - - lines = lines - // - .replace(/\r?\n?\t/g, ' ') + // + // NOTE: we don't want toString('binary') like above mailsplit code has otherwise we lose unicode chars + const lines = headers.headers + .toString() .replace(/[\r\n]+$/, '') .split(/\r?\n/); - for (const line of lines) { + for (let i = lines.length - 1; i >= 0; i--) { + const chr = lines[i].charAt(0); + if (i && (chr === ' ' || chr === '\t')) { + lines[i - 1] += '\r\n' + lines[i]; + lines.splice(i, 1); + } else { + const line = lines[i]; + if (!i && /^from /i.test(line)) { + lines.splice(i, 1); + continue; + } else if (!i && /^post /i.test(line)) { + lines.splice(i, 1); + continue; + } + + const key = headers._normalizeHeader( + line.slice(0, Math.max(0, line.indexOf(':'))) + ); + lines[i] = { + key, + line + }; + } + } + + for (const { line } of lines) { const index = line.indexOf(':'); const key = line.slice(0, index); + const lc = key.toLowerCase(); + + // only keep the first value for certain keys + // + if (_headers[key] && SINGLE_KEYS.has(lc)) continue; + + _headers[key] = line + .slice(index + 1) + .trim() + .replace(/\s*\r?\n\s*/g, ' '); - _headers[key] = line.slice(index + 1).trim(); + // + // NOTE: it's recommended but we don't decode certain keys, we decode all + // (e.g. a List-Unsubscribe header with "<" could get encoded as + // =?us-ascii?Q??=\n' + + // + // https://github.com/nodemailer/wildduck/blob/7daa0e35d5462c46ff4228638f2e9e5f30ed880d/lib/message-handler.js#L1271-L1274 + // + // if (ENCODED_HEADERS.has(lc)) { + try { + _headers[key] = headers.libmime.decodeWords(_headers[key]); + } catch {} + // } - if ( - typeof specificKey === 'string' && - key.toLowerCase() === specificKey.toLowerCase() - ) + if (typeof specificKey === 'string' && lc === specificKey.toLowerCase()) return _headers[key]; } diff --git a/helpers/get-transporter.js b/helpers/get-transporter.js index 6d23d37ac3..7eeb09af23 100644 --- a/helpers/get-transporter.js +++ b/helpers/get-transporter.js @@ -112,7 +112,7 @@ async function getTransporter(options = {}, err) { // // this is required since custom port forwarding would be recursive otherwise - if (port === 25) { + if (env.NODE_ENV === 'test' || port === 25) { // mx = await asyncMxConnect({ ignoreMXHosts, diff --git a/helpers/on-data-mx.js b/helpers/on-data-mx.js index 12e2db439c..216fb84ed1 100644 --- a/helpers/on-data-mx.js +++ b/helpers/on-data-mx.js @@ -120,7 +120,7 @@ async function sendBounce(bounce, headers, session, sealedMessage) { messageId: headers.getFirst('message-id'), date: session.arrivalDate }, - bounce.error, + bounce.err, sealedMessage ); diff --git a/package.json b/package.json index b3a72a886b..8f4a8cd672 100644 --- a/package.json +++ b/package.json @@ -23,9 +23,9 @@ ], "dependencies": { "@ava/get-port": "2.0.0", - "@aws-sdk/client-s3": "3.700.0", - "@aws-sdk/lib-storage": "3.700.0", - "@aws-sdk/s3-request-presigner": "3.700.0", + "@aws-sdk/client-s3": "3.705.0", + "@aws-sdk/lib-storage": "3.705.0", + "@aws-sdk/s3-request-presigner": "3.705.0", "@forevolve/bootstrap-dark": "4.0.2", "@fortawesome/fontawesome-free": "5.15.4", "@forwardemail/bytes": "4.0.0", @@ -62,7 +62,7 @@ "@shopify/semaphore": "3.1.0", "@sidoshi/random-string": "1.0.0", "@tkrotoff/bootstrap-floating-label": "0.8", - "@ungap/structured-clone": "1.2.0", + "@ungap/structured-clone": "1.2.1", "@zainundin/mongoose-factory": "1.1.1", "@zxcvbn-ts/core": "3.0.4", "@zxcvbn-ts/language-ar": "3.1.0", @@ -144,7 +144,7 @@ "humanize-string": "2", "ical.js": "2.1.0", "iconv": "3.0.1", - "imapflow": "1.0.169", + "imapflow": "1.0.171", "into-stream": "6.0.0", "ip": "2.0.1", "ipaddr.js": "2.2.0", @@ -170,11 +170,11 @@ "koa-views-render": "0.0.1", "lazyframe": "2.2.7", "lazyload": "2.0.0-rc.2", - "libmime": "5.3.5", + "libmime": "5.3.6", "lodash": "4.17.21", "mailauth": "4.8.1", - "mailparser": "3.7.1", - "mailsplit": "5.4.0", + "mailparser": "3.7.2", + "mailsplit": "5.4.2", "mandarin": "5.0.6", "manifest-rev": "1.0.3", "markdown-it": "13.0.2", @@ -226,7 +226,7 @@ "piscina": "4.7.0", "plist": "3.1.0", "pluralize": "8.0.0", - "pm2": "5.4.2", + "pm2": "5.4.3", "popper.js": "1.16.1", "postcss-100vh-fix": "1.0.2", "postcss-css-variables": "0.19.0", @@ -243,7 +243,7 @@ "pug": "3.0.3", "puppeteer": "23.7.0", "qrcode": "1.5.4", - "qs": "6.13.0", + "qs": "6.13.1", "re2": "1.21.4", "regex-parser": "2.3.0", "reserved-email-addresses-list": "2.0.14",