diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd2e145b..a72a7f93 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,11 +10,6 @@ jobs: strategy: matrix: name: - - Node.js 0.10 - - Node.js 0.12 - - io.js 1.x - - io.js 2.x - - io.js 3.x - Node.js 4.x - Node.js 5.x - Node.js 6.x @@ -33,26 +28,6 @@ jobs: - Node.js 19.x include: - - name: Node.js 0.10 - node-version: "0.10" - npm-i: mocha@3.5.3 nyc@10.3.2 supertest@2.0.0 - - - name: Node.js 0.12 - node-version: "0.12" - npm-i: mocha@3.5.3 nyc@10.3.2 supertest@2.0.0 - - - name: io.js 1.x - node-version: "1.8" - npm-i: mocha@3.5.3 nyc@10.3.2 supertest@2.0.0 - - - name: io.js 2.x - node-version: "2.5" - npm-i: mocha@3.5.3 nyc@10.3.2 supertest@2.0.0 - - - name: io.js 3.x - node-version: "3.3" - npm-i: mocha@3.5.3 nyc@10.3.2 supertest@2.0.0 - - name: Node.js 4.x node-version: "4.9" npm-i: mocha@5.2.0 nyc@11.9.0 supertest@3.4.2 @@ -71,11 +46,11 @@ jobs: - name: Node.js 8.x node-version: "8.17" - npm-i: mocha@7.2.0 + npm-i: mocha@6.2.2 nyc@14.1.1 supertest@6.1.6 - name: Node.js 9.x node-version: "9.11" - npm-i: mocha@7.2.0 + npm-i: mocha@6.2.2 nyc@14.1.1 supertest@6.1.6 - name: Node.js 10.x node-version: "10.24" diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 00000000..39372a22 --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,69 @@ +# This workflow uses actions that are not certified by GitHub. They are provided +# by a third-party and are governed by separate terms of service, privacy +# policy, and support documentation. + +name: Scorecard supply-chain security +on: + # For Branch-Protection check. Only the default branch is supported. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection + branch_protection_rule: + # To guarantee Maintained check is occasionally updated. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained + schedule: + - cron: '16 21 * * 1' + push: + branches: [ "master" ] + +# Declare default permissions as read only. +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + permissions: + # Needed to upload the results to code-scanning dashboard. + security-events: write + # Needed to publish results and get a badge (see publish_results below). + id-token: write + + steps: + - name: "Checkout code" + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.2 + with: + persist-credentials: false + + - name: "Run analysis" + uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 + with: + results_file: results.sarif + results_format: sarif + # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: + # - you want to enable the Branch-Protection check on a *public* repository, or + # - you are installing Scorecard on a *private* repository + # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat. + # repo_token: ${{ secrets.SCORECARD_TOKEN }} + + # Public repositories: + # - Publish results to OpenSSF REST API for easy access by consumers + # - Allows the repository to include the Scorecard badge. + # - See https://github.com/ossf/scorecard-action#publishing-results. + # For private repositories: + # - `publish_results` will always be set to `false`, regardless + # of the value entered here. + publish_results: true + + # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF + # format to the repository Actions tab. + - name: "Upload artifact" + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + # Upload the results to GitHub's code scanning dashboard. + - name: "Upload to code-scanning" + uses: github/codeql-action/upload-sarif@2f93e4319b2f04a2efc38fa7f78bd681bc3f7b2f # v2.23.2 + with: + sarif_file: results.sarif diff --git a/README.md b/README.md index c9cbf023..60528082 100644 --- a/README.md +++ b/README.md @@ -87,16 +87,27 @@ specifies the number of bytes; if it is a string, the value is passed to the [bytes](https://www.npmjs.com/package/bytes) library for parsing. Defaults to `'100kb'`. +##### parser + +The `parser` option is the function called against the request body to convert +it to a Javascript object. If a `reviver` is supplied, it is supplied as the +second argument to this function. + +``` +parser(body, reviver) -> req.body +``` + +Defaults to `JSON.parse`. + ##### reviver -The `reviver` option is passed directly to `JSON.parse` as the second -argument. You can find more information on this argument +You can find more information on this argument [in the MDN documentation about JSON.parse](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/parse#Example.3A_Using_the_reviver_parameter). ##### strict When set to `true`, will only accept arrays and objects; when `false` will -accept anything `JSON.parse` accepts. Defaults to `true`. +accept anything the `parser` accepts. Defaults to `true`. ##### type @@ -273,6 +284,16 @@ The `verify` option, if supplied, is called as `verify(req, res, buf, encoding)` where `buf` is a `Buffer` of the raw request body and `encoding` is the encoding of the request. The parsing can be aborted by throwing an error. +##### parser + +The `parser` option, if supplied, is used to in place of the default parser to +convert the request body into a Javascript object. If this option is supplied, +both the `extended` and `parameterLimit` options are ignored. + +``` +parser(body) -> req.body +``` + ## Errors The middlewares provided by this module create errors using the diff --git a/index.js b/index.js index 81fb9046..b6bdf9f9 100644 --- a/index.js +++ b/index.js @@ -73,6 +73,17 @@ Object.defineProperty(exports, 'urlencoded', { get: createParserGetter('urlencoded') }) +/** + * Generic parser used to build parsers. + * @public + */ + +Object.defineProperty(exports, 'generic', { + configurable: true, + enumerable: true, + get: createParserGetter('generic') +}) + /** * Create a middleware to parse json and urlencoded bodies. * @@ -123,6 +134,9 @@ function loadParser (parserName) { case 'urlencoded': parser = require('./lib/types/urlencoded') break + case 'generic': + parser = require('./lib/generic-parser') + break } // store to prevent invoking require() diff --git a/lib/generic-parser.js b/lib/generic-parser.js new file mode 100644 index 00000000..deb96d10 --- /dev/null +++ b/lib/generic-parser.js @@ -0,0 +1,160 @@ +/*! + * body-parser + * Copyright(c) 2014 Jonathan Ong + * Copyright(c) 2014-2015 Douglas Christopher Wilson + * MIT Licensed + */ + +'use strict' + +/** + * Module dependencies. + * @private + */ + +var bytes = require('bytes') +var contentType = require('content-type') +var createError = require('http-errors') +var debug = require('debug')('body-parser:generic') +var isFinished = require('on-finished').isFinished +var read = require('./read') +var typeis = require('type-is') + +/** + * Module exports. + */ + +module.exports = generic + +/** + * Use this to create a middleware that parses request bodies + * + * @param {object} [options] + * @return {function} + * @public + */ + +function generic (parserOptions, parserOverrides) { + // Squash the options and the overrides down into one object + var opts = Object.create(parserOptions) + Object.assign(opts, parserOverrides) + + var limit = typeof opts.limit !== 'number' + ? bytes.parse(opts.limit || '100kb') + : opts.limit + var charset = opts.charset + var inflate = opts.inflate !== false + var verify = opts.verify || false + var parse = opts.parse || defaultParse + var defaultReqCharset = opts.defaultCharset || 'utf-8' + var type = opts.type + + if (verify !== false && typeof verify !== 'function') { + throw new TypeError('option verify must be function') + } + + // create the appropriate type checking function + var shouldParse = typeof type !== 'function' + ? typeChecker(type) + : type + + // create the appropriate charset validating function + var validCharset = typeof charset !== 'function' + ? charsetValidator(charset) + : charset + + return function genericParser (req, res, next) { + if (isFinished(req)) { + debug('body already parsed') + next() + return + } + + if (!('body' in req)) { + req.body = undefined + } + + // skip requests without bodies + if (!typeis.hasBody(req)) { + debug('skip empty body') + next() + return + } + + debug('content-type %j', req.headers['content-type']) + + // determine if request should be parsed + if (!shouldParse(req)) { + debug('skip parsing') + next() + return + } + + // assert charset per RFC 7159 sec 8.1 + var reqCharset = null + if (charset !== undefined) { + reqCharset = getCharset(req) || defaultReqCharset + if (!validCharset(reqCharset)) { + debug('invalid charset') + next(createError(415, 'unsupported charset "' + reqCharset.toUpperCase() + '"', { + charset: reqCharset, + type: 'charset.unsupported' + })) + return + } + } + + // read + read(req, res, next, parse, debug, { + encoding: reqCharset, + inflate: inflate, + limit: limit, + verify: verify + }) + } +} + +function defaultParse (buf) { + return buf +} + +/** + * Get the charset of a request. + * + * @param {object} req + * @api private + */ + +function getCharset (req) { + try { + return (contentType.parse(req).parameters.charset || '').toLowerCase() + } catch (e) { + return undefined + } +} + +/** + * Get the simple type checker. + * + * @param {string} type + * @return {function} + */ + +function typeChecker (type) { + return function checkType (req) { + return Boolean(typeis(req, type)) + } +} + +/** + * Get the simple charset validator. + * + * @param {string} type + * @return {function} + */ + +function charsetValidator (charset) { + return function validateCharset (reqCharset) { + return charset === reqCharset + } +} diff --git a/lib/types/json.js b/lib/types/json.js index 30bf8cab..ae1af796 100644 --- a/lib/types/json.js +++ b/lib/types/json.js @@ -12,13 +12,8 @@ * @private */ -var bytes = require('bytes') -var contentType = require('content-type') -var createError = require('http-errors') +var genericParser = require('../..').generic var debug = require('debug')('body-parser:json') -var isFinished = require('on-finished').isFinished -var read = require('../read') -var typeis = require('type-is') /** * Module exports. @@ -54,97 +49,45 @@ var JSON_SYNTAX_REGEXP = /#+/g function json (options) { var opts = options || {} - var limit = typeof opts.limit !== 'number' - ? bytes.parse(opts.limit || '100kb') - : opts.limit - var inflate = opts.inflate !== false var reviver = opts.reviver var strict = opts.strict !== false + var parser = opts.parser || JSON.parse var type = opts.type || 'application/json' - var verify = opts.verify || false - if (verify !== false && typeof verify !== 'function') { - throw new TypeError('option verify must be function') - } - - // create the appropriate type checking function - var shouldParse = typeof type !== 'function' - ? typeChecker(type) - : type + return genericParser(opts, { + type: type, - function parse (body) { - if (body.length === 0) { - // special-case empty json body, as it's a common client-side mistake - // TODO: maybe make this configurable or part of "strict" option - return {} - } + charset: function validateCharset (charset) { + return charset.slice(0, 4) === 'utf-' + }, - if (strict) { - var first = firstchar(body) - - if (first !== '{' && first !== '[') { - debug('strict violation') - throw createStrictSyntaxError(body, first) + parse: function parse (buf) { + if (buf.length === 0) { + // special-case empty json body, as it's a common client-side mistake + // TODO: maybe make this configurable or part of "strict" option + return {} } - } - try { - debug('parse json') - return JSON.parse(body, reviver) - } catch (e) { - throw normalizeJsonSyntaxError(e, { - message: e.message, - stack: e.stack - }) - } - } + if (strict) { + var first = firstchar(buf) - return function jsonParser (req, res, next) { - if (isFinished(req)) { - debug('body already parsed') - next() - return - } - - if (!('body' in req)) { - req.body = undefined - } - - // skip requests without bodies - if (!typeis.hasBody(req)) { - debug('skip empty body') - next() - return - } - - debug('content-type %j', req.headers['content-type']) - - // determine if request should be parsed - if (!shouldParse(req)) { - debug('skip parsing') - next() - return - } + if (first !== '{' && first !== '[') { + debug('strict violation') + throw createStrictSyntaxError(parser, reviver, buf, first) + } + } - // assert charset per RFC 7159 sec 8.1 - var charset = getCharset(req) || 'utf-8' - if (charset.slice(0, 4) !== 'utf-') { - debug('invalid charset') - next(createError(415, 'unsupported charset "' + charset.toUpperCase() + '"', { - charset: charset, - type: 'charset.unsupported' - })) - return + try { + debug('parse json') + return parser(buf, reviver) + } catch (e) { + throw normalizeJsonSyntaxError(e, { + message: e.message, + stack: e.stack + }) + } } - - // read - read(req, res, next, parse, debug, { - encoding: charset, - inflate: inflate, - limit: limit, - verify: verify - }) - } + }) } /** @@ -156,7 +99,7 @@ function json (options) { * @private */ -function createStrictSyntaxError (str, char) { +function createStrictSyntaxError (parser, reviver, str, char) { var index = str.indexOf(char) var partial = '' @@ -169,7 +112,7 @@ function createStrictSyntaxError (str, char) { } try { - JSON.parse(partial); /* istanbul ignore next */ throw new SyntaxError('strict violation') + parser(partial, reviver); /* istanbul ignore next */ throw new SyntaxError('strict violation') } catch (e) { return normalizeJsonSyntaxError(e, { message: e.message.replace(JSON_SYNTAX_REGEXP, function (placeholder) { @@ -196,21 +139,6 @@ function firstchar (str) { : undefined } -/** - * Get the charset of a request. - * - * @param {object} req - * @api private - */ - -function getCharset (req) { - try { - return (contentType.parse(req).parameters.charset || '').toLowerCase() - } catch (e) { - return undefined - } -} - /** * Normalize a SyntaxError for JSON.parse. * @@ -235,16 +163,3 @@ function normalizeJsonSyntaxError (error, obj) { return error } - -/** - * Get the simple type checker. - * - * @param {string} type - * @return {function} - */ - -function typeChecker (type) { - return function checkType (req) { - return Boolean(typeis(req, type)) - } -} diff --git a/lib/types/raw.js b/lib/types/raw.js index bfe274cf..b40cfd5d 100644 --- a/lib/types/raw.js +++ b/lib/types/raw.js @@ -10,11 +10,7 @@ * Module dependencies. */ -var bytes = require('bytes') -var debug = require('debug')('body-parser:raw') -var isFinished = require('on-finished').isFinished -var read = require('../read') -var typeis = require('type-is') +var genericParser = require('../..').generic /** * Module exports. @@ -33,72 +29,9 @@ module.exports = raw function raw (options) { var opts = options || {} - var inflate = opts.inflate !== false - var limit = typeof opts.limit !== 'number' - ? bytes.parse(opts.limit || '100kb') - : opts.limit var type = opts.type || 'application/octet-stream' - var verify = opts.verify || false - if (verify !== false && typeof verify !== 'function') { - throw new TypeError('option verify must be function') - } - - // create the appropriate type checking function - var shouldParse = typeof type !== 'function' - ? typeChecker(type) - : type - - function parse (buf) { - return buf - } - - return function rawParser (req, res, next) { - if (isFinished(req)) { - debug('body already parsed') - next() - return - } - - if (!('body' in req)) { - req.body = undefined - } - - // skip requests without bodies - if (!typeis.hasBody(req)) { - debug('skip empty body') - next() - return - } - - debug('content-type %j', req.headers['content-type']) - - // determine if request should be parsed - if (!shouldParse(req)) { - debug('skip parsing') - next() - return - } - - // read - read(req, res, next, parse, debug, { - encoding: null, - inflate: inflate, - limit: limit, - verify: verify - }) - } -} - -/** - * Get the simple type checker. - * - * @param {string} type - * @return {function} - */ - -function typeChecker (type) { - return function checkType (req) { - return Boolean(typeis(req, type)) - } + return genericParser(opts, { + type: type + }) } diff --git a/lib/types/text.js b/lib/types/text.js index b153931b..87e65793 100644 --- a/lib/types/text.js +++ b/lib/types/text.js @@ -10,12 +10,7 @@ * Module dependencies. */ -var bytes = require('bytes') -var contentType = require('content-type') -var debug = require('debug')('body-parser:text') -var isFinished = require('on-finished').isFinished -var read = require('../read') -var typeis = require('type-is') +var genericParser = require('../..').generic /** * Module exports. @@ -35,90 +30,11 @@ function text (options) { var opts = options || {} var defaultCharset = opts.defaultCharset || 'utf-8' - var inflate = opts.inflate !== false - var limit = typeof opts.limit !== 'number' - ? bytes.parse(opts.limit || '100kb') - : opts.limit var type = opts.type || 'text/plain' - var verify = opts.verify || false - if (verify !== false && typeof verify !== 'function') { - throw new TypeError('option verify must be function') - } - - // create the appropriate type checking function - var shouldParse = typeof type !== 'function' - ? typeChecker(type) - : type - - function parse (buf) { - return buf - } - - return function textParser (req, res, next) { - if (isFinished(req)) { - debug('body already parsed') - next() - return - } - - if (!('body' in req)) { - req.body = undefined - } - - // skip requests without bodies - if (!typeis.hasBody(req)) { - debug('skip empty body') - next() - return - } - - debug('content-type %j', req.headers['content-type']) - - // determine if request should be parsed - if (!shouldParse(req)) { - debug('skip parsing') - next() - return - } - - // get charset - var charset = getCharset(req) || defaultCharset - - // read - read(req, res, next, parse, debug, { - encoding: charset, - inflate: inflate, - limit: limit, - verify: verify - }) - } -} - -/** - * Get the charset of a request. - * - * @param {object} req - * @api private - */ - -function getCharset (req) { - try { - return (contentType.parse(req).parameters.charset || '').toLowerCase() - } catch (e) { - return undefined - } -} - -/** - * Get the simple type checker. - * - * @param {string} type - * @return {function} - */ - -function typeChecker (type) { - return function checkType (req) { - return Boolean(typeis(req, type)) - } + return genericParser(opts, { + type: type, + charset: function validateCharset () { return true }, + defaultCharset: defaultCharset + }) } diff --git a/lib/types/urlencoded.js b/lib/types/urlencoded.js index f4ba2cd0..c9362757 100644 --- a/lib/types/urlencoded.js +++ b/lib/types/urlencoded.js @@ -12,13 +12,9 @@ * @private */ -var bytes = require('bytes') -var contentType = require('content-type') var createError = require('http-errors') var debug = require('debug')('body-parser:urlencoded') -var isFinished = require('on-finished').isFinished -var read = require('../read') -var typeis = require('type-is') +var genericParser = require('../..').generic /** * Module exports. @@ -39,85 +35,29 @@ var parsers = Object.create(null) * @return {function} * @public */ - function urlencoded (options) { var opts = options || {} var extended = Boolean(opts.extended) - var inflate = opts.inflate !== false - var limit = typeof opts.limit !== 'number' - ? bytes.parse(opts.limit || '100kb') - : opts.limit var type = opts.type || 'application/x-www-form-urlencoded' - var verify = opts.verify || false - - if (verify !== false && typeof verify !== 'function') { - throw new TypeError('option verify must be function') - } - - // create the appropriate query parser - var queryparse = extended - ? extendedparser(opts) - : simpleparser(opts) - - // create the appropriate type checking function - var shouldParse = typeof type !== 'function' - ? typeChecker(type) - : type - - function parse (body) { - return body.length - ? queryparse(body) - : {} - } - - return function urlencodedParser (req, res, next) { - if (isFinished(req)) { - debug('body already parsed') - next() - return - } - - if (!('body' in req)) { - req.body = undefined - } - - // skip requests without bodies - if (!typeis.hasBody(req)) { - debug('skip empty body') - next() - return + var charset = opts.charset || 'utf-8' + + var queryparse = opts.parser || ( + extended + ? extendedparser(opts) + : simpleparser(opts) + ) + + return genericParser(opts, { + type: type, + charset: charset, + + parse: function parse (buf) { + return buf.length + ? queryparse(buf) + : {} } - - debug('content-type %j', req.headers['content-type']) - - // determine if request should be parsed - if (!shouldParse(req)) { - debug('skip parsing') - next() - return - } - - // assert charset - var charset = getCharset(req) || 'utf-8' - if (charset !== 'utf-8') { - debug('invalid charset') - next(createError(415, 'unsupported charset "' + charset.toUpperCase() + '"', { - charset: charset, - type: 'charset.unsupported' - })) - return - } - - // read - read(req, res, next, parse, debug, { - debug: debug, - encoding: charset, - inflate: inflate, - limit: limit, - verify: verify - }) - } + }) } /** @@ -162,21 +102,6 @@ function extendedparser (options) { } } -/** - * Get the charset of a request. - * - * @param {object} req - * @api private - */ - -function getCharset (req) { - try { - return (contentType.parse(req).parameters.charset || '').toLowerCase() - } catch (e) { - return undefined - } -} - /** * Count the number of parameters, stopping once limit reached * @@ -266,16 +191,3 @@ function simpleparser (options) { return parse(body, undefined, undefined, { maxKeys: parameterLimit }) } } - -/** - * Get the simple type checker. - * - * @param {string} type - * @return {function} - */ - -function typeChecker (type) { - return function checkType (req) { - return Boolean(typeis(req, type)) - } -} diff --git a/package.json b/package.json index 5201dea5..aa64b77a 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "lint": "eslint .", "test": "mocha --require test/support/env --reporter spec --check-leaks --bail test/", "test-ci": "nyc --reporter=lcov --reporter=text npm test", - "test-cov": "nyc --reporter=html --reporter=text npm test" + "test-cov": "nyc --reporter=html --reporter=text npm test", + "test-debug": "npm test -- --timeout 0" } -} +} \ No newline at end of file diff --git a/test/json.js b/test/json.js index 1040d56b..0a069c1f 100644 --- a/test/json.js +++ b/test/json.js @@ -99,6 +99,18 @@ describe('bodyParser.json()', function () { .expect(200, '{"user":"tobi"}', done) }) + it('should use external parsers', function (done) { + request(createServer({ + parser: function (body) { + return { foo: 'bar' } + } + })) + .post('/') + .set('Content-Type', 'application/json') + .send('{"str":') + .expect(200, '{"foo":"bar"}', done) + }) + describe('when JSON is invalid', function () { before(function () { this.server = createServer() diff --git a/test/urlencoded.js b/test/urlencoded.js index 8be8a5a0..c22bab71 100644 --- a/test/urlencoded.js +++ b/test/urlencoded.js @@ -25,6 +25,16 @@ describe('bodyParser.urlencoded()', function () { .expect(200, '{"user":"tobi"}', done) }) + it('should parse x-www-form-urlencoded with custom parser', function (done) { + request(createServer({ + parser: function (input) { return input.toUpperCase() } + })) + .post('/') + .set('Content-Type', 'application/x-www-form-urlencoded') + .send('user=tobi') + .expect(200, '"USER=TOBI"', done) + }) + it('should 400 when invalid content-length', function (done) { var urlencodedParser = bodyParser.urlencoded() var server = createServer(function (req, res, next) {