diff --git a/package.json b/package.json index 12971007d..18501ceb2 100644 --- a/package.json +++ b/package.json @@ -27,11 +27,11 @@ "eslint-plugin-node": "11.1.0", "eslint-plugin-promise": "6.1.1", "eslint-plugin-standard": "4.1.0", - "got": "11.8.6", "media-typer": "1.1.0", "mocha": "10.2.0", "nyc": "15.1.0", - "stream-to-array": "2.3.0" + "stream-to-array": "2.3.0", + "undici": "7.1.0" }, "files": [ "HISTORY.md", diff --git a/scripts/fetch-apache.js b/scripts/fetch-apache.js index ef29b113b..5967df556 100644 --- a/scripts/fetch-apache.js +++ b/scripts/fetch-apache.js @@ -11,8 +11,8 @@ * Convert these text files to JSON for browser usage. */ -var got = require('got') var writedb = require('./lib/write-db') +var { request } = require('./lib/request') /** * Mime types and associated extensions are stored in the form: @@ -32,14 +32,15 @@ var TYPE_LINE_REGEXP = /^(?:# )?([\w-]+\/[\w+.-]+)((?:\s+[\w-]+)*)$/gm var URL = 'https://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types' ;(async function () { - const res = await got(URL) + const res = await request(URL) var json = {} var match = null + var body = await res.body.text() TYPE_LINE_REGEXP.index = 0 - while ((match = TYPE_LINE_REGEXP.exec(res.body))) { + while ((match = TYPE_LINE_REGEXP.exec(body))) { var mime = match[1] if (mime.slice(-8) === '/example') { diff --git a/scripts/fetch-iana.js b/scripts/fetch-iana.js index 60766210a..ec6cd4ba5 100644 --- a/scripts/fetch-iana.js +++ b/scripts/fetch-iana.js @@ -9,7 +9,7 @@ * Convert the IANA definitions from CSV to local. */ -var got = require('got') +var { request } = require('./lib/request') var parser = require('csv-parse') var toArray = require('stream-to-array') var typer = require('media-typer') @@ -93,15 +93,16 @@ async function addTemplateData (data, options) { if (!data.template) { return } - - let res = await got('https://www.iana.org/assignments/media-types/' + data.template) + let url = 'https://www.iana.org/assignments/media-types/' + data.template + let res = await request(url) var ref = data.type + '/' + data.name var rfc = getRfcReferences(data.reference)[0] if (res.statusCode === 404 && data.template !== ref) { console.log('template ' + data.template + ' not found, retry as ' + ref) data.template = ref - res = await got('https://www.iana.org/assignments/media-types/' + ref) + url = 'https://www.iana.org/assignments/media-types/' + ref + res = await request(url) // replace the guessed mime if (res.statusCode === 200) { @@ -111,7 +112,8 @@ async function addTemplateData (data, options) { if (res.statusCode === 404 && rfc !== undefined) { console.log('template ' + data.template + ' not found, fetch ' + rfc) - res = await got('https://tools.ietf.org/rfc/' + rfc.toLowerCase() + '.txt') + url = 'https://www.rfc-editor.org/rfc/' + rfc.toLowerCase() + '.txt' + res = await request(url) } if (res.statusCode === 404) { @@ -123,11 +125,11 @@ async function addTemplateData (data, options) { throw new Error('got status code ' + res.statusCode + ' from template ' + data.template) } - var body = getTemplateBody(res.body) + var body = getTemplateBody(await res.body.text()) var mime = extractTemplateMime(body) // add the template as a source - addSource(data, res.url) + addSource(data, url) if (mimeEql(mime, data.mime)) { // use extracted mime @@ -213,13 +215,14 @@ function extractTemplateExtensions (body) { } async function get (type, options) { - const res = await got('https://www.iana.org/assignments/media-types/' + encodeURIComponent(type) + '.csv') + const res = await request('https://www.iana.org/assignments/media-types/' + encodeURIComponent(type) + '.csv') if (res.statusCode !== 200) { throw new Error('got status code ' + res.statusCode + ' from ' + type) } - const mimes = await toArray(parser(res.body)) + const responseText = await res.body.text() + const mimes = await toArray(parser(responseText)) var headers = mimes.shift().map(normalizeHeader) var reduceRows = generateRowMapper(headers) const results = [] diff --git a/scripts/fetch-nginx.js b/scripts/fetch-nginx.js index 758193eb5..d42b14c7d 100644 --- a/scripts/fetch-nginx.js +++ b/scripts/fetch-nginx.js @@ -11,7 +11,7 @@ * Convert these text files to JSON for browser usage. */ -var got = require('got') +var { request } = require('./lib/request') var writedb = require('./lib/write-db') /** @@ -30,14 +30,15 @@ var TYPE_LINE_REGEXP = /^\s*([\w-]+\/[\w+.-]+)((?:\s+[\w-]+)*);\s*$/gm var URL = 'https://raw.githubusercontent.com/nginx/nginx/master/conf/mime.types' ;(async function () { - const res = await got(URL) + const res = await request(URL) var json = {} var match = null + var body = await res.body.text() TYPE_LINE_REGEXP.index = 0 - while ((match = TYPE_LINE_REGEXP.exec(res.body))) { + while ((match = TYPE_LINE_REGEXP.exec(body))) { var mime = match[1] // parse the extensions diff --git a/scripts/lib/request.js b/scripts/lib/request.js new file mode 100644 index 000000000..09db2a85c --- /dev/null +++ b/scripts/lib/request.js @@ -0,0 +1,9 @@ +const { request } = require('undici') + +// Undici does not send the user-agent header, which causes the request to fail for the IANA script. +module.exports.request = async function (url, options) { + return request(url, { + ...options, + headers: { 'User-Agent': 'node' } + }) +}