Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

replace got with undici #352

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

replace got with undici #352

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7cfbfe0
replace got with undici
bjohansebas Dec 13, 2024
973c43d
Update scripts/fetch-iana.js
bjohansebas Jan 6, 2025
e7548fb
reduce code
bjohansebas Jan 7, 2025
077dd84
rename file
bjohansebas Jan 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,11 @@
"eslint-plugin-node": "11.1.0",
"eslint-plugin-promise": "6.1.1",
"eslint-plugin-standard": "4.1.0",
"got": "11.8.6",
"media-typer": "1.1.0",
"mocha": "10.2.0",
"nyc": "15.1.0",
"stream-to-array": "2.3.0"
"stream-to-array": "2.3.0",
"undici": "7.1.0"
},
"files": [
"HISTORY.md",
Expand Down
7 changes: 4 additions & 3 deletions scripts/fetch-apache.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@
* Convert these text files to JSON for browser usage.
*/

var got = require('got')
var writedb = require('./lib/write-db')
var { request } = require('./utils')

/**
* Mime types and associated extensions are stored in the form:
Expand All @@ -32,14 +32,15 @@ var TYPE_LINE_REGEXP = /^(?:# )?([\w-]+\/[\w+.-]+)((?:\s+[\w-]+)*)$/gm
var URL = 'https://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types'

;(async function () {
const res = await got(URL)
const res = await request(URL)

var json = {}
var match = null
var body = await res.body.text()

TYPE_LINE_REGEXP.index = 0

while ((match = TYPE_LINE_REGEXP.exec(res.body))) {
while ((match = TYPE_LINE_REGEXP.exec(body))) {
var mime = match[1]

if (mime.slice(-8) === '/example') {
Expand Down
20 changes: 11 additions & 9 deletions scripts/fetch-iana.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
* Convert the IANA definitions from CSV to local.
*/

var got = require('got')
var { request } = require('./utils')
var parser = require('csv-parse')
var toArray = require('stream-to-array')
var typer = require('media-typer')
Expand Down Expand Up @@ -93,15 +93,16 @@ async function addTemplateData (data, options) {
if (!data.template) {
return
}

let res = await got('https://www.iana.org/assignments/media-types/' + data.template)
let url = 'https://www.iana.org/assignments/media-types/' + data.template
let res = await request(url)
var ref = data.type + '/' + data.name
var rfc = getRfcReferences(data.reference)[0]

if (res.statusCode === 404 && data.template !== ref) {
console.log('template ' + data.template + ' not found, retry as ' + ref)
data.template = ref
res = await got('https://www.iana.org/assignments/media-types/' + ref)
url = 'https://www.iana.org/assignments/media-types/' + ref
res = await request(url)

// replace the guessed mime
if (res.statusCode === 200) {
Expand All @@ -111,7 +112,8 @@ async function addTemplateData (data, options) {

if (res.statusCode === 404 && rfc !== undefined) {
console.log('template ' + data.template + ' not found, fetch ' + rfc)
res = await got('https://tools.ietf.org/rfc/' + rfc.toLowerCase() + '.txt')
url = 'https://www.rfc-editor.org/rfc/' + rfc.toLowerCase() + '.txt'
res = await request(url)
}

if (res.statusCode === 404) {
Expand All @@ -123,11 +125,11 @@ async function addTemplateData (data, options) {
throw new Error('got status code ' + res.statusCode + ' from template ' + data.template)
}

var body = getTemplateBody(res.body)
var body = getTemplateBody(await res.body.text())
var mime = extractTemplateMime(body)

// add the template as a source
addSource(data, res.url)
Copy link
Member

@jonchurch jonchurch Jan 7, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Im not sure that we are getting redirects from IANA, but the two previous http clients being used would follow redirects if we got them. So the res.url here was functional in that sense, it would resolve to whatever location the server redirected us to vs the url we initiated a request to.

Something to keep in mind here, if we did get a redirect we'd break or otherwise drop the data. Hopefully we'd notice that in the diff.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Undici does not provide the url property, so I had to create a variable and update it manually to replicate the behavior.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We're also no longer receiving redirects since I changed the URL that was causing the redirect in:

https://github.com/jshttp/mime-db/blob/e7548fb503f2f1015026c09012b47e2abe7a82a6/scripts/fetch-iana.js#L113-115

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jonchurch do you consider this resolved? If so I would like to merge this.

addSource(data, url)

if (mimeEql(mime, data.mime)) {
// use extracted mime
Expand Down Expand Up @@ -213,13 +215,13 @@ function extractTemplateExtensions (body) {
}

async function get (type, options) {
const res = await got('https://www.iana.org/assignments/media-types/' + encodeURIComponent(type) + '.csv')
const res = await request('https://www.iana.org/assignments/media-types/' + encodeURIComponent(type) + '.csv')

if (res.statusCode !== 200) {
throw new Error('got status code ' + res.statusCode + ' from ' + type)
}

const mimes = await toArray(parser(res.body))
const mimes = await toArray(parser(await res.body.text()))
bjohansebas marked this conversation as resolved.
Show resolved Hide resolved
var headers = mimes.shift().map(normalizeHeader)
var reduceRows = generateRowMapper(headers)
const results = []
Expand Down
7 changes: 4 additions & 3 deletions scripts/fetch-nginx.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
* Convert these text files to JSON for browser usage.
*/

var got = require('got')
var { request } = require('./utils')
var writedb = require('./lib/write-db')

/**
Expand All @@ -30,14 +30,15 @@ var TYPE_LINE_REGEXP = /^\s*([\w-]+\/[\w+.-]+)((?:\s+[\w-]+)*);\s*$/gm
var URL = 'https://raw.githubusercontent.com/nginx/nginx/master/conf/mime.types'

;(async function () {
const res = await got(URL)
const res = await request(URL)

var json = {}
var match = null
var body = await res.body.text()

TYPE_LINE_REGEXP.index = 0

while ((match = TYPE_LINE_REGEXP.exec(res.body))) {
while ((match = TYPE_LINE_REGEXP.exec(body))) {
var mime = match[1]

// parse the extensions
Expand Down
10 changes: 10 additions & 0 deletions scripts/utils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
const { request } = require('undici')
bjohansebas marked this conversation as resolved.
Show resolved Hide resolved

module.exports.request = async function (url, options) {
const res = await request(url, {
bjohansebas marked this conversation as resolved.
Show resolved Hide resolved
...options,
headers: { 'User-Agent': 'node' }
})

return res
}
Loading