-
-
Notifications
You must be signed in to change notification settings - Fork 367
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add onUploadProgress option #632
Open
jadedevin13
wants to merge
16
commits into
sindresorhus:main
Choose a base branch
from
jadedevin13:onUploadProgress
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from 14 commits
Commits
Show all changes
16 commits
Select commit
Hold shift + click to select a range
2879f64
Add onUploadProgress option
jadedevin13 ca5bc8a
Add onUploadProgress to registry
jadedevin13 80f0b4e
Update to use globalThis.Response to getReader instead of Blob
jadedevin13 7ba8edc
Update onUploadProgress to report done progress
jadedevin13 e612cc4
Reuse DownloadProgress to Progress
jadedevin13 682d34e
update progress type in test
jadedevin13 3b1a25f
add doc for onUploadProgress
jadedevin13 8590ec2
add tests for upload progress
jadedevin13 d214b68
fix type error for getTotalBytes
jadedevin13 d18f532
fix type error for getTotalBytes
jadedevin13 dcfc115
remove log for test in uploadprogress
jadedevin13 d6283a2
Change error message for request stream onUploadProgress stream support
jadedevin13 0b2ed43
Update options.ts
sindresorhus 7f8aec3
Update readme.md
sindresorhus fdd668d
Update Ky.ts
jadedevin13 10d4cd8
Update Ky.ts
jadedevin13 File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -74,7 +74,7 @@ export class Ky { | |
throw new Error('Streams are not supported in your environment. `ReadableStream` is missing.'); | ||
} | ||
|
||
return ky._stream(response.clone(), ky._options.onDownloadProgress); | ||
return ky._streamResponse(response.clone(), ky._options.onDownloadProgress); | ||
} | ||
|
||
return response; | ||
|
@@ -203,6 +203,24 @@ export class Ky { | |
// The spread of `this.request` is required as otherwise it misses the `duplex` option for some reason and throws. | ||
this.request = new globalThis.Request(new globalThis.Request(url, {...this.request}), this._options as RequestInit); | ||
} | ||
|
||
// Add onUploadProgress handling | ||
if (this._options.onUploadProgress && typeof this._options.onUploadProgress === 'function') { | ||
if (!supportsRequestStreams) { | ||
throw new Error('Request streams are not supported in your environment. The `duplex` option for `Request` is not available.'); | ||
} | ||
|
||
const originalBody = this.request.body; | ||
if (originalBody) { | ||
const totalBytes = this._getTotalBytes(this._options.body); | ||
this.request | ||
= new globalThis.Request(this._input, { | ||
...this._options, | ||
body: this._streamRequest( | ||
originalBody, totalBytes, this._options.onUploadProgress), | ||
}); | ||
} | ||
} | ||
} | ||
|
||
protected _calculateRetryDelay(error: unknown) { | ||
|
@@ -310,7 +328,7 @@ export class Ky { | |
} | ||
|
||
/* istanbul ignore next */ | ||
protected _stream(response: Response, onDownloadProgress: Options['onDownloadProgress']) { | ||
protected _streamResponse(response: Response, onDownloadProgress: Options['onDownloadProgress']) { | ||
const totalBytes = Number(response.headers.get('content-length')) || 0; | ||
let transferredBytes = 0; | ||
|
||
|
@@ -365,4 +383,99 @@ export class Ky { | |
}, | ||
); | ||
} | ||
|
||
// eslint-disable-next-line @typescript-eslint/ban-types | ||
protected _getTotalBytes(body?: globalThis.BodyInit | null): number { | ||
if (!body) { | ||
return 0; | ||
} | ||
|
||
if (body instanceof globalThis.FormData) { | ||
// This is an approximation, as FormData size calculation is not straightforward | ||
let size = 0; | ||
// eslint-disable-next-line unicorn/no-array-for-each -- FormData uses forEach method | ||
body.forEach((value: globalThis.FormDataEntryValue, key: string) => { | ||
if (typeof value === 'string') { | ||
size += new globalThis.TextEncoder().encode(value).length; | ||
} else if (typeof value === 'object' && value !== null && 'size' in value) { | ||
// This catches File objects as well, as File extends Blob | ||
size += (value as Blob).size; | ||
} | ||
|
||
// Add some bytes for field name and multipart boundaries | ||
size += new TextEncoder().encode(key).length + 40; // 40 is an approximation for multipart overhead | ||
}); | ||
|
||
return size; | ||
} | ||
|
||
if (body instanceof globalThis.Blob) { | ||
return body.size; | ||
} | ||
|
||
if (body instanceof globalThis.ArrayBuffer) { | ||
return body.byteLength; | ||
} | ||
|
||
if (typeof body === 'string') { | ||
return new globalThis.TextEncoder().encode(body).length; | ||
} | ||
|
||
if (body instanceof URLSearchParams) { | ||
return new globalThis.TextEncoder().encode(body.toString()).length; | ||
} | ||
|
||
if ('byteLength' in body) { | ||
return (body).byteLength; | ||
} | ||
|
||
if (typeof body === 'object' && body !== null) { | ||
try { | ||
const jsonString = JSON.stringify(body); | ||
return new TextEncoder().encode(jsonString).length; | ||
} catch (error) { | ||
console.warn('Unable to stringify object:', error); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. As a library, it's generally considered bad practice for Ky to output non-errors to the console by default. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. removed |
||
return 0; | ||
} | ||
} | ||
|
||
return 0; // Default case, unable to determine size | ||
} | ||
|
||
protected _streamRequest( | ||
body: BodyInit, | ||
totalBytes: number, | ||
onUploadProgress: (progress: {percent: number; transferredBytes: number; totalBytes: number}) => void, | ||
): globalThis.ReadableStream<Uint8Array> { | ||
let transferredBytes = 0; | ||
|
||
return new globalThis.ReadableStream({ | ||
async start(controller) { | ||
const reader = body instanceof globalThis.ReadableStream ? body.getReader() : new globalThis.Response(body).body!.getReader(); | ||
|
||
async function read() { | ||
const {done, value} = await reader.read(); | ||
if (done) { | ||
// Ensure 100% progress is reported when the upload is complete | ||
onUploadProgress({percent: 1, transferredBytes, totalBytes: Math.max(totalBytes, transferredBytes)}); | ||
controller.close(); | ||
return; | ||
} | ||
|
||
transferredBytes += value.byteLength as number; | ||
let percent = totalBytes === 0 ? 0 : transferredBytes / totalBytes; | ||
if (totalBytes < transferredBytes || percent === 1) { | ||
percent = 0.99; | ||
} | ||
|
||
onUploadProgress({percent: Number(percent.toFixed(2)), transferredBytes, totalBytes}); | ||
|
||
controller.enqueue(value); | ||
await read(); | ||
} | ||
|
||
await read(); | ||
}, | ||
}); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
// Helper function to create a large Blob | ||
export function createLargeBlob(sizeInMB: number): Blob { | ||
const chunkSize = 1024 * 1024; // 1MB | ||
// eslint-disable-next-line unicorn/no-new-array | ||
const chunks = new Array(sizeInMB).fill('x'.repeat(chunkSize)); | ||
return new Blob(chunks, {type: 'application/octet-stream'}); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,3 +1,4 @@ | ||
export * from './create-http-test-server.js'; | ||
export * from './parse-body.js'; | ||
export * from './with-page.js'; | ||
export * from './create-large-file.js'; |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The TS doc comments and readme should be in sync.