diff --git a/modules/xmpp/JingleSessionPC.js b/modules/xmpp/JingleSessionPC.js index 38dad11c94..eff72edae8 100644 --- a/modules/xmpp/JingleSessionPC.js +++ b/modules/xmpp/JingleSessionPC.js @@ -817,310 +817,588 @@ export default class JingleSessionPC extends JingleSession { } /** - * Accepts incoming Jingle 'session-initiate' and should send 'session-accept' in result. + * Sends 'content-modify' IQ in order to ask the remote peer to either stop or resume sending video media or to + * adjust sender's video constraints. * - * @param jingleOffer jQuery selector pointing to the jingle element of the offer IQ - * @param success callback called when we accept incoming session successfully and receive RESULT packet to - * 'session-accept' sent. - * @param failure function(error) called if for any reason we fail to accept the incoming offer. 'error' argument - * can be used to log some details about the error. - * @param {Array} [localTracks] the optional list of the local tracks that will be added, before - * the offer/answer cycle executes. We allow the localTracks to optionally be passed in so that the addition of the - * local tracks and the processing of the initial offer can all be done atomically. We want to make sure that any - * other operations which originate in the XMPP Jingle messages related with this session to be executed with an - * assumption that the initial offer/answer cycle has been executed already. + * @returns {void} + * @private */ - acceptOffer(jingleOffer, success, failure, localTracks = []) { - this.setOfferAnswerCycle( - jingleOffer, - () => { - // FIXME we may not care about RESULT packet for session-accept - // then we should either call 'success' here immediately or - // modify sendSessionAccept method to do that - this.sendSessionAccept(() => { - // Start processing tasks on the modification queue. - logger.debug(`${this} Resuming the modification queue after session is established!`); - this.modificationQueue.resume(); - - success(); - this.room.eventEmitter.emit(XMPPEvents.SESSION_ACCEPT, this); - - // The first video track is added to the peerconnection and signaled as part of the session-accept. - // Add secondary video tracks (that were already added to conference) to the peerconnection here. - // This will happen when someone shares a secondary source to a two people call, the other user - // leaves and joins the call again, a new peerconnection is created for p2p/jvb connection. At this - // point, there are 2 video tracks which need to be signaled to the remote peer. - const videoTracks = localTracks.filter(track => track.getType() === MediaType.VIDEO); - - videoTracks.length && videoTracks.splice(0, 1); - videoTracks.length && this.addTracks(videoTracks); - }, - error => { - failure(error); - this.room.eventEmitter.emit(XMPPEvents.SESSION_ACCEPT_ERROR, this, error); + _sendContentModify() { + const senders = this._localSendReceiveVideoActive ? 'both' : 'none'; + const sessionModify + = $iq({ + to: this.remoteJid, + type: 'set' + }) + .c('jingle', { + xmlns: 'urn:xmpp:jingle:1', + action: 'content-modify', + initiator: this.initiatorJid, + sid: this.sid + }) + .c('content', { + name: MediaType.VIDEO, + senders }); - }, - failure, - localTracks); - } - /** - * {@inheritDoc} - */ - addIceCandidates(elem) { - if (this.peerconnection.signalingState === 'closed') { - logger.warn(`${this} Ignored add ICE candidate when in closed state`); + if (typeof this._sourceReceiverConstraints !== 'undefined') { + this._sourceReceiverConstraints.forEach((maxHeight, sourceName) => { + sessionModify + .c('source-frame-height', { xmlns: 'http://jitsi.org/jitmeet/video' }) + .attrs({ + sourceName, + maxHeight + }); - return; + sessionModify.up(); + logger.info(`${this} sending content-modify for source-name: ${sourceName}, maxHeight: ${maxHeight}`); + }); } - const iceCandidates = []; - - elem.find('>content>transport>candidate') - .each((idx, candidate) => { - let line = SDPUtil.candidateFromJingle(candidate); - - line = line.replace('\r\n', '').replace('a=', ''); - - // FIXME this code does not care to handle - // non-bundle transport - const rtcCandidate = new RTCIceCandidate({ - sdpMLineIndex: 0, + logger.debug(sessionModify.tree()); - // FF comes up with more complex names like audio-23423, - // Given that it works on both Chrome and FF without - // providing it, let's leave it like this for the time - // being... - // sdpMid: 'audio', - sdpMid: '', - candidate: line - }); + this.connection.sendIQ( + sessionModify, + null, + this.newJingleErrorHandler(sessionModify), + IQ_TIMEOUT); + } - iceCandidates.push(rtcCandidate); - }); + /** + * Sends given candidate in Jingle 'transport-info' message. + * + * @param {RTCIceCandidate} candidate the WebRTC ICE candidate instance + * @returns {void} + * @private + */ + _sendIceCandidate(candidate) { + const localSDP = new SDP(this.peerconnection.localDescription.sdp); - if (!iceCandidates.length) { - logger.error(`${this} No ICE candidates to add ?`, elem[0] && elem[0].outerHTML); + if (candidate && candidate.candidate.length && !this.lasticecandidate) { + const ice = SDPUtil.iceparams(localSDP.media[candidate.sdpMLineIndex], localSDP.session); + const jcand = SDPUtil.candidateToJingle(candidate.candidate); - return; - } + if (!(ice && jcand)) { + logger.error('failed to get ice && jcand'); - // We want to have this task queued, so that we know it is executed, - // after the initial sRD/sLD offer/answer cycle was done (based on - // the assumption that candidates are spawned after the offer/answer - // and XMPP preserves order). - const workFunction = finishedCallback => { - for (const iceCandidate of iceCandidates) { - this.peerconnection.addIceCandidate(iceCandidate) - .then( - () => logger.debug(`${this} addIceCandidate ok!`), - err => logger.error(`${this} addIceCandidate failed!`, err)); + return; } + ice.xmlns = XEP.ICE_UDP_TRANSPORT; - finishedCallback(); - logger.debug(`${this} ICE candidates task finished`); - }; + if (this.usedrip) { + if (this.dripContainer.length === 0) { + setTimeout(() => { + if (this.dripContainer.length === 0) { + return; + } + this._sendIceCandidates(this.dripContainer); + this.dripContainer = []; + }, ICE_CAND_GATHERING_TIMEOUT); + } + this.dripContainer.push(candidate); + } else { + this._sendIceCandidates([ candidate ]); + } + } else { + logger.log(`${this} _sendIceCandidate: last candidate`); - logger.debug(`${this} Queued add (${iceCandidates.length}) ICE candidates task`); - this.modificationQueue.push(workFunction); + // FIXME: remember to re-think in ICE-restart + this.lasticecandidate = true; + } } /** - * Handles a Jingle source-add message for this Jingle session. + * Sends given candidates in Jingle 'transport-info' message. * - * @param {Array} elem an array of Jingle "content" elements. - * @returns {Promise} resolved when the operation is done or rejected with an error. + * @param {Array} candidates an array of the WebRTC ICE candidate instances. + * @returns {void} + * @private */ - addRemoteStream(elem) { - this._addOrRemoveRemoteStream(true /* add */, elem); - } + _sendIceCandidates(candidates) { + if (!this._assertNotEnded('_sendIceCandidates')) { - /** - * Adds a new track to the peerconnection. This method needs to be called only when a secondary JitsiLocalTrack is - * being added to the peerconnection for the first time. - * - * @param {Array} localTracks - Tracks to be added to the peer connection. - * @returns {Promise} that resolves when the track is successfully added to the peerconnection, rejected - * otherwise. - */ - addTracks(localTracks = null) { - if (!localTracks?.length) { - Promise.reject(new Error('No tracks passed')); + return; } - if (localTracks.find(track => track.getType() !== MediaType.VIDEO)) { - return Promise.reject(new Error('Multiple tracks of the given media type are not supported')); - } + logger.log(`${this} _sendIceCandidates ${JSON.stringify(candidates)}`); + const cand = $iq({ to: this.remoteJid, + type: 'set' }) + .c('jingle', { xmlns: 'urn:xmpp:jingle:1', + action: 'transport-info', + initiator: this.initiatorJid, + sid: this.sid }); - const replaceTracks = []; - const workFunction = finishedCallback => { - const remoteSdp = new SDP(this.peerconnection.peerconnection.remoteDescription.sdp); - const recvOnlyTransceiver = this.peerconnection.peerconnection.getTransceivers() - .find(t => t.receiver.track.kind === MediaType.VIDEO - && t.direction === MediaDirection.RECVONLY - && t.currentDirection === MediaDirection.RECVONLY); + const localSDP = new SDP(this.peerconnection.localDescription.sdp); - // Add transceivers by adding a new mline in the remote description for each track. Do not create a new - // m-line if a recv-only transceiver exists in the p2p case. The new track will be attached to the - // existing one in that case. - for (const track of localTracks) { - if (!this.isP2P || !recvOnlyTransceiver) { - remoteSdp.addMlineForNewLocalSource(track.getType()); - } - } + for (let mid = 0; mid < localSDP.media.length; mid++) { + const cands = candidates.filter(el => el.sdpMLineIndex === mid); + const mline + = SDPUtil.parseMLine(localSDP.media[mid].split('\r\n')[0]); - this._renegotiate(remoteSdp.raw) - .then(() => { - // Replace the tracks on the newly generated transceivers. - for (const track of localTracks) { - replaceTracks.push(this.peerconnection.replaceTrack(null, track)); - } + if (cands.length > 0) { + const ice + = SDPUtil.iceparams(localSDP.media[mid], localSDP.session); - return Promise.all(replaceTracks); - }) + ice.xmlns = XEP.ICE_UDP_TRANSPORT; + cand.c('content', { + creator: this.initiatorJid === this.localJid + ? 'initiator' : 'responder', + name: cands[0].sdpMid ? cands[0].sdpMid : mline.media + }).c('transport', ice); + for (let i = 0; i < cands.length; i++) { + const candidate + = SDPUtil.candidateToJingle(cands[i].candidate); - // Trigger a renegotiation here since renegotiations are suppressed at TPC.replaceTrack for screenshare - // tracks. This is done here so that presence for screenshare tracks is sent before signaling. - .then(() => this._renegotiate()) - .then(() => finishedCallback(), error => finishedCallback(error)); - }; + // Mangle ICE candidate if 'failICE' test option is enabled - return new Promise((resolve, reject) => { - logger.debug(`${this} Queued renegotiation after addTrack`); + if (this.failICE) { + candidate.ip = '1.1.1.1'; + } + cand.c('candidate', candidate).up(); + } - this.modificationQueue.push( - workFunction, - error => { - if (error) { - if (error instanceof ClearedQueueError) { - // The session might have been terminated before the task was executed, making it obsolete. - logger.debug(`${this} renegotiation after addTrack aborted: session terminated`); - resolve(); + // add fingerprint + const fingerprintLine + = SDPUtil.findLine( + localSDP.media[mid], + 'a=fingerprint:', localSDP.session); - return; - } - logger.error(`${this} renegotiation after addTrack error`, error); - reject(error); - } else { - logger.debug(`${this} renegotiation after addTrack executed - OK`); - resolve(); - } - }); - }); - } + if (fingerprintLine) { + const tmp = SDPUtil.parseFingerprint(fingerprintLine); - /** - * Adds local track back to the peerconnection associated with this session. - * - * @param {JitsiLocalTrack} track - the local track to be added back to the peerconnection. - * @return {Promise} a promise that will resolve once the local track is added back to this session and - * renegotiation succeeds (if its warranted). Will be rejected with a string that provides some error - * details in case something goes wrong. - * @returns {Promise} - */ - addTrackToPc(track) { - return this._addRemoveTrack(false /* add */, track) - .then(() => { - // Configure the video encodings after the track is unmuted. If the user joins the call muted and - // unmutes it the first time, all the parameters need to be configured. - if (track.isVideoTrack()) { - return this.peerconnection.configureVideoSenderEncodings(track); + tmp.required = true; + cand.c( + 'fingerprint', + { xmlns: 'urn:xmpp:jingle:apps:dtls:0' }) + .t(tmp.fingerprint); + delete tmp.fingerprint; + cand.attrs(tmp); + cand.up(); } - }); + cand.up(); // transport + cand.up(); // content + } + } + + // might merge last-candidate notification into this, but it is called + // a lot later. See webrtc issue #2340 + // logger.log('was this the last candidate', this.lasticecandidate); + this.connection.sendIQ( + cand, null, this.newJingleErrorHandler(cand), IQ_TIMEOUT); } /** - * Closes the underlying peerconnection and shuts down the modification queue. + * Sends Jingle 'session-accept' message. * + * @param {function()} success callback called when we receive 'RESULT' packet for the 'session-accept'. + * @param {function(error)} failure called when we receive an error response or when the request has timed out. * @returns {void} + * @private */ - close() { - this.state = JingleSessionState.ENDED; - this.establishmentDuration = undefined; + _sendSessionAccept(success, failure) { + // NOTE: since we're just reading from it, we don't need to be within + // the modification queue to access the local description + const localSDP = new SDP(this.peerconnection.localDescription.sdp, this.isP2P); + const accept = $iq({ to: this.remoteJid, + type: 'set' }) + .c('jingle', { xmlns: 'urn:xmpp:jingle:1', + action: 'session-accept', + initiator: this.initiatorJid, + responder: this.responderJid, + sid: this.sid }); - if (this.peerconnection) { - this.peerconnection.onicecandidate = null; - this.peerconnection.oniceconnectionstatechange = null; - this.peerconnection.onnegotiationneeded = null; - this.peerconnection.onsignalingstatechange = null; + if (this.webrtcIceTcpDisable) { + localSDP.removeTcpCandidates = true; } + if (this.webrtcIceUdpDisable) { + localSDP.removeUdpCandidates = true; + } + if (this.failICE) { + localSDP.failICE = true; + } + if (typeof this.options.channelLastN === 'number' && this.options.channelLastN >= 0) { + localSDP.initialLastN = this.options.channelLastN; + } + localSDP.toJingle( + accept, + this.initiatorJid === this.localJid ? 'initiator' : 'responder'); - logger.debug(`${this} Clearing modificationQueue`); - - // Remove any pending tasks from the queue - this.modificationQueue.clear(); - - logger.debug(`${this} Queued PC close task`); - this.modificationQueue.push(finishCallback => { - // do not try to close if already closed. - this.peerconnection && this.peerconnection.close(); - finishCallback(); - logger.debug(`${this} PC close task done!`); - }); + logger.info(`${this} Sending session-accept`); + logger.debug(accept.tree()); + this.connection.sendIQ(accept, + success, + this.newJingleErrorHandler(accept, error => { + failure(error); - logger.debug(`${this} Shutdown modificationQueue!`); + // 'session-accept' is a critical timeout and we'll + // have to restart + this.room.eventEmitter.emit( + XMPPEvents.SESSION_ACCEPT_TIMEOUT, this); + }), + IQ_TIMEOUT); - // No more tasks can go in after the close task - this.modificationQueue.shutdown(); + // XXX Videobridge needs WebRTC's answer (ICE ufrag and pwd, DTLS + // fingerprint and setup) ASAP in order to start the connection + // establishment. + // + // FIXME Flushing the connection at this point triggers an issue with + // BOSH request handling in Prosody on slow connections. + // + // The problem is that this request will be quite large and it may take + // time before it reaches Prosody. In the meantime Strophe may decide + // to send the next one. And it was observed that a small request with + // 'transport-info' usually follows this one. It does reach Prosody + // before the previous one was completely received. 'rid' on the server + // is increased and Prosody ignores the request with 'session-accept'. + // It will never reach Jicofo and everything in the request table is + // lost. Removing the flush does not guarantee it will never happen, but + // makes it much less likely('transport-info' is bundled with + // 'session-accept' and any immediate requests). + // + // this.connection.flush(); } /** - * @inheritDoc - * @param {JingleSessionPCOptions} options - a set of config options. + * Sends 'session-initiate' to the remote peer. + * + * NOTE this method is synchronous and we're not waiting for the RESULT + * response which would delay the startup process. + * + * @param {string} offerSdp - The local session description which will be used to generate an offer. * @returns {void} + * @private */ - doInitialize(options) { - this.failICE = Boolean(options.failICE); - this.lasticecandidate = false; - this.options = options; - - /** - * {@code true} if reconnect is in progress. - * @type {boolean} - */ - this.isReconnect = false; + _sendSessionInitiate(offerSdp) { + let init = $iq({ + to: this.remoteJid, + type: 'set' + }).c('jingle', { + xmlns: 'urn:xmpp:jingle:1', + action: 'session-initiate', + initiator: this.initiatorJid, + sid: this.sid + }); - /** - * Set to {@code true} if the connection was ever stable - * @type {boolean} - */ - this.wasstable = false; - this.webrtcIceUdpDisable = Boolean(options.webrtcIceUdpDisable); - this.webrtcIceTcpDisable = Boolean(options.webrtcIceTcpDisable); + new SDP(offerSdp, this.isP2P).toJingle( + init, + this.isInitiator ? 'initiator' : 'responder'); + init = init.tree(); + logger.debug(`${this} Session-initiate: `, init); + this.connection.sendIQ(init, + () => { + logger.info(`${this} Got RESULT for "session-initiate"`); + }, + error => { + logger.error(`${this} "session-initiate" error`, error); + }, + IQ_TIMEOUT); + } - const pcOptions = { disableRtx: options.disableRtx }; + /** + * Accepts incoming Jingle 'session-initiate' and should send 'session-accept' in result. + * + * @param jingleOffer jQuery selector pointing to the jingle element of the offer IQ + * @param success callback called when we accept incoming session successfully and receive RESULT packet to + * 'session-accept' sent. + * @param failure function(error) called if for any reason we fail to accept the incoming offer. 'error' argument + * can be used to log some details about the error. + * @param {Array} [localTracks] the optional list of the local tracks that will be added, before + * the offer/answer cycle executes. We allow the localTracks to optionally be passed in so that the addition of the + * local tracks and the processing of the initial offer can all be done atomically. We want to make sure that any + * other operations which originate in the XMPP Jingle messages related with this session to be executed with an + * assumption that the initial offer/answer cycle has been executed already. + */ + acceptOffer(jingleOffer, success, failure, localTracks = []) { + this.setOfferAnswerCycle( + jingleOffer, + () => { + // FIXME we may not care about RESULT packet for session-accept + // then we should either call 'success' here immediately or + // modify sendSessionAccept method to do that + this._sendSessionAccept(() => { + // Start processing tasks on the modification queue. + logger.debug(`${this} Resuming the modification queue after session is established!`); + this.modificationQueue.resume(); - if (options.gatherStats) { - pcOptions.maxstats = DEFAULT_MAX_STATS; - } - pcOptions.capScreenshareBitrate = false; - pcOptions.codecSettings = options.codecSettings; - pcOptions.enableInsertableStreams = options.enableInsertableStreams; - pcOptions.usesCodecSelectionAPI = this.usesCodecSelectionAPI - = browser.supportsCodecSelectionAPI() && options.testing?.enableCodecSelectionAPI && !this.isP2P; + success(); + this.room.eventEmitter.emit(XMPPEvents.SESSION_ACCEPT, this); - if (options.videoQuality) { - const settings = Object.entries(options.videoQuality) - .map(entry => { - entry[0] = entry[0].toLowerCase(); + // The first video track is added to the peerconnection and signaled as part of the session-accept. + // Add secondary video tracks (that were already added to conference) to the peerconnection here. + // This will happen when someone shares a secondary source to a two people call, the other user + // leaves and joins the call again, a new peerconnection is created for p2p/jvb connection. At this + // point, there are 2 video tracks which need to be signaled to the remote peer. + const videoTracks = localTracks.filter(track => track.getType() === MediaType.VIDEO); - return entry; - }); + videoTracks.length && videoTracks.splice(0, 1); + videoTracks.length && this.addTracks(videoTracks); + }, + error => { + failure(error); + this.room.eventEmitter.emit(XMPPEvents.SESSION_ACCEPT_ERROR, this, error); + }); + }, + failure, + localTracks); + } - pcOptions.videoQuality = Object.fromEntries(settings); - } - pcOptions.forceTurnRelay = options.forceTurnRelay; - pcOptions.audioQuality = options.audioQuality; - pcOptions.disableSimulcast = this.isP2P ? true : options.disableSimulcast; + /** + * {@inheritDoc} + */ + addIceCandidates(elem) { + if (this.peerconnection.signalingState === 'closed') { + logger.warn(`${this} Ignored add ICE candidate when in closed state`); - if (!this.isP2P) { - // Do not send lower spatial layers for low fps screenshare and enable them only for high fps screenshare. - pcOptions.capScreenshareBitrate = !(options.desktopSharingFrameRate?.max > SS_DEFAULT_FRAME_RATE); + return; } - if (options.startSilent) { - pcOptions.startSilent = true; - } + const iceCandidates = []; + + elem.find('>content>transport>candidate') + .each((idx, candidate) => { + let line = SDPUtil.candidateFromJingle(candidate); + + line = line.replace('\r\n', '').replace('a=', ''); + + // FIXME this code does not care to handle + // non-bundle transport + const rtcCandidate = new RTCIceCandidate({ + sdpMLineIndex: 0, + + // FF comes up with more complex names like audio-23423, + // Given that it works on both Chrome and FF without + // providing it, let's leave it like this for the time + // being... + // sdpMid: 'audio', + sdpMid: '', + candidate: line + }); + + iceCandidates.push(rtcCandidate); + }); + + if (!iceCandidates.length) { + logger.error(`${this} No ICE candidates to add ?`, elem[0] && elem[0].outerHTML); + + return; + } + + // We want to have this task queued, so that we know it is executed, + // after the initial sRD/sLD offer/answer cycle was done (based on + // the assumption that candidates are spawned after the offer/answer + // and XMPP preserves order). + const workFunction = finishedCallback => { + for (const iceCandidate of iceCandidates) { + this.peerconnection.addIceCandidate(iceCandidate) + .then( + () => logger.debug(`${this} addIceCandidate ok!`), + err => logger.error(`${this} addIceCandidate failed!`, err)); + } + + finishedCallback(); + logger.debug(`${this} ICE candidates task finished`); + }; + + logger.debug(`${this} Queued add (${iceCandidates.length}) ICE candidates task`); + this.modificationQueue.push(workFunction); + } + + /** + * Handles a Jingle source-add message for this Jingle session. + * + * @param {Array} elem an array of Jingle "content" elements. + * @returns {Promise} resolved when the operation is done or rejected with an error. + */ + addRemoteStream(elem) { + this._addOrRemoveRemoteStream(true /* add */, elem); + } + + /** + * Adds a new track to the peerconnection. This method needs to be called only when a secondary JitsiLocalTrack is + * being added to the peerconnection for the first time. + * + * @param {Array} localTracks - Tracks to be added to the peer connection. + * @returns {Promise} that resolves when the track is successfully added to the peerconnection, rejected + * otherwise. + */ + addTracks(localTracks = null) { + if (!localTracks?.length) { + Promise.reject(new Error('No tracks passed')); + } + + if (localTracks.find(track => track.getType() !== MediaType.VIDEO)) { + return Promise.reject(new Error('Multiple tracks of the given media type are not supported')); + } + + const replaceTracks = []; + const workFunction = finishedCallback => { + const remoteSdp = new SDP(this.peerconnection.peerconnection.remoteDescription.sdp); + const recvOnlyTransceiver = this.peerconnection.peerconnection.getTransceivers() + .find(t => t.receiver.track.kind === MediaType.VIDEO + && t.direction === MediaDirection.RECVONLY + && t.currentDirection === MediaDirection.RECVONLY); + + // Add transceivers by adding a new mline in the remote description for each track. Do not create a new + // m-line if a recv-only transceiver exists in the p2p case. The new track will be attached to the + // existing one in that case. + for (const track of localTracks) { + if (!this.isP2P || !recvOnlyTransceiver) { + remoteSdp.addMlineForNewLocalSource(track.getType()); + } + } + + this._renegotiate(remoteSdp.raw) + .then(() => { + // Replace the tracks on the newly generated transceivers. + for (const track of localTracks) { + replaceTracks.push(this.peerconnection.replaceTrack(null, track)); + } + + return Promise.all(replaceTracks); + }) + + // Trigger a renegotiation here since renegotiations are suppressed at TPC.replaceTrack for screenshare + // tracks. This is done here so that presence for screenshare tracks is sent before signaling. + .then(() => this._renegotiate()) + .then(() => finishedCallback(), error => finishedCallback(error)); + }; + + return new Promise((resolve, reject) => { + logger.debug(`${this} Queued renegotiation after addTrack`); + + this.modificationQueue.push( + workFunction, + error => { + if (error) { + if (error instanceof ClearedQueueError) { + // The session might have been terminated before the task was executed, making it obsolete. + logger.debug(`${this} renegotiation after addTrack aborted: session terminated`); + resolve(); + + return; + } + logger.error(`${this} renegotiation after addTrack error`, error); + reject(error); + } else { + logger.debug(`${this} renegotiation after addTrack executed - OK`); + resolve(); + } + }); + }); + } + + /** + * Adds local track back to the peerconnection associated with this session. + * + * @param {JitsiLocalTrack} track - the local track to be added back to the peerconnection. + * @return {Promise} a promise that will resolve once the local track is added back to this session and + * renegotiation succeeds (if its warranted). Will be rejected with a string that provides some error + * details in case something goes wrong. + * @returns {Promise} + */ + addTrackToPc(track) { + return this._addRemoveTrack(false /* add */, track) + .then(() => { + // Configure the video encodings after the track is unmuted. If the user joins the call muted and + // unmutes it the first time, all the parameters need to be configured. + if (track.isVideoTrack()) { + return this.peerconnection.configureVideoSenderEncodings(track); + } + }); + } + + /** + * Closes the underlying peerconnection and shuts down the modification queue. + * + * @returns {void} + */ + close() { + this.state = JingleSessionState.ENDED; + this.establishmentDuration = undefined; + + if (this.peerconnection) { + this.peerconnection.onicecandidate = null; + this.peerconnection.oniceconnectionstatechange = null; + this.peerconnection.onnegotiationneeded = null; + this.peerconnection.onsignalingstatechange = null; + } + + logger.debug(`${this} Clearing modificationQueue`); + + // Remove any pending tasks from the queue + this.modificationQueue.clear(); + + logger.debug(`${this} Queued PC close task`); + this.modificationQueue.push(finishCallback => { + // do not try to close if already closed. + this.peerconnection && this.peerconnection.close(); + finishCallback(); + logger.debug(`${this} PC close task done!`); + }); + + logger.debug(`${this} Shutdown modificationQueue!`); + + // No more tasks can go in after the close task + this.modificationQueue.shutdown(); + } + + /** + * @inheritDoc + * @param {JingleSessionPCOptions} options - a set of config options. + * @returns {void} + */ + doInitialize(options) { + this.failICE = Boolean(options.failICE); + this.lasticecandidate = false; + this.options = options; + + /** + * {@code true} if reconnect is in progress. + * @type {boolean} + */ + this.isReconnect = false; + + /** + * Set to {@code true} if the connection was ever stable + * @type {boolean} + */ + this.wasstable = false; + this.webrtcIceUdpDisable = Boolean(options.webrtcIceUdpDisable); + this.webrtcIceTcpDisable = Boolean(options.webrtcIceTcpDisable); + + const pcOptions = { disableRtx: options.disableRtx }; + + if (options.gatherStats) { + pcOptions.maxstats = DEFAULT_MAX_STATS; + } + pcOptions.capScreenshareBitrate = false; + pcOptions.codecSettings = options.codecSettings; + pcOptions.enableInsertableStreams = options.enableInsertableStreams; + pcOptions.usesCodecSelectionAPI = this.usesCodecSelectionAPI + = browser.supportsCodecSelectionAPI() && options.testing?.enableCodecSelectionAPI && !this.isP2P; + + if (options.videoQuality) { + const settings = Object.entries(options.videoQuality) + .map(entry => { + entry[0] = entry[0].toLowerCase(); + + return entry; + }); + + pcOptions.videoQuality = Object.fromEntries(settings); + } + pcOptions.forceTurnRelay = options.forceTurnRelay; + pcOptions.audioQuality = options.audioQuality; + pcOptions.disableSimulcast = this.isP2P ? true : options.disableSimulcast; + + if (!this.isP2P) { + // Do not send lower spatial layers for low fps screenshare and enable them only for high fps screenshare. + pcOptions.capScreenshareBitrate = !(options.desktopSharingFrameRate?.max > SS_DEFAULT_FRAME_RATE); + } + + if (options.startSilent) { + pcOptions.startSilent = true; + } this.peerconnection = this.rtc.createPeerConnection( @@ -1176,7 +1454,7 @@ export default class JingleSessionPC extends JingleSession { this._gatheringReported = true; } if (this.isP2P) { - this.sendIceCandidate(candidate); + this._sendIceCandidate(candidate); } }; @@ -1404,7 +1682,7 @@ export default class JingleSessionPC extends JingleSession { .then(offerSdp => this.peerconnection.setLocalDescription(offerSdp)) .then(() => { this.peerconnection.processLocalSdpForTransceiverInfo(localTracks); - this.sendSessionInitiate(this.peerconnection.localDescription.sdp); + this._sendSessionInitiate(this.peerconnection.localDescription.sdp); }) .then(() => { logger.debug(`${this} invite executed - OK`); @@ -1929,284 +2207,6 @@ export default class JingleSessionPC extends JingleSession { }); } - /** - * Sends 'content-modify' IQ in order to ask the remote peer to either stop or resume sending video media or to - * adjust sender's video constraints. - * - * @returns {void} - * @private - */ - sendContentModify() { - const senders = this._localSendReceiveVideoActive ? 'both' : 'none'; - const sessionModify - = $iq({ - to: this.remoteJid, - type: 'set' - }) - .c('jingle', { - xmlns: 'urn:xmpp:jingle:1', - action: 'content-modify', - initiator: this.initiatorJid, - sid: this.sid - }) - .c('content', { - name: MediaType.VIDEO, - senders - }); - - if (typeof this._sourceReceiverConstraints !== 'undefined') { - this._sourceReceiverConstraints.forEach((maxHeight, sourceName) => { - sessionModify - .c('source-frame-height', { xmlns: 'http://jitsi.org/jitmeet/video' }) - .attrs({ - sourceName, - maxHeight - }); - - sessionModify.up(); - logger.info(`${this} sending content-modify for source-name: ${sourceName}, maxHeight: ${maxHeight}`); - }); - } - - logger.debug(sessionModify.tree()); - - this.connection.sendIQ( - sessionModify, - null, - this.newJingleErrorHandler(sessionModify), - IQ_TIMEOUT); - } - - /** - * Sends given candidate in Jingle 'transport-info' message. - * - * @param {RTCIceCandidate} candidate the WebRTC ICE candidate instance - * @private - * @returns {void} - */ - sendIceCandidate(candidate) { - const localSDP = new SDP(this.peerconnection.localDescription.sdp); - - if (candidate && candidate.candidate.length && !this.lasticecandidate) { - const ice = SDPUtil.iceparams(localSDP.media[candidate.sdpMLineIndex], localSDP.session); - const jcand = SDPUtil.candidateToJingle(candidate.candidate); - - if (!(ice && jcand)) { - logger.error('failed to get ice && jcand'); - - return; - } - ice.xmlns = XEP.ICE_UDP_TRANSPORT; - - if (this.usedrip) { - if (this.dripContainer.length === 0) { - setTimeout(() => { - if (this.dripContainer.length === 0) { - return; - } - this.sendIceCandidates(this.dripContainer); - this.dripContainer = []; - }, ICE_CAND_GATHERING_TIMEOUT); - } - this.dripContainer.push(candidate); - } else { - this.sendIceCandidates([ candidate ]); - } - } else { - logger.log(`${this} sendIceCandidate: last candidate`); - - // FIXME: remember to re-think in ICE-restart - this.lasticecandidate = true; - } - } - - /** - * Sends given candidates in Jingle 'transport-info' message. - * - * @param {Array} candidates an array of the WebRTC ICE candidate instances. - * @returns {void} - * @private - */ - sendIceCandidates(candidates) { - if (!this._assertNotEnded('sendIceCandidates')) { - - return; - } - - logger.log(`${this} sendIceCandidates ${JSON.stringify(candidates)}`); - const cand = $iq({ to: this.remoteJid, - type: 'set' }) - .c('jingle', { xmlns: 'urn:xmpp:jingle:1', - action: 'transport-info', - initiator: this.initiatorJid, - sid: this.sid }); - - const localSDP = new SDP(this.peerconnection.localDescription.sdp); - - for (let mid = 0; mid < localSDP.media.length; mid++) { - const cands = candidates.filter(el => el.sdpMLineIndex === mid); - const mline - = SDPUtil.parseMLine(localSDP.media[mid].split('\r\n')[0]); - - if (cands.length > 0) { - const ice - = SDPUtil.iceparams(localSDP.media[mid], localSDP.session); - - ice.xmlns = XEP.ICE_UDP_TRANSPORT; - cand.c('content', { - creator: this.initiatorJid === this.localJid - ? 'initiator' : 'responder', - name: cands[0].sdpMid ? cands[0].sdpMid : mline.media - }).c('transport', ice); - for (let i = 0; i < cands.length; i++) { - const candidate - = SDPUtil.candidateToJingle(cands[i].candidate); - - // Mangle ICE candidate if 'failICE' test option is enabled - - if (this.failICE) { - candidate.ip = '1.1.1.1'; - } - cand.c('candidate', candidate).up(); - } - - // add fingerprint - const fingerprintLine - = SDPUtil.findLine( - localSDP.media[mid], - 'a=fingerprint:', localSDP.session); - - if (fingerprintLine) { - const tmp = SDPUtil.parseFingerprint(fingerprintLine); - - tmp.required = true; - cand.c( - 'fingerprint', - { xmlns: 'urn:xmpp:jingle:apps:dtls:0' }) - .t(tmp.fingerprint); - delete tmp.fingerprint; - cand.attrs(tmp); - cand.up(); - } - cand.up(); // transport - cand.up(); // content - } - } - - // might merge last-candidate notification into this, but it is called - // a lot later. See webrtc issue #2340 - // logger.log('was this the last candidate', this.lasticecandidate); - this.connection.sendIQ( - cand, null, this.newJingleErrorHandler(cand), IQ_TIMEOUT); - } - - /** - * Sends Jingle 'session-accept' message. - * - * @param {function()} success callback called when we receive 'RESULT' packet for the 'session-accept'. - * @param {function(error)} failure called when we receive an error response or when the request has timed out. - * @returns {void} - * @private - */ - sendSessionAccept(success, failure) { - // NOTE: since we're just reading from it, we don't need to be within - // the modification queue to access the local description - const localSDP = new SDP(this.peerconnection.localDescription.sdp, this.isP2P); - const accept = $iq({ to: this.remoteJid, - type: 'set' }) - .c('jingle', { xmlns: 'urn:xmpp:jingle:1', - action: 'session-accept', - initiator: this.initiatorJid, - responder: this.responderJid, - sid: this.sid }); - - if (this.webrtcIceTcpDisable) { - localSDP.removeTcpCandidates = true; - } - if (this.webrtcIceUdpDisable) { - localSDP.removeUdpCandidates = true; - } - if (this.failICE) { - localSDP.failICE = true; - } - if (typeof this.options.channelLastN === 'number' && this.options.channelLastN >= 0) { - localSDP.initialLastN = this.options.channelLastN; - } - localSDP.toJingle( - accept, - this.initiatorJid === this.localJid ? 'initiator' : 'responder'); - - logger.info(`${this} Sending session-accept`); - logger.debug(accept.tree()); - this.connection.sendIQ(accept, - success, - this.newJingleErrorHandler(accept, error => { - failure(error); - - // 'session-accept' is a critical timeout and we'll - // have to restart - this.room.eventEmitter.emit( - XMPPEvents.SESSION_ACCEPT_TIMEOUT, this); - }), - IQ_TIMEOUT); - - // XXX Videobridge needs WebRTC's answer (ICE ufrag and pwd, DTLS - // fingerprint and setup) ASAP in order to start the connection - // establishment. - // - // FIXME Flushing the connection at this point triggers an issue with - // BOSH request handling in Prosody on slow connections. - // - // The problem is that this request will be quite large and it may take - // time before it reaches Prosody. In the meantime Strophe may decide - // to send the next one. And it was observed that a small request with - // 'transport-info' usually follows this one. It does reach Prosody - // before the previous one was completely received. 'rid' on the server - // is increased and Prosody ignores the request with 'session-accept'. - // It will never reach Jicofo and everything in the request table is - // lost. Removing the flush does not guarantee it will never happen, but - // makes it much less likely('transport-info' is bundled with - // 'session-accept' and any immediate requests). - // - // this.connection.flush(); - } - - /** - * Sends 'session-initiate' to the remote peer. - * - * NOTE this method is synchronous and we're not waiting for the RESULT - * response which would delay the startup process. - * - * @param {string} offerSdp - The local session description which will be used to generate an offer. - * @returns {void} - * @private - */ - sendSessionInitiate(offerSdp) { - let init = $iq({ - to: this.remoteJid, - type: 'set' - }).c('jingle', { - xmlns: 'urn:xmpp:jingle:1', - action: 'session-initiate', - initiator: this.initiatorJid, - sid: this.sid - }); - - new SDP(offerSdp, this.isP2P).toJingle( - init, - this.isInitiator ? 'initiator' : 'responder'); - init = init.tree(); - logger.debug(`${this} Session-initiate: `, init); - this.connection.sendIQ(init, - () => { - logger.info(`${this} Got RESULT for "session-initiate"`); - }, - error => { - logger.error(`${this} "session-initiate" error`, error); - }, - IQ_TIMEOUT); - } - /** * Sets the answer received from the remote peer as the remote description. * @@ -2237,7 +2237,7 @@ export default class JingleSessionPC extends JingleSession { this.modificationQueue.resume(); const newLocalSdp = new SDP(this.peerconnection.localDescription.sdp); - this.sendContentModify(); + this._sendContentModify(); this.notifyMySSRCUpdate(oldLocalSdp, newLocalSdp); } }) @@ -2331,7 +2331,7 @@ export default class JingleSessionPC extends JingleSession { // up our SDP translation chain (simulcast, video mute, RTX etc.) // #2 Sends the max frame height if it was set, before the session-initiate/accept if (this.isP2P && (!this._localSendReceiveVideoActive || this._sourceReceiverConstraints)) { - this.sendContentModify(); + this._sendContentModify(); } } @@ -2363,7 +2363,7 @@ export default class JingleSessionPC extends JingleSession { if (this._localSendReceiveVideoActive !== videoActive) { this._localSendReceiveVideoActive = videoActive; if (this.isP2P && this.state === JingleSessionState.ACTIVE) { - this.sendContentModify(); + this._sendContentModify(); } return this.peerconnection @@ -2388,7 +2388,7 @@ export default class JingleSessionPC extends JingleSession { // Tell the remote peer about our receive constraint. If Jingle session is not yet active the state will // be synced after offer/answer. if (this.state === JingleSessionState.ACTIVE) { - this.sendContentModify(); + this._sendContentModify(); } } } diff --git a/types/hand-crafted/modules/xmpp/JingleSessionPC.d.ts b/types/hand-crafted/modules/xmpp/JingleSessionPC.d.ts index 4d8e36ca46..dc113a2460 100644 --- a/types/hand-crafted/modules/xmpp/JingleSessionPC.d.ts +++ b/types/hand-crafted/modules/xmpp/JingleSessionPC.d.ts @@ -19,8 +19,6 @@ export default class JingleSessionPC extends JingleSession { setAnswer: ( jingleAnswer: unknown ) => void; // TODO: setOfferAnswerCycle: ( jingleOfferAnswerIq: JQuery, success: ( params: unknown ) => unknown, failure: ( params: unknown ) => unknown, localTracks?: JitsiLocalTrack[] ) => void; // TODO: setVideoCodecs: ( preferred?: CodecMimeType, disabled?: CodecMimeType ) => void; - sendSessionAccept: ( success: ( params: unknown ) => unknown, failure: ( params: unknown ) => unknown ) => void; // TODO: - sendContentModify: () => void; setReceiverVideoConstraint: ( maxFrameHeight: number ) => void; setSenderMaxBitrates: () => Promise; setSenderVideoConstraint: ( maxFrameHeight: number ) => Promise; // TODO: