WebrtcNode, publish-sdp offer 流程(1)

1. AmqpClient - New message received

sdp offer 的消息

AmqpClient - RpcServer New message received {method: 'onTransportSignaling',args: ['aa230ce0863e42baa8bae5c14e91e809',{sdp: 'v=0\r\n' +'o=- 2367615733001925388 2 IN IP4 127.0.0.1\r\n' +'s=-\r\n' +'t=0 0\r\n' +'a=group:BUNDLE 0 1\r\n' +'a=msid-semantic: WMS MediaStream-0176f9fc-a754-4838-b002-0bdda6d4deaa\r\n' +'m=audio 9 UDP/TLS/RTP/SAVPF 111 103 104 9 102 0 8 106 105 13 110 112 113 126\r\n' +'c=IN IP4 0.0.0.0\r\n' +'a=rtcp:9 IN IP4 0.0.0.0\r\n' +'a=ice-ufrag:YH25\r\n' +'a=ice-pwd:l47tWnjyoR1kn3nR0JWjuotU\r\n' +'a=ice-options:trickle\r\n' +'a=fingerprint:sha-256 65:8A:AE:75:0E:6B:50:51:A4:93:A4:01:76:43:3D:83:26:31:E7:75:B8:8A:DC:D4:13:78:14:4E:4D:0B:93:3E\r\n' +'a=setup:actpass\r\n' +'a=mid:0\r\n' +'a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\n' +'a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\n' +'a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\n' +'a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\n' +'a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\n' +'a=extmap:6 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\n' +'a=sendonly\r\n' +'a=msid:MediaStream-0176f9fc-a754-4838-b002-0bdda6d4deaa AudioTrack-abd86f8a-1f74-4c73-b6fa-c20f5b175a1d\r\n' +'a=rtcp-mux\r\n' +'a=rtpmap:111 opus/48000/2\r\n' +'a=rtcp-fb:111 transport-cc\r\n' +'a=fmtp:111 minptime=10;useinbandfec=1\r\n' +'a=rtpmap:103 ISAC/16000\r\n' +'a=rtpmap:104 ISAC/32000\r\n' +'a=rtpmap:9 G722/8000\r\n' +'a=rtpmap:102 ILBC/8000\r\n' +'a=rtpmap:0 PCMU/8000\r\n' +'a=rtpmap:8 PCMA/8000\r\n' +'a=rtpmap:106 CN/32000\r\n' +'a=rtpmap:105 CN/16000\r\n' +'a=rtpmap:13 CN/8000\r\n' +'a=rtpmap:110 telephone-event/48000\r\n' +'a=rtpmap:112 telephone-event/32000\r\n' +'a=rtpmap:113 telephone-event/16000\r\n' +'a=rtpmap:126 telephone-event/8000\r\n' +'a=ssrc:3582341671 cname:JTBK6RkjYuislL5i\r\n' +'a=ssrc:3582341671 msid:MediaStream-0176f9fc-a754-4838-b002-0bdda6d4deaa AudioTrack-abd86f8a-1f74-4c73-b6fa-c20f5b175a1d\r\n' +'a=ssrc:3582341671 mslabel:MediaStream-0176f9fc-a754-4838-b002-0bdda6d4deaa\r\n' +'a=ssrc:3582341671 label:AudioTrack-abd86f8a-1f74-4c73-b6fa-c20f5b175a1d\r\n' +'m=video 9 UDP/TLS/RTP/SAVPF 123 114 115 116 119\r\n' +'c=IN IP4 0.0.0.0\r\n' +'a=rtcp:9 IN IP4 0.0.0.0\r\n' +'a=ice-ufrag:YH25\r\n' +'a=ice-pwd:l47tWnjyoR1kn3nR0JWjuotU\r\n' +'a=ice-options:trickle\r\n' +'a=fingerprint:sha-256 65:8A:AE:75:0E:6B:50:51:A4:93:A4:01:76:43:3D:83:26:31:E7:75:B8:8A:DC:D4:13:78:14:4E:4D:0B:93:3E\r\n' +'a=setup:actpass\r\n' +'a=mid:1\r\n' +'a=extmap:14 urn:ietf:params:rtp-hdrext:toffset\r\n' +'a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\n' +'a=extmap:13 urn:3gpp:video-orientation\r\n' +'a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\n' +'a=extmap:12 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay\r\n' +'a=extmap:11 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type\r\n' +'a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing\r\n' +'a=extmap:8 http://tools.ietf.org/html/draft-ietf-avtext-framemarking-07\r\n' +'a=extmap:9 http://www.webrtc.org/experiments/rtp-hdrext/color-space\r\n' +'a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\n' +'a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\n' +'a=extmap:6 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\n' +'a=sendonly\r\n' +'a=msid:MediaStream-0176f9fc-a754-4838-b002-0bdda6d4deaa VideoTrack-3e9cd783-7906-49e1-8887-99736156a1d2\r\n' +'a=rtcp-mux\r\n' +'a=rtcp-rsize\r\n' +'a=rtpmap:123 AV1/90000\r\n' +'a=rtcp-fb:123 goog-remb\r\n' +'a=rtcp-fb:123 transport-cc\r\n' +'a=rtcp-fb:123 ccm fir\r\n' +'a=rtcp-fb:123 nack\r\n' +'a=rtcp-fb:123 nack pli\r\n' +'a=rtpmap:114 red/90000\r\n' +'a=rtpmap:115 rtx/90000\r\n' +'a=fmtp:115 apt=114\r\n' +'a=rtpmap:116 ulpfec/90000\r\n' +'a=rtpmap:119 rtx/90000\r\n' +'a=fmtp:119 apt=123\r\n' +'a=ssrc-group:FID 4143402920 1935417042\r\n' +'a=ssrc:4143402920 cname:JTBK6RkjYuislL5i\r\n' +'a=ssrc:4143402920 msid:MediaStream-0176f9fc-a754-4838-b002-0bdda6d4deaa VideoTrack-3e9cd783-7906-49e1-8887-99736156a1d2\r\n' +'a=ssrc:4143402920 mslabel:MediaStream-0176f9fc-a754-4838-b002-0bdda6d4deaa\r\n' +'a=ssrc:4143402920 label:VideoTrack-3e9cd783-7906-49e1-8887-99736156a1d2\r\n' +'a=ssrc:1935417042 cname:JTBK6RkjYuislL5i\r\n' +'a=ssrc:1935417042 msid:MediaStream-0176f9fc-a754-4838-b002-0bdda6d4deaa VideoTrack-3e9cd783-7906-49e1-8887-99736156a1d2\r\n' +'a=ssrc:1935417042 mslabel:MediaStream-0176f9fc-a754-4838-b002-0bdda6d4deaa\r\n' +'a=ssrc:1935417042 label:VideoTrack-3e9cd783-7906-49e1-8887-99736156a1d2\r\n',type: 'offer'}],corrID: 14,replyTo: 'amq.gen-pj6CYWZ4AOsOkYmlAePDEA'
}

2. WebrtcNode - onTransportSignaling

dist/webrtc_agent/webrtc/index.js

// connectionId 应该就是operationId
that.onTransportSignaling = function (connectionId, msg, callback) {log.debug('onTransportSignaling, connection id:', connectionId, 'msg:', msg);// 通过 operationId 从mappingTransports 获取 transportId，// 接着通过transportId从peerConnections 获取WrtcConnection// 参考小结 2.1var conn = getWebRTCConnection(connectionId);// WrtcConnection connif (conn) {// 参考小结 2.2conn.onSignalling(msg, connectionId);callback('callback', 'ok');} else {callback('callback', 'error', 'Connection does NOT exist:' + connectionId);}};

2.1 WebrtcNode - getWebRTCConnection——返回WrtcConnection

dist/webrtc_agent/webrtc/index.js

    // 返回的是WrtcConnectionvar getWebRTCConnection = function (operationId) {// 在WebrtcNode-createWebRTCConnection 创建var transportId = mappingTransports.get(operationId);if (peerConnections.has(transportId)) {return peerConnections.get(transportId);}return null;};

2.2 WrtcConnection.onSignalling

dist/webrtc_agent/webrtc/wrtcConnection.js

处理 offer sdp。

  that.onSignalling = function (msg, operationId) {var processSignalling = function () {// 这里type=’offer‘if (msg.type === 'offer') {log.debug('on offer:', msg.sdp);processOffer(msg.sdp);} else if (msg.type === 'candidate') {...} else if (msg.type === 'removed-candidates') {...}};// wrtc 是 Connectionif (wrtc) {processSignalling();} else {// should not reach herelog.error('wrtc is not ready');}};

2.3 ==========WrtcConnection.processOffer

dist/webrtc_agent/webrtc/wrtcConnection.js

这里的 remoteSdp 是null，

 const processOffer = function (sdp) {// 第一进来，remoteSdp 是nullif (!remoteSdp) {// First offerremoteSdp = new SdpInfo(sdp);// Check mid， 获取到mid 数组const mids = remoteSdp.mids();for (const mid of mids) {// 设置finalformatprocessOfferMedia(mid);}// 创建answer sdplocalSdp = remoteSdp.answer();// Setup transportlet opId = null;for (const mid of mids) {if (remoteSdp.getMediaPort(mid) !== 0) {opId = setupTransport(mid);}}if (opId) {on_track({type: 'tracks-complete',operationId: opId});}} else {...}};

  var remoteSdp = null;var localSdp = null;

2.3.1 SdpInfo.SdpInfo

dist/webrtc_agent/webrtc/sdpInfo.js

2.3.2 WrtcConnection.processOfferMedia——设置finalformat

WrtcConnection.addTrackOperation, 创建了元素operationMap

设置finalFormat

 const processOfferMedia = function (mid) {...// Determine media format in offerif (remoteSdp.mediaType(mid) === 'audio') {const audioPreference = operationMap.get(mid).formatPreference;const audioFormat = remoteSdp.filterAudioPayload(mid, audioPreference);operationMap.get(mid).finalFormat = audioFormat;} else if (remoteSdp.mediaType(mid) === 'video') {const videoPreference = operationMap.get(mid).formatPreference;const videoFormat = remoteSdp.filterVideoPayload(mid, videoPreference);operationMap.get(mid).finalFormat = videoFormat;}};

operationMap 的元素就是WrtcConnection.addTrackOperation中添加的，这里是给finalFormat赋值

// mid => { operationId, sdpDirection, type, formatPreference, rids, enabled, finalFormat }var operationMap = new Map();

dist/webrtc_agent/webrtc/wrtcConnection.js，在WebrtcNode-publish流程.md 的addTrackOperation 小节 有详细说明。

2.3.3 SdpInfo.answer——localSdp

通过offer到sdp，进行对应的修改，作为answer的sdp。赋值给localSdp中，在WrtcConnection.setupTransport用到。

 answer() {const answer = new SdpInfo(this.toString());answer.obj.origin = {username: '-',sessionId: '0',sessionVersion: 0,netType: 'IN',ipVer: 4,address: '127.0.0.1'};answer.obj.media.forEach(mediaInfo => {mediaInfo.port = 1;mediaInfo.rtcp = {port: 1,netType: 'IN',ipVer: 4,address: '0.0.0.0'};if (mediaInfo.setup === 'active') {mediaInfo.setup = 'passive';} else {mediaInfo.setup = 'active';}delete mediaInfo.iceOptions;  delete mediaInfo.rtcpRsize;if (mediaInfo.direction === 'recvonly') {mediaInfo.direction = 'sendonly';} else if (mediaInfo.direction === 'sendonly') {delete mediaInfo.msid;delete mediaInfo.ssrcGroups;delete mediaInfo.ssrcs;mediaInfo.direction = 'recvonly';}if (mediaInfo.ext && Array.isArray(mediaInfo.ext)) {const extMappings = ['urn:ietf:params:rtp-hdrext:ssrc-audio-level',// 'http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01','urn:ietf:params:rtp-hdrext:sdes:mid','urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id','urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id','urn:ietf:params:rtp-hdrext:toffset','http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time',// 'urn:3gpp:video-orientation',// 'http://www.webrtc.org/experiments/rtp-hdrext/playout-delay',];if (mediaInfo.type === 'video') {extMappings.push('http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01');}mediaInfo.ext = mediaInfo.ext.filter((e) => {return extMappings.includes(e.uri);});}if (mediaInfo.rids && Array.isArray(mediaInfo.rids)) {// Reverse rids directionmediaInfo.rids.forEach(r => {r.direction = (r.direction === 'send') ? 'recv' : 'send';});}if (mediaInfo.simulcast) {// Reverse simulcast directionif (mediaInfo.simulcast.dir1 === 'send') {mediaInfo.simulcast.dir1 = 'recv';} else {mediaInfo.simulcast.dir1 = 'send';}}});return answer;}

SdpInfo.getMediaPort

SdpInfo.mediaType

operationMap——存放的是track相关的内容

dist/webrtc_agent/webrtc/wrtcConnection.js

在WebrtcNode-publish流程.md 的addTrackOperation 小节 有详细说明

  // mid => { operationId, sdpDirection, type, formatPreference, rids, enabled, finalFormat }var operationMap = new Map();

2.3.4 WrtcConnection.setupTransport——创建WrtcStream

dist/webrtc_agent/webrtc/wrtcConnection.js

小节 3

2.3.5 on_track——这是创建WrtcConnection 传入的callback,tracks-complete

在WebrtcNode-publish流程.md 的 3.WebrtcNode - createWebRTCConnection——返回WrtcConnection 中传入了

function onTrack(trackInfo) {handleTrackInfo(transportId, trackInfo, controller);}

调用的代码

        on_track({type: 'tracks-complete',operationId: opId});

详细的看3.3， 3.4 小节

3. WrtcConnection.setupTransport——创建WrtcStream

主要是创建MediaStream。

dist/webrtc_agent/webrtc/wrtcConnection.js

const setupTransport = function (mid) {let rids = remoteSdp.rids(mid);const opSettings = operationMap.get(mid);const direction = (opSettings.sdpDirection === 'sendonly') ? 'in' : 'out';const simSsrcs = remoteSdp.getLegacySimulcast(mid);const trackSettings = remoteSdp.getMediaSettings(mid);const mediaType = remoteSdp.mediaType(mid);trackSettings.owner = owner;trackSettings.enableBWE = that.enableBWE;if (opSettings.finalFormat) {trackSettings[mediaType].format = opSettings.finalFormat;if (opSettings.finalFormat.codec === 'vp9' && simSsrcs) {// Legacy simulcast for VP9 SVCrids = null;trackSettings['video'].scalabilityMode = opSettings.scalabilityMode;}}if (rids) {...} else {// 走这里的代码// No simulcastif (!trackMap.has(mid)) {// 1. Connection wrtctrackMap.set(mid, new WrtcStream(mid, wrtc, direction, trackSettings));// Set ssrc in local sdp for out direction// direct 是inif (direction === 'out') {...}// 2. Connection wrtc// remoteSdp 是在processOffer 创建的对象wrtc.setRemoteSdp(remoteSdp.singleMediaSdp(mid).toString(), mid);// 3. Notify new track， 详细见小节 3.3， 3.4on_track({type: 'track-added',track: trackMap.get(mid),operationId: opSettings.operationId,mid: mid});} else {log.warn(`Conflict trackId ${mid} for ${wrtcId}`);}}return opSettings.operationId;};

SdpInfo.rids

trackMap 属性——存放了WrtcStream

  // composedId => WrtcStreamvar trackMap = new Map();

composeId

  const composeId = function (mid, rid) {return mid + ':' + rid;};

3.1 new WrtcStream

小节 4
创建了MediaStream

3.2 Connection.setRemoteSdp

// streamId = mid
setRemoteSdp(sdp, streamId) {// webRtcConnection wrtcthis.wrtc.setRemoteSdp(sdp, streamId || this.id);}

3.2.1 NAN_METHOD(WebRtcConnection::setRemoteSdp)

NAN_METHOD(WebRtcConnection::setRemoteSdp) {WebRtcConnection* obj = Nan::ObjectWrap::Unwrap<WebRtcConnection>(info.Holder());std::shared_ptr<erizo::WebRtcConnection> me = obj->me;if (!me) {return;}std::string sdp = getString(info[0]);std::string stream_id = getString(info[1]);bool r = me->setRemoteSdp(sdp, stream_id);info.GetReturnValue().Set(Nan::New(r));
}

3.2.2 WebRtcConnection::setRemoteSdp

bool WebRtcConnection::setRemoteSdpInfo(std::shared_ptr<SdpInfo> sdp, std::string stream_id) {asyncTask([sdp, stream_id] (std::shared_ptr<WebRtcConnection> connection) {ELOG_DEBUG("%s message: setting remote SDPInfo", connection->toLog());if (!connection->sending_) {return;}connection->remote_sdp_ = sdp;// mid connection->processRemoteSdp(stream_id);});return true;
}

??? WebRtcConnection::processRemoteSdp

source/agent/webrtc/rtcConn/erizo/src/erizo/WebRtcConnection.cpp

SdpInfo.singleMediaSdp

dist-debug/webrtc_agent/webrtc/sdpInfo.js

  singleMediaSdp(mid) {const sdp = new SdpInfo(this.toString());sdp.obj.media = sdp.obj.media.filter(m => m.mid.toString() === mid);sdp.setBundleMids([mid]);return sdp;}

3.3 on__track——track add

dist/webrtc_agent/webrtc/wrtcConnection.js

通知到WebrtcNode（/dist/webrtc_agent/webrtc/index.js）的createWebRTCConnection 中注册的callback, 回调一个对象，

        // Notify new trackon_track({type: 'track-added',track: trackMap.get(mid), // WrtcStreamoperationId: opSettings.operationId,mid: mid})

3.4 ======WebrtcNode.handleTrackInfo

dist/webrtc_agent/webrtc/index.js

var connection = new WrtcConnection({connectionId: transportId,threadPool: threadPool,ioThreadPool: ioThreadPool,network_interfaces: global.config.webrtc.network_interfaces,owner,}, function onTransportStatus(status) {notifyTransportStatus(controller, transportId, status);}, function onTrack(trackInfo) {handleTrackInfo(transportId, trackInfo, controller);});

// trackInfo 就是on__track 回调回来
var handleTrackInfo = function (transportId, trackInfo, controller) {var publicTrackId;var updateInfo;if (trackInfo.type === 'track-added') {// Generate public track IDconst track = trackInfo.track; // WrtcStreampublicTrackId = transportId + '-' + track.id;if (mediaTracks.has(publicTrackId)) {log.error('Conflict public track id:', publicTrackId, transportId, track.id);return;}mediaTracks.set(publicTrackId, track);mappingPublicId.get(transportId).set(track.id, publicTrackId);if (track.direction === 'in') {const trackSource = track.sender();router.addLocalSource(publicTrackId, 'webrtc', trackSource).catch(e => log.warn('Unexpected error during track add:', e));} else {router.addLocalDestination(publicTrackId, 'webrtc', track).catch(e => log.warn('Unexpected error during track add:', e));}// Bind media-update handlertrack.on('media-update', (jsonUpdate) => {log.debug('notifyMediaUpdate:', publicTrackId, jsonUpdate);notifyMediaUpdate(controller, publicTrackId, track.direction, JSON.parse(jsonUpdate));});// Notify controllerconst mediaType = track.format('audio') ? 'audio' : 'video';updateInfo = {type: 'track-added',trackId: publicTrackId,mediaType: track.format('audio') ? 'audio' : 'video',mediaFormat: track.format(mediaType),direction: track.direction,operationId: trackInfo.operationId,mid: trackInfo.mid,rid: trackInfo.rid,active: true,};log.debug('notifyTrackUpdate', controller, publicTrackId, updateInfo);notifyTrackUpdate(controller, transportId, updateInfo);} else if (trackInfo.type === 'track-removed') {publicTrackId = mappingPublicId.get(transportId).get(trackInfo.trackId);if (!mediaTracks.has(publicTrackId)) {log.error('Non-exist public track id:', publicTrackId, transportId, trackInfo.trackId);return;}log.debug('track removed:', publicTrackId);router.removeConnection(publicTrackId).then(ok => {mediaTracks.get(publicTrackId).close();mediaTracks.delete(publicTrackId);mappingPublicId.get(transportId).delete(trackInfo.trackId);}).catch(e => log.warn('Unexpected error during track remove:', e));// Notify controllerupdateInfo = {type: 'track-removed',trackId: publicTrackId,};notifyTrackUpdate(controller, transportId, updateInfo);} else if (trackInfo.type === 'tracks-complete') {updateInfo = {type: 'tracks-complete',operationId: trackInfo.operationId};notifyTrackUpdate(controller, transportId, updateInfo);}};

时序图