wfs.js之h264转码mp4分析

准备源文件

下载源文件

git clone https://github.com/ChihChengYang/wfs.js.git

编译后得到wfs.js这个文件

调用

在demo/index.html中，前端对wfs.js进行了调用

var video1 = document.getElementById("video1"),
wfs = new Wfs();    
wfs.attachMedia(video1,'ch1');

对应wfs.js中的function Wfs() 函数

key: 'attachMedia',value: function attachMedia(media) {//arg1 通道var channelName = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 'chX';//arg2 媒体格式var mediaType = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 'H264Raw';//arg3 未定义var websocketName = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : 'play2';// 'H264Raw' 'FMp4'    this.mediaType = mediaType;this.media = media;//MEDIA_ATTACHING 事件处理this.trigger(_events2.default.MEDIA_ATTACHING, { media: media, channelName: channelName, mediaType: mediaType, websocketName: websocketName });

MEDIA_ATTACHING 事件的处理

MEDIA_ATTACHING: 'wfsMediaAttaching',

在var BufferController = function (_EventHandler)中

    key: 'onMediaAttaching',value: function onMediaAttaching(data) {var media = this.media = data.media;this.mediaType = data.mediaType;this.websocketName = data.websocketName;this.channelName = data.channelName;if (media) {// setup the media sourcevar ms = this.mediaSource = new MediaSource();//Media Source listeners//绑定ms的监听事件//打开时候onMediaSourceOpenthis.onmso = this.onMediaSourceOpen.bind(this);this.onmse = this.onMediaSourceEnded.bind(this);this.onmsc = this.onMediaSourceClose.bind(this);ms.addEventListener('sourceopen', this.onmso);ms.addEventListener('sourceended', this.onmse);ms.addEventListener('sourceclose', this.onmsc);// link video and media Source//连接video和刚刚创建的ms media Sourcemedia.src = URL.createObjectURL(ms);}}

ms的onMediaSourceOpen

在ms被打开的时候。调用 onMediaSourceOpen

key: 'onMediaSourceOpen',value: function onMediaSourceOpen() {var mediaSource = this.mediaSource;if (mediaSource) {// once received, don't listen anymore to sourceopen eventmediaSource.removeEventListener('sourceopen', this.onmso);}if (this.mediaType === 'FMp4') {this.checkPendingTracks();}//调用MEDIA_ATTACHED事件处理this.wfs.trigger(_events2.default.MEDIA_ATTACHED, { media: this.media, channelName: this.channelName, mediaType: this.mediaType, websocketName: this.websocketName });

MEDIA_ATTACHED事件处理

MEDIA_ATTACHED: 'wfsMediaAttached',

    key: 'onMediaAttached',value: function onMediaAttached(data) {if (data.websocketName != undefined) {//创建WebSocket连接var client = new WebSocket('ws://' + window.location.host + '/' + data.websocketName);//调用attachWebsocket方法this.wfs.attachWebsocket(client, data.channelName);} else {console.log('websocketName ERROE!!!');}}

    key: 'attachWebsocket',value: function attachWebsocket(websocket, channelName) {//调用WEBSOCKET_ATTACHING事件处理this.trigger(_events2.default.WEBSOCKET_ATTACHING, { websocket: websocket, mediaType: this.mediaType, channelName: channelName });}

WEBSOCKET_ATTACHING事件处理

WEBSOCKET_ATTACHING: 'wfsWebsocketAttaching',

    key: 'onWebsocketAttaching',value: function onWebsocketAttaching(data) {this.mediaType = data.mediaType;this.channelName = data.channelName;if (data.websocket instanceof WebSocket) {this.client = data.websocket;//websocket 的 onopen事件绑定绑定initSocketClientthis.client.onopen = this.initSocketClient.bind(this);this.client.onclose = function (e) {console.log('Websocket Disconnected!');};}}

    key: 'initSocketClient',value: function initSocketClient(client) {this.client.binaryType = 'arraybuffer';//websocket 的 onmessage 事件绑定绑定receiveSocketMessagethis.client.onmessage = this.receiveSocketMessage.bind(this);//WEBSOCKET_MESSAGE_SENDING事件处理this.wfs.trigger(_events2.default.WEBSOCKET_MESSAGE_SENDING, { commandType: "open", channelName: this.channelName, commandValue: "NA" });console.log('Websocket Open!');}

//WEBSOCKET_MESSAGE_SENDING事件处理

WEBSOCKET_MESSAGE_SENDING: 'wfsWebsocketMessageSending'

    key: 'onWebsocketMessageSending',value: function onWebsocketMessageSending(event) {this.client.send(JSON.stringify({ t: event.commandType, c: event.channelName, v: event.commandValue }));}

这样把信息通过json发送出去。

接收信息的处理

上面websocket 的 onmessage 事件绑定绑定receiveSocketMessage，当websocket接收到信息时，调用receiveSocketMessage

    key: 'receiveSocketMessage',value: function receiveSocketMessage(event) {if (document['hidden']) return;this.buf = new Uint8Array(event.data);var copy = new Uint8Array(this.buf);if (this.mediaType === 'FMp4') {this.wfs.trigger(_events2.default.WEBSOCKET_ATTACHED, { payload: copy });}if (this.mediaType === 'H264Raw') {//H264_DATA_PARSING事件处理this.wfs.trigger(_events2.default.H264_DATA_PARSING, { data: copy });}}

H264_DATA_PARSING事件处理

H264_DATA_PARSING: 'wfsH264DataParsing',

        key: 'onH264DataParsing',value: function onH264DataParsing(event) {//读取数据this._read(event.data);var $this = this;//对于每个nalthis.nals.forEach(function (nal) {//H264_DATA_PARSED事件处理$this.wfs.trigger(_events2.default.H264_DATA_PARSED, {data: nal});});}

H264_DATA_PARSED事件处理

H264_DATA_PARSED: 'wfsH264DataParsed',

    key: 'onH264DataParsed',value: function onH264DataParsed(event) {//分析AVCTrackthis._parseAVCTrack(event.data);if (this.browserType === 1 || this._avcTrack.samples.length >= 20) {// Firefoxthis.remuxer.pushVideo(0, this.sn, this._avcTrack, this.timeOffset, this.contiguous);this.sn += 1;}}

数据经过_parseAVCTrack处理后，调用remuxer.pushVideo方法

    key: 'pushVideo',value: function pushVideo(level, sn, videoTrack, timeOffset, contiguous) {this.level = level;this.sn = sn;var videoData = void 0;// generate Init Segment if neededif (!this.ISGenerated) {this.generateVideoIS(videoTrack, timeOffset);}if (this.ISGenerated) {if (videoTrack.samples.length) {this.remuxVideo_2(videoTrack, timeOffset, contiguous);}}}

最后调用remuxVideo_2方法

    key: 'remuxVideo_2',value: function remuxVideo_2(track, timeOffset, contiguous, audioTrackLength) {var offset = 8,pesTimeScale = this.PES_TIMESCALE,pes2mp4ScaleFactor = this.PES2MP4SCALEFACTOR,mp4SampleDuration,mdat,moof,firstPTS,firstDTS,nextDTS,inputSamples = track.samples,outputSamples = [];/* concatenate the video data and construct the mdat in place(need 8 more bytes to fill length and mpdat type) */mdat = new Uint8Array(track.len + 4 * track.nbNalu + 8);var view = new DataView(mdat.buffer);view.setUint32(0, mdat.byteLength);mdat.set(_mp4Generator2.default.types.mdat, 4);var sampleDuration = 0;var ptsnorm = void 0,dtsnorm = void 0,mp4Sample = void 0,lastDTS = void 0;for (var i = 0; i < inputSamples.length; i++) {var avcSample = inputSamples[i],mp4SampleLength = 0,compositionTimeOffset = void 0;// convert NALU bitstream to MP4 format (prepend NALU with size field)while (avcSample.units.units.length) {var unit = avcSample.units.units.shift();view.setUint32(offset, unit.data.byteLength);offset += 4;mdat.set(unit.data, offset);offset += unit.data.byteLength;mp4SampleLength += 4 + unit.data.byteLength;}var pts = avcSample.pts - this._initPTS;var dts = avcSample.dts - this._initDTS;dts = Math.min(pts, dts);if (lastDTS !== undefined) {ptsnorm = this._PTSNormalize(pts, lastDTS);dtsnorm = this._PTSNormalize(dts, lastDTS);sampleDuration = dtsnorm - lastDTS;if (sampleDuration <= 0) {_logger.logger.log('invalid sample duration at PTS/DTS: ' + avcSample.pts + '/' + avcSample.dts + '|dts norm: ' + dtsnorm + '|lastDTS: ' + lastDTS + ':' + sampleDuration);sampleDuration = 1;}} else {var nextAvcDts = this.nextAvcDts,delta;ptsnorm = this._PTSNormalize(pts, nextAvcDts);dtsnorm = this._PTSNormalize(dts, nextAvcDts);if (nextAvcDts) {delta = Math.round(dtsnorm - nextAvcDts);if ( /*contiguous ||*/Math.abs(delta) < 600) {if (delta) {if (delta > 1) {_logger.logger.log('AVC:' + delta + ' ms hole between fragments detected,filling it');} else if (delta < -1) {_logger.logger.log('AVC:' + -delta + ' ms overlapping between fragments detected');}dtsnorm = nextAvcDts;ptsnorm = Math.max(ptsnorm - delta, dtsnorm);_logger.logger.log('Video/PTS/DTS adjusted: ' + ptsnorm + '/' + dtsnorm + ',delta:' + delta);}}}this.firstPTS = Math.max(0, ptsnorm);this.firstDTS = Math.max(0, dtsnorm);sampleDuration = 0.03;}outputSamples.push({size: mp4SampleLength,duration: this.H264_TIMEBASE,cts: 0,flags: {isLeading: 0,isDependedOn: 0,hasRedundancy: 0,degradPrio: 0,dependsOn: avcSample.key ? 2 : 1,isNonSync: avcSample.key ? 0 : 1}});lastDTS = dtsnorm;}var lastSampleDuration = 0;if (outputSamples.length >= 2) {lastSampleDuration = outputSamples[outputSamples.length - 2].duration;outputSamples[0].duration = lastSampleDuration;}this.nextAvcDts = dtsnorm + lastSampleDuration;var dropped = track.dropped;track.len = 0;track.nbNalu = 0;track.dropped = 0;if (outputSamples.length && navigator.userAgent.toLowerCase().indexOf('chrome') > -1) {var flags = outputSamples[0].flags;flags.dependsOn = 2;flags.isNonSync = 0;}track.samples = outputSamples;moof = _mp4Generator2.default.moof(track.sequenceNumber++, dtsnorm, track);track.samples = [];var data = {id: this.id,level: this.level,sn: this.sn,data1: moof,data2: mdat,startPTS: ptsnorm,endPTS: ptsnorm,startDTS: dtsnorm,endDTS: dtsnorm,type: 'video',nb: outputSamples.length,dropped: dropped};this.observer.trigger(_events2.default.FRAG_PARSING_DATA, data);return data;}

在处理完数据后，进行FRAG_PARSING_DATA事件处理

  FRAG_PARSING_DATA: 'wfsFragParsingData',

    key: 'onFragParsingData',value: function onFragParsingData(data) {var _this2 = this;if (data.type === 'video') {}[data.data1, data.data2].forEach(function (buffer) {if (buffer) {_this2.pendingAppending++;_this2.wfs.trigger(_events2.default.BUFFER_APPENDING, { type: data.type, data: buffer, parent: 'main' });}});}

BUFFER_APPENDING事件处理

BUFFER_APPENDING: 'wfsBufferAppending',

    key: 'onBufferAppending',value: function onBufferAppending(data) {if (!this.segments) {this.segments = [data];} else {this.segments.push(data);}this.doAppending();}

    key: 'doAppending',value: function doAppending() {var wfs = this.wfs,sourceBuffer = this.sourceBuffer,segments = this.segments;if (Object.keys(sourceBuffer).length) {if (this.media.error) {this.segments = [];console.log('trying to append although a media error occured, flush segment and abort');return;}if (this.appending) {return;}if (segments && segments.length) {var segment = segments.shift();try {if (sourceBuffer[segment.type]) {this.parent = segment.parent;sourceBuffer[segment.type].appendBuffer(segment.data);this.appendError = 0;this.appended++;this.appending = true;} else {}} catch (err) {// in case any error occured while appending, put back segment in segments table segments.unshift(segment);var event = { type: ErrorTypes.MEDIA_ERROR };if (err.code !== 22) {if (this.appendError) {this.appendError++;} else {this.appendError = 1;}event.details = ErrorDetails.BUFFER_APPEND_ERROR;event.frag = this.fragCurrent;if (this.appendError > wfs.config.appendErrorMaxRetry) {segments = [];event.fatal = true;return;} else {event.fatal = false;}} else {this.segments = [];event.details = ErrorDetails.BUFFER_FULL_ERROR;return;}}}}}