公司网站建设北京,商业计划的网站建设费用,网站建设行业怎么样,社区电商小程序文章目录 1.视频输出模块1.1 视频输出初始化1.1.1 视频输出初始化主要流程1.1.2 calculate_display_rect初始化显示窗口大小 1.2 视频输出逻辑1.2.1 event_loop开始处理SDL事件1.2.2 video_refresh1.2.2.1 计算上一帧显示时长,判断是否还要继续上一帧1.2.2.2 估算当前帧显示时长… 文章目录 1.视频输出模块1.1 视频输出初始化1.1.1 视频输出初始化主要流程1.1.2 calculate_display_rect初始化显示窗口大小 1.2 视频输出逻辑1.2.1 event_loop开始处理SDL事件1.2.2 video_refresh1.2.2.1 计算上一帧显示时长,判断是否还要继续上一帧1.2.2.2 估算当前帧显示时长,判断是否要丢帧1.2.2.3 调用video_display进行显示1.2.2.4 realloc_texture()重新分配vid_texture1.2.2.5 sws_getCachedContext1.2.2.6 sws_scale 图像转换 对于flags算法测试推荐文章: 1.视频输出模块 1.1 视频输出初始化 1.1.1 视频输出初始化主要流程 初始化SDL,SDL_Init,主要是SDL_INIT_VIDEO的支持SDL_CreateWindow,创建主窗口SDL_CreateRender,基于主窗口创建renderer,用于渲染输出stream_openevent_loop,播放控制事件的相应循环,但也负责了video的显示输出 int main(int argc, char **argv) {/* 是否显示视频 */if (display_disable) {video_disable 1;}// 3. SDL的初始化flags SDL_INIT_VIDEO | SDL_INIT_AUDIO | SDL_INIT_TIMER;/* 是否运行音频 */if (audio_disable)flags ~SDL_INIT_AUDIO;else {/* Try to work around an occasional ALSA buffer underflow issue when the* period size is NPOT due to ALSA resampling by forcing the buffer size. */if (!SDL_getenv(SDL_AUDIO_ALSA_SET_BUFFER_SIZE))SDL_setenv(SDL_AUDIO_ALSA_SET_BUFFER_SIZE,1, 1);}if (display_disable)flags ~SDL_INIT_VIDEO;if (SDL_Init (flags)) {av_log(NULL, AV_LOG_FATAL, Could not initialize SDL - %s\n, SDL_GetError());av_log(NULL, AV_LOG_FATAL, (Did you set the DISPLAY variable?)\n);exit(1);}SDL_EventState(SDL_SYSWMEVENT, SDL_IGNORE);SDL_EventState(SDL_USEREVENT, SDL_IGNORE);av_init_packet(flush_pkt); // 初始化flush_packetflush_pkt.data (uint8_t *)flush_pkt; // 初始化为数据指向自己本身// 4. 创建窗口if (!display_disable) {int flags SDL_WINDOW_HIDDEN;if (alwaysontop) #if SDL_VERSION_ATLEAST(2,0,5)flags | SDL_WINDOW_ALWAYS_ON_TOP; #elseav_log(NULL, AV_LOG_WARNING, Your SDL version doesnt support SDL_WINDOW_ALWAYS_ON_TOP. Feature will be inactive.\n); #endifif (borderless)flags | SDL_WINDOW_BORDERLESS;elseflags | SDL_WINDOW_RESIZABLE;window SDL_CreateWindow(program_name, SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, default_width, default_height, flags);SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, linear);if (window) {// 创建rendererrenderer SDL_CreateRenderer(window, -1, SDL_RENDERER_ACCELERATED | SDL_RENDERER_PRESENTVSYNC);if (!renderer) {av_log(NULL, AV_LOG_WARNING, Failed to initialize a hardware accelerated renderer: %s\n, SDL_GetError());renderer SDL_CreateRenderer(window, -1, 0);}if (renderer) {if (!SDL_GetRendererInfo(renderer, renderer_info))av_log(NULL, AV_LOG_VERBOSE, Initialized %s renderer.\n, renderer_info.name);}}if (!window || !renderer || !renderer_info.num_texture_formats) {av_log(NULL, AV_LOG_FATAL, Failed to create window or renderer: %s, SDL_GetError());do_exit(NULL);}}// 5. 通过stream_open函数开启read_thread读取线程is stream_open(input_filename, file_iformat);if (!is) {av_log(NULL, AV_LOG_FATAL, Failed to initialize VideoState!\n);do_exit(NULL);}// 6. 事件响应event_loop(is);/* never returns */return 0; } //7 从待处理流中获取相关参数设置显示窗口的宽度、高度及宽高比if (st_index[AVMEDIA_TYPE_VIDEO] 0) {AVStream *st ic-streams[st_index[AVMEDIA_TYPE_VIDEO]];AVCodecParameters *codecpar st-codecpar;//根据流和帧宽高比猜测视频帧的像素宽高比像素的宽高比注意不是图像的AVRational sar av_guess_sample_aspect_ratio(ic, st, NULL);if (codecpar-width) {// 设置显示窗口的大小和宽高比set_default_window_size(codecpar-width, codecpar-height, sar);}}这里重点讲解一下set_default_window_size函数 static void set_default_window_size(int width, int height, AVRational sar) {SDL_Rect rect;int max_width screen_width ? screen_width : INT_MAX; // 确定是否指定窗口最大宽度int max_height screen_height ? screen_height : INT_MAX; // 确定是否指定窗口最大高度if (max_width INT_MAX max_height INT_MAX)max_height height; // 没有指定最大高度时则使用视频的高度calculate_display_rect(rect, 0, 0, max_width, max_height, width, height, sar);default_width rect.w; // 实际是渲染区域的宽高default_height rect.h; }screen_width和screen_height可以在ffplay启动时用命令行进行设置 -x -y 如果没有指定那么就使用视频帧的高度 重点就是calculate_display_rect函数! 1.1.2 calculate_display_rect初始化显示窗口大小 static void calculate_display_rect(SDL_Rect *rect,int scr_xleft, int scr_ytop, int scr_width, int scr_height,int pic_width, int pic_height, AVRational pic_sar) {AVRational aspect_ratio pic_sar; // 比率int64_t width, height, x, y;if (av_cmp_q(aspect_ratio, av_make_q(0, 1)) 0)aspect_ratio av_make_q(1, 1);// 如果aspect_ratio是负数或者为0,设置为1:1// 转成真正的播放比例aspect_ratio av_mul_q(aspect_ratio, av_make_q(pic_width, pic_height));/* XXX: we suppose the screen has a 1.0 pixel ratio */// 计算显示视频帧区域的宽高// 先以高度为基准height scr_height;// ~1, 取偶数宽度 1110width av_rescale(height, aspect_ratio.num, aspect_ratio.den) ~1;if (width scr_width) {// 当以高度为基准,发现计算出来的需要的窗口宽度不足时调整为以窗口宽度为基准width scr_width;height av_rescale(width, aspect_ratio.den, aspect_ratio.num) ~1;}// 计算显示视频帧区域的起始坐标在显示窗口内部的区域x (scr_width - width) / 2;y (scr_height - height) / 2;rect-x scr_xleft x;rect-y scr_ytop y;rect-w FFMAX((int)width, 1);rect-h FFMAX((int)height, 1); } 这个函数设置了计算了窗口的宽高和位置 函数是先计算宽高比,如果宽高比没有设置的话则使用实际宽高来计算宽高比,然后先以高度为基准使用av_rescale函数计算宽度,如果宽度大于scr_width就转变为以宽度为基准. 然后就是计算顶点坐标了,scr_width和scr_height窗口大小,width和heigth是视频大小,我们要计算出视频的左上角位置 看图: 代码中rect就是渲染的视频部分,也就是图中绿色部分! 1.2 视频输出逻辑 main()-event_loop()-refresh_loop_wait_event()-video_refresh()-video_display()-video_image_display()-upload_texture()1.2.1 event_loop开始处理SDL事件 static void event_loop(VideoState *cur_stream) {SDL_Event event;double incr, pos, frac;for (;;) {double x;refresh_loop_wait_event(cur_stream, event); //video是在这里显示的switch (event.type) {case SDL_KEYDOWN: /* 键盘事件 */if (exit_on_keydown || event.key.keysym.sym SDLK_ESCAPE || event.key.keysym.sym SDLK_q) {do_exit(cur_stream);break;}if (!cur_stream-width)continue;switch (event.key.keysym.sym) {case SDLK_f:toggle_full_screen(cur_stream);cur_stream-force_refresh 1;break;case SDLK_p:case SDLK_SPACE: //按空格键触发暂停/恢复toggle_pause(cur_stream);break;case SDLK_m:toggle_mute(cur_stream);break;case SDLK_KP_MULTIPLY:case SDLK_0:update_volume(cur_stream, 1, SDL_VOLUME_STEP);break;case SDLK_KP_DIVIDE:case SDLK_9:update_volume(cur_stream, -1, SDL_VOLUME_STEP);break;case SDLK_s: // S: Step to next framestep_to_next_frame(cur_stream);break;case SDLK_a:stream_cycle_channel(cur_stream, AVMEDIA_TYPE_AUDIO);break;case SDLK_v:stream_cycle_channel(cur_stream, AVMEDIA_TYPE_VIDEO);break;case SDLK_c:stream_cycle_channel(cur_stream, AVMEDIA_TYPE_VIDEO);stream_cycle_channel(cur_stream, AVMEDIA_TYPE_AUDIO);stream_cycle_channel(cur_stream, AVMEDIA_TYPE_SUBTITLE);break;case SDLK_t:stream_cycle_channel(cur_stream, AVMEDIA_TYPE_SUBTITLE);break;case SDLK_w: #if CONFIG_AVFILTERif (cur_stream-show_mode SHOW_MODE_VIDEO cur_stream-vfilter_idx nb_vfilters - 1) {if (cur_stream-vfilter_idx nb_vfilters)cur_stream-vfilter_idx 0;} else {cur_stream-vfilter_idx 0;toggle_audio_display(cur_stream);} #elsetoggle_audio_display(cur_stream); #endifbreak;case SDLK_PAGEUP:if (cur_stream-ic-nb_chapters 1) {incr 600.0;goto do_seek;}seek_chapter(cur_stream, 1);break;case SDLK_PAGEDOWN:if (cur_stream-ic-nb_chapters 1) {incr -600.0;goto do_seek;}seek_chapter(cur_stream, -1);break;case SDLK_LEFT:incr seek_interval ? -seek_interval : -10.0;goto do_seek;case SDLK_RIGHT:incr seek_interval ? seek_interval : 10.0;goto do_seek;case SDLK_UP:incr 60.0;goto do_seek;case SDLK_DOWN:incr -60.0;do_seek:if (seek_by_bytes) {pos -1;if (pos 0 cur_stream-video_stream 0)pos frame_queue_last_pos(cur_stream-pictq);if (pos 0 cur_stream-audio_stream 0)pos frame_queue_last_pos(cur_stream-sampq);if (pos 0)pos avio_tell(cur_stream-ic-pb);if (cur_stream-ic-bit_rate)incr * cur_stream-ic-bit_rate / 8.0;elseincr * 180000.0;pos incr;stream_seek(cur_stream, pos, incr, 1);} else {pos get_master_clock(cur_stream);if (isnan(pos))pos (double)cur_stream-seek_pos / AV_TIME_BASE;pos incr; // 现在是秒的单位if (cur_stream-ic-start_time ! AV_NOPTS_VALUE pos cur_stream-ic-start_time / (double)AV_TIME_BASE)pos cur_stream-ic-start_time / (double)AV_TIME_BASE;stream_seek(cur_stream, (int64_t)(pos * AV_TIME_BASE), (int64_t)(incr * AV_TIME_BASE), 0);}break;default:break;}break;case SDL_MOUSEBUTTONDOWN: /* 鼠标按下事件 */if (exit_on_mousedown) {do_exit(cur_stream);break;}if (event.button.button SDL_BUTTON_LEFT) {static int64_t last_mouse_left_click 0;if (av_gettime_relative() - last_mouse_left_click 500000) {//连续鼠标左键点击2次显示窗口间隔小于0.5秒则进行全屏或者恢复原始窗口toggle_full_screen(cur_stream);cur_stream-force_refresh 1;last_mouse_left_click 0;} else {last_mouse_left_click av_gettime_relative();}}case SDL_MOUSEMOTION: /* 鼠标移动事件 */if (cursor_hidden) {SDL_ShowCursor(1);cursor_hidden 0;}cursor_last_shown av_gettime_relative();if (event.type SDL_MOUSEBUTTONDOWN) {if (event.button.button ! SDL_BUTTON_RIGHT)break;x event.button.x;} else {if (!(event.motion.state SDL_BUTTON_RMASK))break;x event.motion.x;}if (seek_by_bytes || cur_stream-ic-duration 0) {uint64_t size avio_size(cur_stream-ic-pb); // 整个文件的字节stream_seek(cur_stream, size*x/cur_stream-width, 0, 1);} else {int64_t ts;int ns, hh, mm, ss;int tns, thh, tmm, tss;tns cur_stream-ic-duration / 1000000LL;thh tns / 3600;tmm (tns % 3600) / 60;tss (tns % 60);frac x / cur_stream-width;ns frac * tns;hh ns / 3600;mm (ns % 3600) / 60;ss (ns % 60);av_log(NULL, AV_LOG_INFO,Seek to %2.0f%% (%2d:%02d:%02d) of total duration (%2d:%02d:%02d) \n, frac*100,hh, mm, ss, thh, tmm, tss);ts frac * cur_stream-ic-duration;if (cur_stream-ic-start_time ! AV_NOPTS_VALUE)ts cur_stream-ic-start_time;stream_seek(cur_stream, ts, 0, 0);}break;case SDL_WINDOWEVENT: /* 窗口事件 */switch (event.window.event) {case SDL_WINDOWEVENT_SIZE_CHANGED:screen_width cur_stream-width event.window.data1;screen_height cur_stream-height event.window.data2;if (cur_stream-vis_texture) {SDL_DestroyTexture(cur_stream-vis_texture);cur_stream-vis_texture NULL;}case SDL_WINDOWEVENT_EXPOSED:cur_stream-force_refresh 1;}break;case SDL_QUIT:case FF_QUIT_EVENT: /* ffplay自定义事件,用于主动退出 */do_exit(cur_stream);break;default:break;}} }这个函数主要是通过refresh_loop_wait_event函数等待事件,然后event_loop进行处理事件. video的显示主要在refresh_loop_wait_event中: static void refresh_loop_wait_event(VideoState *is, SDL_Event *event) {double remaining_time 0.0; /* 休眠等待remaining_time的计算在video_refresh中 *//* 调用SDL_PeepEvents前先调用SDL_PumpEvents将输入设备的事件抽到事件队列中 */SDL_PumpEvents();/** SDL_PeepEvents check是否事件比如鼠标移入显示区等* 从事件队列中拿一个事件放到event中如果没有事件则进入循环中* SDL_PeekEvents用于读取事件在调用该函数之前必须调用SDL_PumpEvents搜集键盘等事件*/while (!SDL_PeepEvents(event, 1, SDL_GETEVENT, SDL_FIRSTEVENT, SDL_LASTEVENT)) {if (!cursor_hidden av_gettime_relative() - cursor_last_shown CURSOR_HIDE_DELAY) {SDL_ShowCursor(0);cursor_hidden 1;}/** remaining_time就是用来进行音视频同步的。* 在video_refresh函数中根据当前帧显示时刻(display time)和实际时刻(actual time)* 计算需要sleep的时间保证帧按时显示*/if (remaining_time 0.0) //sleep控制画面输出的时机av_usleep((int64_t)(remaining_time * 1000000.0)); // remaining_time REFRESH_RATEremaining_time REFRESH_RATE;if (is-show_mode ! SHOW_MODE_NONE // 显示模式不等于SHOW_MODE_NONE(!is-paused // 非暂停状态|| is-force_refresh) // 强制刷新状态) {video_refresh(is, remaining_time);}/* 从输入设备中搜集事件推动这些事件进入事件队列更新事件队列的状态* 不过它还有一个作用是进行视频子系统的设备状态更新如果不调用这个函数* 所显示的视频会在大约10秒后丢失色彩。没有调用SDL_PumpEvents将不会* 有任何的输入设备事件进入队列这种情况下SDL就无法响应任何的键盘等硬件输入。*/SDL_PumpEvents();} }SDL_PeepEvent通过SDL_GETEVENT非阻塞查询队列中是否有事件,如果不为0则有事件发生(-1表示发生错误),那么函数就会返回,让event_loop进行处理;否则调用video_refresh进行显示画面,并且通过输出参数remaining_time获取下一轮应当sleep的时间以保证画面稳定输出. 是否调用video_refresh的前置条件为: 显示模式不为SHOW_MODE_NONE(如果只包含audio的画,也可能为其波形图)当前没有被暂停当设置了force_refresh(强制刷新),分析一下出现的情况: video_refresh里面帧显示,常规情况.SDL_WINDOWEVENT_EXPOSED,窗口需要重新绘制SDL_MOUSEBUTTONDOWN SDL_BUTTON_LEFT 鼠标按下并且按左键连续间隔小于0.5sSDLK_f,按f键进行全屏或者恢复原始播放窗口 有可能理解不了这个强制刷新是什么,看一下没有强制刷新的效果图: 可见下面一部分残缺了,因为我们改变了窗口大小,但是我们渲染的页面没有改变,这时我们重新刷新一下可以放渲染页面跟着窗口大小进行改变 1.2.2 video_refresh static void video_refresh(void *opaque, double *remaining_time) {VideoState *is opaque;double time;Frame *sp, *sp2;if (!is-paused get_master_sync_type(is) AV_SYNC_EXTERNAL_CLOCK is-realtime)check_external_clock_speed(is);if (!display_disable is-show_mode ! SHOW_MODE_VIDEO is-audio_st) {time av_gettime_relative() / 1000000.0;if (is-force_refresh || is-last_vis_time rdftspeed time) {video_display(is);is-last_vis_time time;}*remaining_time FFMIN(*remaining_time, is-last_vis_time rdftspeed - time);}if (is-video_st) {retry:if (frame_queue_nb_remaining(is-pictq) 0) {// 帧队列是否为空// nothing to do, no picture to display in the queue// 什么都不做队列中没有图像可显示} else { // 重点是音视频同步double last_duration, duration, delay;Frame *vp, *lastvp;/* dequeue the picture */// 从队列取出上一个Framelastvp frame_queue_peek_last(is-pictq);//读取上一帧vp frame_queue_peek(is-pictq); // 读取待显示帧// lastvp 上一帧(正在显示的帧)// vp 等待显示的帧if (vp-serial ! is-videoq.serial) {// 如果不是最新的播放序列则将其出队列以尽快读取最新序列的帧frame_queue_next(is-pictq);goto retry;}if (lastvp-serial ! vp-serial) {// 新的播放序列重置当前时间is-frame_timer av_gettime_relative() / 1000000.0;}if (is-paused){goto display;printf(视频暂停is-paused);}/* compute nominal last_duration *///lastvp上一帧vp当前帧 nextvp下一帧//last_duration 计算上一帧应显示的时长last_duration vp_duration(is, lastvp, vp);// 经过compute_target_delay方法计算出待显示帧vp需要等待的时间// 如果以video同步则delay直接等于last_duration。// 如果以audio或外部时钟同步则需要比对主时钟调整待显示帧vp要等待的时间。delay compute_target_delay(last_duration, is); // 上一帧需要维持的时间time av_gettime_relative()/1000000.0;// is-frame_timer 实际上就是上一帧lastvp的播放时间,// is-frame_timer delay 是待显示帧vp该播放的时间if (time is-frame_timer delay) { //判断是否继续显示上一帧// 当前系统时刻还未到达上一帧的结束时刻那么还应该继续显示上一帧。// 计算出最小等待时间*remaining_time FFMIN(is-frame_timer delay - time, *remaining_time);goto display;}// 走到这一步说明已经到了或过了该显示的时间待显示帧vp的状态变更为当前要显示的帧is-frame_timer delay; // 更新当前帧播放的时间if (delay 0 time - is-frame_timer AV_SYNC_THRESHOLD_MAX) {is-frame_timer time; //如果和系统时间差距太大就纠正为系统时间}SDL_LockMutex(is-pictq.mutex);if (!isnan(vp-pts))update_video_pts(is, vp-pts, vp-pos, vp-serial); // 更新video时钟SDL_UnlockMutex(is-pictq.mutex);//丢帧逻辑if (frame_queue_nb_remaining(is-pictq) 1) {//有nextvp才会检测是否该丢帧Frame *nextvp frame_queue_peek_next(is-pictq);duration vp_duration(is, vp, nextvp);if(!is-step // 非逐帧模式才检测是否需要丢帧 is-step1 为逐帧播放 (framedrop0 || // cpu解帧过慢(framedrop get_master_sync_type(is) ! AV_SYNC_VIDEO_MASTER)) // 非视频同步方式 time is-frame_timer duration // 确实落后了一帧数据) {printf(%s(%d) dif:%lfs, drop frame\n, __FUNCTION__, __LINE__,(is-frame_timer duration) - time);is-frame_drops_late; // 统计丢帧情况frame_queue_next(is-pictq); // 这里实现真正的丢帧//(这里不能直接while丢帧因为很可能audio clock重新对时了这样delay值需要重新计算)goto retry; //回到函数开始位置继续重试}}if (is-subtitle_st) {while (frame_queue_nb_remaining(is-subpq) 0) {sp frame_queue_peek(is-subpq);if (frame_queue_nb_remaining(is-subpq) 1)sp2 frame_queue_peek_next(is-subpq);elsesp2 NULL;if (sp-serial ! is-subtitleq.serial|| (is-vidclk.pts (sp-pts ((float) sp-sub.end_display_time / 1000)))|| (sp2 is-vidclk.pts (sp2-pts ((float) sp2-sub.start_display_time / 1000)))){if (sp-uploaded) {int i;for (i 0; i sp-sub.num_rects; i) {AVSubtitleRect *sub_rect sp-sub.rects[i];uint8_t *pixels;int pitch, j;if (!SDL_LockTexture(is-sub_texture, (SDL_Rect *)sub_rect, (void **)pixels, pitch)) {for (j 0; j sub_rect-h; j, pixels pitch)memset(pixels, 0, sub_rect-w 2);SDL_UnlockTexture(is-sub_texture);}}}frame_queue_next(is-subpq);} else {break;}}}frame_queue_next(is-pictq); // 当前vp帧出队列is-force_refresh 1; /* 说明需要刷新视频帧 */if (is-step !is-paused)stream_toggle_pause(is); // 逐帧的时候那继续进入暂停状态}display:/* display picture */if (!display_disable is-force_refresh is-show_mode SHOW_MODE_VIDEO is-pictq.rindex_shown)video_display(is); // 重点是显示}is-force_refresh 0;if (show_status) {static int64_t last_time;int64_t cur_time;int aqsize, vqsize, sqsize;double av_diff;cur_time av_gettime_relative();if (!last_time || (cur_time - last_time) 30000) {aqsize 0;vqsize 0;sqsize 0;if (is-audio_st)aqsize is-audioq.size;if (is-video_st)vqsize is-videoq.size;if (is-subtitle_st)sqsize is-subtitleq.size;av_diff 0;if (is-audio_st is-video_st)av_diff get_clock(is-audclk) - get_clock(is-vidclk);else if (is-video_st)av_diff get_master_clock(is) - get_clock(is-vidclk);else if (is-audio_st)av_diff get_master_clock(is) - get_clock(is-audclk);av_log(NULL, AV_LOG_INFO,%7.2f %s:%7.3f fd%4d aq%5dKB vq%5dKB sq%5dB f%PRId64/%PRId64 \r,get_master_clock(is),(is-audio_st is-video_st) ? A-V : (is-video_st ? M-V : (is-audio_st ? M-A : )),av_diff,is-frame_drops_early is-frame_drops_late,aqsize / 1024,vqsize / 1024,sqsize,is-video_st ? is-viddec.avctx-pts_correction_num_faulty_dts : 0,is-video_st ? is-viddec.avctx-pts_correction_num_faulty_pts : 0);fflush(stdout);last_time cur_time;}} }流程图 看主流程图 取出上一帧和待显示的帧计算上一帧显示的时长,判断当前是否继续上一帧估算当前帧显示时长,判断是否要丢帧调用video_display显示 1.2.2.1 计算上一帧显示时长,判断是否还要继续上一帧 首先判断pictq是否为空(调用frame_queue_nb_remaining判断是否还有未显示的帧),如果为空则继续调用video_display显示上一帧 在进一步计算上一帧显示时间之前,需要先判断下一帧vp是否为最新序列,也就是说if(vp-serial! is-videoq.serial),如果条件成立就是发生过seek等操作,此时应该丢弃lastvp. 故调用frame_queue_next抛弃lastvp后,返回流程开头重试. 接下来 可以计算lastvp显示时长了. 计算代码为: last_duration vp_duration(is, lastvp, vp); delay compute_target_delay(last_duration, is);//返回当前显示帧要持续播放的时间本质就是通过上一帧和待显示帧pts来计算的,如果考虑到同步,则还需要考虑当前与主时钟的差距来决定是重复上一帧,还是丢帧,还是正常显示下一帧(待显示帧) time av_gettime_relative()/1000000.0;// is-frame_timer 实际上就是上一帧lastvp的播放时间,// is-frame_timer delay 是待显示帧vp该播放的时间if (time is-frame_timer delay) { //判断是否继续显示上一帧// 当前系统时刻还未到达上一帧的结束时刻那么还应该继续显示上一帧。// 计算出最小等待时间*remaining_time FFMIN(is-frame_timer delay - time, *remaining_time);goto display;}frame_timer就是当前帧显示时间,如果当前帧显示时间delay显示时间大于当前系统时间的话就继续显示上一帧! 1.2.2.2 估算当前帧显示时长,判断是否要丢帧 is-frame_timer delay; // 更新当前帧播放的时间if (delay 0 time - is-frame_timer AV_SYNC_THRESHOLD_MAX) {is-frame_timer time; //如果和系统时间差距太大就纠正为系统时间}判断是否需要丢帧 if (frame_queue_nb_remaining(is-pictq) 1) {//有nextvp才会检测是否该丢帧Frame *nextvp frame_queue_peek_next(is-pictq);duration vp_duration(is, vp, nextvp);if(!is-step // 非逐帧模式才检测是否需要丢帧 is-step1 为逐帧播放 (framedrop0 || // cpu解帧过慢(framedrop get_master_sync_type(is) ! AV_SYNC_VIDEO_MASTER)) // 非视频同步方式 time is-frame_timer duration // 确实落后了一帧数据) {printf(%s(%d) dif:%lfs, drop frame\n, __FUNCTION__, __LINE__,(is-frame_timer duration) - time);is-frame_drops_late; // 统计丢帧情况frame_queue_next(is-pictq); // 这里实现真正的丢帧//(这里不能直接while丢帧因为很可能audio clock重新对时了这样delay值需要重新计算)goto retry; //回到函数开始位置继续重试}} 通过待显示帧和下一帧来计算当前帧显示时间(前提是得有下一帧) 并且要符合以下条件才会丢帧: 不处于step状态,逐帧状态启动framedrop模式,也就是cpu过慢时需要丢帧,并且不是以video为同步时钟的情况下当前时间已经frame_timerduration 1.2.2.3 调用video_display进行显示 static void video_display(VideoState *is) {if (!is-width)video_open(is); //如果窗口未显示则显示窗口SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255);SDL_RenderClear(renderer);if (is-audio_st is-show_mode ! SHOW_MODE_VIDEO)video_audio_display(is); //图形化显示仅有音轨的文件else if (is-video_st)video_image_display(is); //显示一帧视频画面SDL_RenderPresent(renderer); } static void video_image_display(VideoState *is) {Frame *vp;Frame *sp NULL;SDL_Rect rect;// keep_last的作用就出来了,我们是有调用frame_queue_next, 但最近出队列的帧并没有真正销毁// 所以这里可以读取出来显示vp frame_queue_peek_last(is-pictq); //if (is-subtitle_st) {if (frame_queue_nb_remaining(is-subpq) 0) {sp frame_queue_peek(is-subpq);if (vp-pts sp-pts ((float) sp-sub.start_display_time / 1000)) {if (!sp-uploaded) {uint8_t* pixels[4];int pitch[4];int i;if (!sp-width || !sp-height) {sp-width vp-width;sp-height vp-height;}if (realloc_texture(is-sub_texture, SDL_PIXELFORMAT_ARGB8888, sp-width, sp-height, SDL_BLENDMODE_BLEND, 1) 0)return;for (i 0; i sp-sub.num_rects; i) {AVSubtitleRect *sub_rect sp-sub.rects[i];sub_rect-x av_clip(sub_rect-x, 0, sp-width );sub_rect-y av_clip(sub_rect-y, 0, sp-height);sub_rect-w av_clip(sub_rect-w, 0, sp-width - sub_rect-x);sub_rect-h av_clip(sub_rect-h, 0, sp-height - sub_rect-y);is-sub_convert_ctx sws_getCachedContext(is-sub_convert_ctx,sub_rect-w, sub_rect-h, AV_PIX_FMT_PAL8,sub_rect-w, sub_rect-h, AV_PIX_FMT_BGRA,0, NULL, NULL, NULL);if (!is-sub_convert_ctx) {av_log(NULL, AV_LOG_FATAL, Cannot initialize the conversion context\n);return;}if (!SDL_LockTexture(is-sub_texture, (SDL_Rect *)sub_rect, (void **)pixels, pitch)) {sws_scale(is-sub_convert_ctx, (const uint8_t * const *)sub_rect-data, sub_rect-linesize,0, sub_rect-h, pixels, pitch);SDL_UnlockTexture(is-sub_texture);}}sp-uploaded 1;}} elsesp NULL;}}//将帧宽高按照sar最大适配到窗口并通过rect返回视频帧在窗口的显示位置和宽高calculate_display_rect(rect, is-xleft, is-ytop, is-width, is-height,vp-width, vp-height, vp-sar);// rect.x rect.w /2; // 测试// rect.w rect.w /2; // 缩放实际不是用sws 缩放是sdl去做的if (!vp-uploaded) {// 把yuv数据更新到vid_textureif (upload_texture(is-vid_texture, vp-frame, is-img_convert_ctx) 0)return;vp-uploaded 1;vp-flip_v vp-frame-linesize[0] 0;}set_sdl_yuv_conversion_mode(vp-frame);SDL_RenderCopyEx(renderer, is-vid_texture, NULL, rect, 0, NULL, vp-flip_v ? SDL_FLIP_VERTICAL : 0);set_sdl_yuv_conversion_mode(NULL);if (sp) { #if USE_ONEPASS_SUBTITLE_RENDERSDL_RenderCopy(renderer, is-sub_texture, NULL, rect); #elseint i;double xratio (double)rect.w / (double)sp-width;double yratio (double)rect.h / (double)sp-height;for (i 0; i sp-sub.num_rects; i) {SDL_Rect *sub_rect (SDL_Rect*)sp-sub.rects[i];SDL_Rect target {.x rect.x sub_rect-x * xratio,.y rect.y sub_rect-y * yratio,.w sub_rect-w * xratio,.h sub_rect-h * yratio};SDL_RenderCopy(renderer, is-sub_texture, sub_rect, target);} #endif} } video_image_display整体不算复杂,每次渲染都会调用calculate_display_rect进行重新计算显示窗口等 最主要的显示是调用upload_texture将AVFrame的图像数据传给sdl的纹理进行渲染: static int upload_texture(SDL_Texture **tex, AVFrame *frame, struct SwsContext **img_convert_ctx) {int ret 0;Uint32 sdl_pix_fmt;SDL_BlendMode sdl_blendmode;// 根据frame中的图像格式(FFmpeg像素格式)获取对应的SDL像素格式和blendmodeget_sdl_pix_fmt_and_blendmode(frame-format, sdl_pix_fmt, sdl_blendmode);// 参数tex实际是is-vid_texture此处根据得到的SDL像素格式为is-vid_textureif (realloc_texture(tex, sdl_pix_fmt SDL_PIXELFORMAT_UNKNOWN ? SDL_PIXELFORMAT_ARGB8888 : sdl_pix_fmt,frame-width, frame-height, sdl_blendmode, 0) 0)return -1;//根据sdl_pix_fmt从AVFrame中取数据填充纹理switch (sdl_pix_fmt) {// frame格式是SDL不支持的格式则需要进行图像格式转换转换为目标格式AV_PIX_FMT_BGRA// 对应SDL_PIXELFORMAT_BGRA32case SDL_PIXELFORMAT_UNKNOWN:/* This should only happen if we are not using avfilter... */*img_convert_ctx sws_getCachedContext(*img_convert_ctx,frame-width, frame-height, frame-format,frame-width, frame-height, AV_PIX_FMT_BGRA,sws_flags, NULL, NULL, NULL);if (*img_convert_ctx ! NULL) {uint8_t *pixels[4]; // 之前取Texture的缓存int pitch[4];if (!SDL_LockTexture(*tex, NULL, (void **)pixels, pitch)) {sws_scale(*img_convert_ctx, (const uint8_t * const *)frame-data, frame-linesize,0, frame-height, pixels, pitch);SDL_UnlockTexture(*tex);}} else {av_log(NULL, AV_LOG_FATAL, Cannot initialize the conversion context\n);ret -1;}break;// frame格式对应SDL_PIXELFORMAT_IYUV不用进行图像格式转换调用SDL_UpdateYUVTexture()更新SDL texturecase SDL_PIXELFORMAT_IYUV:if (frame-linesize[0] 0 frame-linesize[1] 0 frame-linesize[2] 0) {ret SDL_UpdateYUVTexture(*tex, NULL, frame-data[0], frame-linesize[0],frame-data[1], frame-linesize[1],frame-data[2], frame-linesize[2]);} else if (frame-linesize[0] 0 frame-linesize[1] 0 frame-linesize[2] 0) {ret SDL_UpdateYUVTexture(*tex, NULL, frame-data[0] frame-linesize[0] * (frame-height - 1), -frame-linesize[0],frame-data[1] frame-linesize[1] * (AV_CEIL_RSHIFT(frame-height, 1) - 1), -frame-linesize[1],frame-data[2] frame-linesize[2] * (AV_CEIL_RSHIFT(frame-height, 1) - 1), -frame-linesize[2]);} else {av_log(NULL, AV_LOG_ERROR, Mixed negative and positive linesizes are not supported.\n);return -1;}break;// frame格式对应其他SDL像素格式不用进行图像格式转换调用SDL_UpdateTexture()更新SDL texturedefault:if (frame-linesize[0] 0) {ret SDL_UpdateTexture(*tex, NULL, frame-data[0] frame-linesize[0] * (frame-height - 1), -frame-linesize[0]);} else {ret SDL_UpdateTexture(*tex, NULL, frame-data[0], frame-linesize[0]);}break;}return ret; }frame中的像素格式是FFmpeg中定义的像素格式,FFmpeg中定义的很多像素格式与SDL中定义的像素格式是同一种格式,只不过是名称不同 根据frame中的像素格式与SDL的像素格式的匹配情况,upload_texture()处理三种类型,对应的是Switch语句的三种分支: 如果frame图像格式对应SDL_PIXELFORMAT_IYUV格式,则不需要图像格式转换,使用SDL_updateYUVBTexture()将数据更新到is-vid_texture中如果frame图像格式对应SDL其他支持的格式,也不需要进行图像格式转换,使用SDL_updateTexture()将数据更新到is-vid_texture中如果frame图像不被SDL支持的话,则需要进行图像格式转换 根据映射表获取frame对应SDL中的像素格式: static void get_sdl_pix_fmt_and_blendmode(int format, Uint32 *sdl_pix_fmt, SDL_BlendMode *sdl_blendmode) {int i;*sdl_blendmode SDL_BLENDMODE_NONE;*sdl_pix_fmt SDL_PIXELFORMAT_UNKNOWN;if (format AV_PIX_FMT_RGB32 ||format AV_PIX_FMT_RGB32_1 ||format AV_PIX_FMT_BGR32 ||format AV_PIX_FMT_BGR32_1)*sdl_blendmode SDL_BLENDMODE_BLEND;for (i 0; i FF_ARRAY_ELEMS(sdl_texture_format_map) - 1; i) {if (format sdl_texture_format_map[i].format) {*sdl_pix_fmt sdl_texture_format_map[i].texture_fmt;return;}} }映射表: static const struct TextureFormatEntry {enum AVPixelFormat format;int texture_fmt;} sdl_texture_format_map[] { // FFmpeg PIX_FMT to SDL_PIX的映射关系{ AV_PIX_FMT_RGB8, SDL_PIXELFORMAT_RGB332 },{ AV_PIX_FMT_RGB444, SDL_PIXELFORMAT_RGB444 },{ AV_PIX_FMT_RGB555, SDL_PIXELFORMAT_RGB555 },{ AV_PIX_FMT_BGR555, SDL_PIXELFORMAT_BGR555 },{ AV_PIX_FMT_RGB565, SDL_PIXELFORMAT_RGB565 },{ AV_PIX_FMT_BGR565, SDL_PIXELFORMAT_BGR565 },{ AV_PIX_FMT_RGB24, SDL_PIXELFORMAT_RGB24 },{ AV_PIX_FMT_BGR24, SDL_PIXELFORMAT_BGR24 },{ AV_PIX_FMT_0RGB32, SDL_PIXELFORMAT_RGB888 },{ AV_PIX_FMT_0BGR32, SDL_PIXELFORMAT_BGR888 },{ AV_PIX_FMT_NE(RGB0, 0BGR), SDL_PIXELFORMAT_RGBX8888 },{ AV_PIX_FMT_NE(BGR0, 0RGB), SDL_PIXELFORMAT_BGRX8888 },{ AV_PIX_FMT_RGB32, SDL_PIXELFORMAT_ARGB8888 },{ AV_PIX_FMT_RGB32_1, SDL_PIXELFORMAT_RGBA8888 },{ AV_PIX_FMT_BGR32, SDL_PIXELFORMAT_ABGR8888 },{ AV_PIX_FMT_BGR32_1, SDL_PIXELFORMAT_BGRA8888 },{ AV_PIX_FMT_YUV420P, SDL_PIXELFORMAT_IYUV },{ AV_PIX_FMT_YUYV422, SDL_PIXELFORMAT_YUY2 },{ AV_PIX_FMT_UYVY422, SDL_PIXELFORMAT_UYVY },{ AV_PIX_FMT_NONE, SDL_PIXELFORMAT_UNKNOWN },};可以看到除了最后⼀项其他格式的图像送给SDL是可以直接显示的不必进行图像转换。 1.2.2.4 realloc_texture()重新分配vid_texture static int realloc_texture(SDL_Texture **texture, Uint32 new_format, int new_width, int new_height,SDL_BlendMode blendmode, int init_texture) {Uint32 format;int access, w, h;if (!*texture || SDL_QueryTexture(*texture, format, access, w, h) 0 || new_width ! w || new_height ! h || new_format ! format) {void *pixels;int pitch;if (*texture)SDL_DestroyTexture(*texture);if (!(*texture SDL_CreateTexture(renderer, new_format, SDL_TEXTUREACCESS_STREAMING, new_width, new_height)))return -1;if (SDL_SetTextureBlendMode(*texture, blendmode) 0)return -1;if (init_texture) {if (SDL_LockTexture(*texture, NULL, pixels, pitch) 0)return -1;memset(pixels, 0, pitch * new_height);SDL_UnlockTexture(*texture);}av_log(NULL, AV_LOG_VERBOSE, Created %dx%d texture with %s.\n, new_width, new_height, SDL_GetPixelFormatName(new_format));}return 0; }什么情况下需要realloc_texture? 用于显示的texture没有分配SDL_QueryTexture无效目前texture的width,height,format和新药显示的Frame不一致 综上所述:窗口大小变化不足以让realloc_texture重新SDL_CreateTextue 1.2.2.5 sws_getCachedContext struct SwsContext *sws_getCachedContext(struct SwsContext *context,int srcW, int srcH, enum AVPixelFormat srcFormat,int dstW, int dstH, enum AVPixelFormat dstFormat,int flags, SwsFilter *srcFilter,SwsFilter *dstFilter, const double *param); 创建一个图像转换的上下文 这里需要说明的flags这个参数,这个参数是选择转换算法的,有很多转换算法,在libswscale/swscale.h文件中 1.2.2.6 sws_scale 图像转换 int sws_scale(struct SwsContext *c, const uint8_t *const srcSlice[],const int srcStride[], int srcSliceY, int srcSliceH,uint8_t *const dst[], const int dstStride[]);if (!SDL_LockTexture(*tex, NULL, (void **)pixels, pitch)) {sws_scale(*img_convert_ctx, (const uint8_t * const *)frame-data, frame-linesize,0, frame-height, pixels, pitch);SDL_UnlockTexture(*tex);}对于flags算法测试推荐文章: (66条消息) ffmpeg中的sws_scale算法性能测试_ffmpeg算法_雷霄骅的博客-CSDN博客