[FFmpeg-devel] [PATCH 2/2] avformat/hlsenc: reopen new http session for http_persistent when upload file failed
Steven Liu
lq at chinaffmpeg.org
Sun Aug 25 17:57:40 EEST 2019
> 在 2019年8月25日，22:53，Steven Liu <lq at chinaffmpeg.org> 写道：
>
> fix ticket: 7975
>
> Signed-off-by: Steven Liu <lq at chinaffmpeg.org>
> ---
> libavformat/hlsenc.c | 66 ++++++++++++++++++++++++++++++++++++++++++----------
> 1 file changed, 54 insertions(+), 12 deletions(-)
>
> diff --git a/libavformat/hlsenc.c b/libavformat/hlsenc.c
> index 18173cdce1..a4cb832766 100644
> --- a/libavformat/hlsenc.c
> +++ b/libavformat/hlsenc.c
> @@ -118,6 +118,7 @@ typedef struct VariantStream {
> AVIOContext *out;
> int packets_written;
> int init_range_length;
> + uint8_t *temp_buffer;
>
> AVFormatContext *avf;
> AVFormatContext *vtt_avf;
> @@ -262,11 +263,12 @@ static int hlsenc_io_open(AVFormatContext *s, AVIOContext **pb, char *filename,
> return err;
> }
>
> -static void hlsenc_io_close(AVFormatContext *s, AVIOContext **pb, char *filename) {
> +static int hlsenc_io_close(AVFormatContext *s, AVIOContext **pb, char *filename) {
> HLSContext *hls = s->priv_data;
> int http_base_proto = filename ? ff_is_http_proto(filename) : 0;
> + int ret = 0;
> if (!*pb)
> - return;
> + return ret;
> if (!http_base_proto || !hls->http_persistent || hls->key_info_file || hls->encrypt) {
> ff_format_io_close(s, pb);
> #if CONFIG_HTTP_PROTOCOL
> @@ -275,8 +277,10 @@ static void hlsenc_io_close(AVFormatContext *s, AVIOContext **pb, char *filename
> av_assert0(http_url_context);
> avio_flush(*pb);
> ffurl_shutdown(http_url_context, AVIO_FLAG_WRITE);
> + ret = ff_http_get_shutdown_status(http_url_context);
> #endif
> }
> + return ret;
> }
>
> static void set_http_options(AVFormatContext *s, AVDictionary **options, HLSContext *c)
> @@ -447,7 +451,6 @@ static void write_styp(AVIOContext *pb)
> static int flush_dynbuf(VariantStream *vs, int *range_length)
> {
> AVFormatContext *ctx = vs->avf;
> - uint8_t *buffer;
>
> if (!ctx->pb) {
> return AVERROR(EINVAL);
> @@ -458,15 +461,20 @@ static int flush_dynbuf(VariantStream *vs, int *range_length)
> avio_flush(ctx->pb);
>
> // write out to file
> - *range_length = avio_close_dyn_buf(ctx->pb, &buffer);
> + *range_length = avio_close_dyn_buf(ctx->pb, &vs->temp_buffer);
> ctx->pb = NULL;
> - avio_write(vs->out, buffer, *range_length);
> - av_free(buffer);
> + avio_write(vs->out, vs->temp_buffer, *range_length);
>
> // re-open buffer
> return avio_open_dyn_buf(&ctx->pb);
> }
>
> +static void reflush_dynbuf(VariantStream *vs, int *range_length)
> +{
> + // re-open buffer
> + avio_write(vs->out, vs->temp_buffer, *range_length);;
> +}
> +
> static int hls_delete_old_segments(AVFormatContext *s, HLSContext *hls,
> VariantStream *vs) {
>
> @@ -1544,7 +1552,10 @@ static int hls_window(AVFormatContext *s, int last, VariantStream *vs)
>
> fail:
> av_dict_free(&options);
> - hlsenc_io_close(s, (byterange_mode || hls->segment_type == SEGMENT_TYPE_FMP4) ? &hls->m3u8_out : &vs->out, temp_filename);
> + ret = hlsenc_io_close(s, (byterange_mode || hls->segment_type == SEGMENT_TYPE_FMP4) ? &hls->m3u8_out : &vs->out, temp_filename);
> + if (ret < 0) {
> + return ret;
> + }
> hlsenc_io_close(s, &hls->sub_m3u8_out, vs->vtt_m3u8_name);
> if (use_temp_file) {
> ff_rename(temp_filename, vs->m3u8_name, s);
> @@ -2399,7 +2410,16 @@ static int hls_write_packet(AVFormatContext *s, AVPacket *pkt)
> if (ret < 0) {
> return ret;
> }
> - hlsenc_io_close(s, &vs->out, filename);
> + ret = hlsenc_io_close(s, &vs->out, filename);
> + if (ret < 0) {
> + av_log(s, AV_LOG_WARNING, "upload segment failed,"
> + "and will retry upload by a new http session.\n");
> + ff_format_io_close(s, &vs->out);
> + ret = hlsenc_io_open(s, &vs->out, filename, &options);
> + reflush_dynbuf(vs, &range_length);
> + ret = hlsenc_io_close(s, &vs->out, filename);
> + }
> + av_free(vs->temp_buffer);
> av_free(filename);
> }
> }
> @@ -2426,8 +2446,13 @@ static int hls_write_packet(AVFormatContext *s, AVPacket *pkt)
> // if we're building a VOD playlist, skip writing the manifest multiple times, and just wait until the end
> if (hls->pl_type != PLAYLIST_TYPE_VOD) {
> if ((ret = hls_window(s, 0, vs)) < 0) {
> - av_free(old_filename);
> - return ret;
> + av_log(s, AV_LOG_WARNING, "update playlist failed, will retry once time\n");
> + ff_format_io_close(s, &vs->out);
> + vs->out = NULL;
> + if ((ret = hls_window(s, 0, vs)) < 0) {
> + av_free(old_filename);
> + return ret;
> + }
> }
> }
>
> @@ -2577,7 +2602,19 @@ static int hls_write_trailer(struct AVFormatContext *s)
> goto failed;
>
> vs->size = range_length;
> - hlsenc_io_close(s, &vs->out, filename);
> + ret = hlsenc_io_close(s, &vs->out, filename);
> + if (ret < 0) {
> + av_log(s, AV_LOG_WARNING, "write segment failed, will close old handle and retry once time\n");
> + ff_format_io_close(s, &vs->out);
> + ret = hlsenc_io_open(s, &vs->out, filename, &options);
> + if (ret < 0) {
> + av_log(s, AV_LOG_ERROR, "Failed to open file '%s'\n", vs->avf->url);
> + goto failed;
> + }
> + reflush_dynbuf(vs, &range_length);
> + ret = hlsenc_io_close(s, &vs->out, filename);
> + }
> + av_free(vs->temp_buffer);
> av_free(filename);
>
> failed:
> @@ -2610,7 +2647,12 @@ failed:
> ff_format_io_close(s, &vtt_oc->pb);
> avformat_free_context(vtt_oc);
> }
> - hls_window(s, 1, vs);
> + ret = hls_window(s, 1, vs);
> + if (ret < 0) {
> + av_log(s, AV_LOG_WARNING, "update playlist failed, will retry once time\n");
> + ff_format_io_close(s, &vs->out);
> + hls_window(s, 1, vs);
> + }
> avformat_free_context(oc);
>
> vs->avf = NULL;
> --
> 2.15.1
>
This patchiest test result:
liuqideMBP:dash liuqi$ ./ffmpeg -f lavfi -i testsrc2=s=176x144 -vcodec libx264 -g 50 -r:v 25 -f hls -hls_time 2 -http_persistent 1 -method PUT -t 20 http://127.0.0.1/output.m3u8
ffmpeg version N-94621-g311803b899 Copyright (c) 2000-2019 the FFmpeg developers
built with Apple LLVM version 10.0.0 (clang-1000.11.45.5)
configuration: --enable-fontconfig --enable-gpl --enable-libass --enable-libbluray --enable-libfreetype --enable-libmp3lame --enable-libspeex --enable-libx264 --enable-libx265 --enable-libfdk-aac --enable-version3 --enable-nonfree --enable-videotoolbox --enable-libxml2 --samples=fate-suite/ --enable-libopencv
libavutil 56. 33.100 / 56. 33.100
libavcodec 58. 55.100 / 58. 55.100
libavformat 58. 31.101 / 58. 31.101
libavdevice 58. 9.100 / 58. 9.100
libavfilter 7. 58.101 / 7. 58.101
libswscale 5. 6.100 / 5. 6.100
libswresample 3. 6.100 / 3. 6.100
libpostproc 55. 6.100 / 55. 6.100
Input #0, lavfi, from 'testsrc2=s=176x144':
Duration: N/A, start: 0.000000, bitrate: N/A
Stream #0:0: Video: rawvideo (I420 / 0x30323449), yuv420p, 176x144 [SAR 1:1 DAR 11:9], 25 tbr, 25 tbn, 25 tbc
Stream mapping:
Stream #0:0 -> #0:0 (rawvideo (native) -> h264 (libx264))
Press [q] to stop, [?] for help
[libx264 @ 0x7f888281b800] using SAR=1/1
[libx264 @ 0x7f888281b800] using cpu capabilities: MMX2 SSE2Fast SSSE3 SSE4.2 AVX
[libx264 @ 0x7f888281b800] profile High, level 1.1
[libx264 @ 0x7f888281b800] 264 - core 148 r2694 3b70645 - H.264/MPEG-4 AVC codec - Copyleft 2003-2016 - http://www.videolan.org/x264.html - options: cabac=1 ref=3 deblock=1:0:0 analyse=0x3:0x113 me=hex subme=7 psy=1 psy_rd=1.00:0.00 mixed_ref=1 me_range=16 chroma_me=1 trellis=1 8x8dct=1 cqm=0 deadzone=21,11 fast_pskip=1 chroma_qp_offset=-2 threads=4 lookahead_threads=1 sliced_threads=0 nr=0 decimate=1 interlaced=0 bluray_compat=0 constrained_intra=0 bframes=3 b_pyramid=2 b_adapt=1 b_bias=0 direct=1 weightb=1 open_gop=0 weightp=2 keyint=50 keyint_min=5 scenecut=40 intra_refresh=0 rc_lookahead=40 rc=crf mbtree=1 crf=23.0 qcomp=0.60 qpmin=0 qpmax=69 qpstep=4 ip_ratio=1.40 aq=1:1.00
Output #0, hls, to 'http://127.0.0.1/output.m3u8':
Metadata:
encoder : Lavf58.31.101
Stream #0:0: Video: h264 (libx264), yuv420p, 176x144 [SAR 1:1 DAR 11:9], q=-1--1, 25 fps, 90k tbn, 25 tbc
Metadata:
encoder : Lavc58.55.100 libx264
Side data:
cpb: bitrate max/min/avg: 0/0/0 buffer size: 0 vbv_delay: 18446744073709551615
[hls @ 0x7f8882818400] Opening 'http://127.0.0.1/output0.ts' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output.m3u8' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output1.ts' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output.m3u8' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output2.ts' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output.m3u8' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output3.ts' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output.m3u8' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output4.ts' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output.m3u8' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output5.ts' for writing
[http @ 0x7f8882406080] URL read error: End of file
[hls @ 0x7f8882818400] upload segment failed,and will retry upload by a new http session.
[hls @ 0x7f8882818400] Opening 'http://127.0.0.1/output5.ts' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output.m3u8' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output6.ts' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output.m3u8' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output7.ts' for writing.2x
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output.m3u8' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output8.ts' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output.m3u8' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output9.ts' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output.m3u8' for writing
frame= 500 fps=0.0 q=-1.0 Lsize=N/A time=00:00:19.92 bitrate=N/A speed=29.4x
video:413kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: unknown
[libx264 @ 0x7f888281b800] frame I:10 Avg QP:20.41 size: 3776
[libx264 @ 0x7f888281b800] frame P:138 Avg QP:26.88 size: 1239
[libx264 @ 0x7f888281b800] frame B:352 Avg QP:32.05 size: 606
[libx264 @ 0x7f888281b800] consecutive B-frames: 3.8% 4.4% 7.8% 84.0%
[libx264 @ 0x7f888281b800] mb I I16..4: 32.3% 15.9% 51.8%
[libx264 @ 0x7f888281b800] mb P I16..4: 3.5% 2.6% 2.6% P16..4: 18.0% 16.0% 13.9% 0.0% 0.0% skip:43.5%
[libx264 @ 0x7f888281b800] mb B I16..4: 0.3% 0.2% 0.1% B16..8: 26.7% 13.0% 4.4% direct: 4.0% skip:51.4% L0:46.8% L1:46.6% BI: 6.6%
[libx264 @ 0x7f888281b800] 8x8 transform intra:24.0% inter:24.8%
[libx264 @ 0x7f888281b800] coded y,uvDC,uvAC intra: 26.1% 56.5% 46.9% inter: 10.0% 25.3% 21.1%
[libx264 @ 0x7f888281b800] i16 v,h,dc,p: 74% 17% 9% 0%
[libx264 @ 0x7f888281b800] i8 v,h,dc,ddl,ddr,vr,hd,vl,hu: 16% 9% 74% 1% 0% 0% 0% 0% 1%
[libx264 @ 0x7f888281b800] i4 v,h,dc,ddl,ddr,vr,hd,vl,hu: 47% 25% 20% 1% 1% 1% 2% 2% 2%
[libx264 @ 0x7f888281b800] i8c dc,h,v,p: 39% 20% 39% 2%
[libx264 @ 0x7f888281b800] Weighted P-Frames: Y:0.0% UV:0.0%
[libx264 @ 0x7f888281b800] ref P L0: 60.1% 8.2% 20.1% 11.6%
[libx264 @ 0x7f888281b800] ref B L0: 78.3% 17.9% 3.8%
[libx264 @ 0x7f888281b800] ref B L1: 90.6% 9.4%
[libx264 @ 0x7f888281b800] kb/s:168.78
liuqideMBP:dash liuqi$
Nginx log:
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output0.ts HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output.m3u8 HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output1.ts HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output.m3u8 HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output2.ts HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output.m3u8 HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output3.ts HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output.m3u8 HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output4.ts HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output.m3u8 HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output5.ts HTTP/1.1" 201 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output.m3u8 HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output6.ts HTTP/1.1" 201 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output.m3u8 HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output7.ts HTTP/1.1" 201 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output.m3u8 HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output8.ts HTTP/1.1" 201 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output.m3u8 HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output9.ts HTTP/1.1" 201 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output.m3u8 HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
the files in http server:
liuqideMBP:~ liuqi$ ls /usr/local/nginx/html/output*
/usr/local/nginx/html/output.m3u8 /usr/local/nginx/html/output2.ts /usr/local/nginx/html/output5.ts /usr/local/nginx/html/output8.ts
/usr/local/nginx/html/output0.ts /usr/local/nginx/html/output3.ts /usr/local/nginx/html/output6.ts /usr/local/nginx/html/output9.ts
/usr/local/nginx/html/output1.ts /usr/local/nginx/html/output4.ts /usr/local/nginx/html/output7.ts
liuqideMBP:~ liuqi$
http config file:
http {
include mime.types;
default_type application/octet-stream;
keepalive_timeout 100;
keepalive_requests 10;
Thanks
Steven
More information about the ffmpeg-devel
mailing list