[FFmpeg-devel] [PATCH v6 3/3] avcodec/qsvdec: Implement SEI parsing for QSV decoders
Xiang, Haihao
haihao.xiang at intel.com
Mon Nov 21 04:44:36 EET 2022
On Tue, 2022-10-25 at 04:03 +0000, softworkz wrote:
> From: softworkz <softworkz at hotmail.com>
>
> Signed-off-by: softworkz <softworkz at hotmail.com>
> ---
> libavcodec/Makefile | 2 +-
> libavcodec/qsvdec.c | 321 ++++++++++++++++++++++++++++++++++++++++++++
> 2 files changed, 322 insertions(+), 1 deletion(-)
>
> diff --git a/libavcodec/Makefile b/libavcodec/Makefile
> index 90c7f113a3..cbddbb0ace 100644
> --- a/libavcodec/Makefile
> +++ b/libavcodec/Makefile
> @@ -146,7 +146,7 @@ OBJS-$(CONFIG_MSS34DSP) += mss34dsp.o
> OBJS-$(CONFIG_PIXBLOCKDSP) += pixblockdsp.o
> OBJS-$(CONFIG_QPELDSP) += qpeldsp.o
> OBJS-$(CONFIG_QSV) += qsv.o
> -OBJS-$(CONFIG_QSVDEC) += qsvdec.o
> +OBJS-$(CONFIG_QSVDEC) += qsvdec.o h264_sei.o hevc_sei.o
> OBJS-$(CONFIG_QSVENC) += qsvenc.o
> OBJS-$(CONFIG_RANGECODER) += rangecoder.o
> OBJS-$(CONFIG_RDFT) += rdft.o
> diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
> index 73405b5747..467a248224 100644
> --- a/libavcodec/qsvdec.c
> +++ b/libavcodec/qsvdec.c
> @@ -41,6 +41,7 @@
> #include "libavutil/time.h"
> #include "libavutil/imgutils.h"
> #include "libavutil/film_grain_params.h"
> +#include <libavutil/reverse.h>
>
> #include "avcodec.h"
> #include "codec_internal.h"
> @@ -49,6 +50,9 @@
> #include "hwconfig.h"
> #include "qsv.h"
> #include "qsv_internal.h"
> +#include "h264_sei.h"
> +#include "hevc_ps.h"
> +#include "hevc_sei.h"
>
> #if QSV_ONEVPL
> #include <mfxdispatcher.h>
> @@ -66,6 +70,8 @@ static const AVRational mfx_tb = { 1, 90000 };
> AV_NOPTS_VALUE : pts_tb.num ? \
> av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts)
>
> +#define PAYLOAD_BUFFER_SIZE 65535
> +
> typedef struct QSVAsyncFrame {
> mfxSyncPoint *sync;
> QSVFrame *frame;
> @@ -107,6 +113,9 @@ typedef struct QSVContext {
>
> mfxExtBuffer **ext_buffers;
> int nb_ext_buffers;
> +
> + mfxU8 payload_buffer[PAYLOAD_BUFFER_SIZE];
> + AVBufferRef *a53_buf_ref;
> } QSVContext;
>
> static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
> @@ -628,6 +637,299 @@ static int qsv_export_film_grain(AVCodecContext *avctx,
> mfxExtAV1FilmGrainParam
> }
> #endif
>
> +static int find_start_offset(mfxU8 data[4])
> +{
> + if (data[0] == 0 && data[1] == 0 && data[2] == 1)
> + return 3;
> +
> + if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1)
> + return 4;
> +
> + return 0;
> +}
> +
> +static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q, AVFrame* out)
> +{
> + H264SEIContext sei = { 0 };
> + GetBitContext gb = { 0 };
> + mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize =
> sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
> + mfxU64 ts;
> + int ret;
> +
> + while (1) {
> + int start;
> + memset(payload.Data, 0, payload.BufSize);
> +
> + ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
> + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on
> GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer),
> payload.BufSize);
> + return 0;
> + }
> + if (ret != MFX_ERR_NONE)
> + return ret;
> +
> + if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
> + break;
> +
> + start = find_start_offset(payload.Data);
> +
> + switch (payload.Type) {
> + case SEI_TYPE_BUFFERING_PERIOD:
> + case SEI_TYPE_PIC_TIMING:
> + continue;
> + }
> +
> + if (init_get_bits(&gb, &payload.Data[start], payload.NumBit - start *
> 8) < 0)
> + av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader
> SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> + else {
> + ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx);
> +
> + if (ret < 0)
> + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI type:
> %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> + else
> + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits
> %d\n", payload.Type, payload.NumBit);
> + }
> + }
> +
> + if (out)
> + return ff_h264_set_sei_to_frame(avctx, &sei, out, NULL, 0);
> +
> + return 0;
> +}
> +
> +static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q, QSVFrame*
> out)
> +{
> + HEVCSEI sei = { 0 };
> + HEVCParamSets ps = { 0 };
> + GetBitContext gb = { 0 };
> + mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize =
> sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
> + mfxFrameSurface1 *surface = &out->surface;
> + mfxU64 ts;
> + int ret, has_logged = 0;
> +
> + while (1) {
> + int start;
> + memset(payload.Data, 0, payload.BufSize);
> +
> + ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
> + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on
> GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer),
> payload.BufSize);
> + return 0;
> + }
> + if (ret != MFX_ERR_NONE)
> + return ret;
> +
> + if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
> + break;
> +
> + if (!has_logged) {
> + has_logged = 1;
> + av_log(avctx, AV_LOG_VERBOSE, "--------------------------------
> ---------\n");
> + av_log(avctx, AV_LOG_VERBOSE, "Start reading SEI - payload
> timestamp: %llu - surface timestamp: %llu\n", ts, surface->Data.TimeStamp);
> + }
> +
> + if (ts != surface->Data.TimeStamp) {
> + av_log(avctx, AV_LOG_WARNING, "GetPayload timestamp (%llu) does
> not match surface timestamp: (%llu)\n", ts, surface->Data.TimeStamp);
> + }
> +
> + start = find_start_offset(payload.Data);
> +
> + av_log(avctx, AV_LOG_VERBOSE, "parsing SEI type: %3d Numbits
> %3d Start: %d\n", payload.Type, payload.NumBit, start);
> +
> + switch (payload.Type) {
> + case SEI_TYPE_BUFFERING_PERIOD:
> + case SEI_TYPE_PIC_TIMING:
> + continue;
> + case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME:
> + // There seems to be a bug in MSDK
> + payload.NumBit -= 8;
> +
> + break;
> + case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO:
> + // There seems to be a bug in MSDK
> + payload.NumBit = 48;
> +
> + break;
> + case SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35:
> + // There seems to be a bug in MSDK
> + if (payload.NumBit == 552)
> + payload.NumBit = 528;
> + break;
> + }
> +
> + if (init_get_bits(&gb, &payload.Data[start], payload.NumBit - start *
> 8) < 0)
> + av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader
> SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> + else {
> + ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps,
> HEVC_NAL_SEI_PREFIX);
> +
> + if (ret < 0)
> + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI type:
> %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> + else
> + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits
> %d\n", payload.Type, payload.NumBit);
> + }
> + }
> +
> + if (has_logged) {
> + av_log(avctx, AV_LOG_VERBOSE, "End reading SEI\n");
> + }
> +
> + if (out && out->frame)
> + return ff_hevc_set_sei_to_frame(avctx, &sei, out->frame, avctx-
> >framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps->bit_depth_chroma);
I got segfault when trying your patchset,
Thread 1 "ffmpeg" received signal SIGSEGV, Segmentation fault.
0x00007ffff67c0497 in parse_sei_hevc (avctx=avctx at entry=0x5555555e4280, q=q at entry=0x555555625288, out=out at entry=0x5555559b6f80) at libavcodec/qsvdec.c:777
777 return ff_hevc_set_sei_to_frame(avctx, &sei, out->frame, avctx->framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps->bit_depth_chroma);
(gdb) bt
#0 0x00007ffff67c0497 in parse_sei_hevc (avctx=avctx at entry=0x5555555e4280, q=q at entry=0x555555625288, out=out at entry=0x5555559b6f80) at libavcodec/qsvdec.c:777
#1 0x00007ffff67c1afe in qsv_decode (avctx=avctx at entry=0x5555555e4280, q=q at entry=0x555555625288, frame=frame at entry=0x5555556df740, got_frame=got_frame at entry=0x7fffffffd6bc,
avpkt=avpkt at entry=0x555555635398) at libavcodec/qsvdec.c:1020
BTW the SDK provides support for hevc HDR metadata, we needn't parse SEI payload
in qsvdec and may get the corresponding info from the SDK, see
https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html
Thanks
Haihao
> +
> + return 0;
> +}
> +
> +#define A53_MAX_CC_COUNT 2000
> +
> +static int mpeg_decode_a53_cc(AVCodecContext *avctx, QSVContext *s,
> + const uint8_t *p, int buf_size)
> +{
> + if (buf_size >= 6 &&
> + p[0] == 'G' && p[1] == 'A' && p[2] == '9' && p[3] == '4' &&
> + p[4] == 3 && (p[5] & 0x40)) {
> + /* extract A53 Part 4 CC data */
> + unsigned cc_count = p[5] & 0x1f;
> + if (cc_count > 0 && buf_size >= 7 + cc_count * 3) {
> + const uint64_t old_size = s->a53_buf_ref ? s->a53_buf_ref->size :
> 0;
> + const uint64_t new_size = (old_size + cc_count
> + * UINT64_C(3));
> + int ret;
> +
> + if (new_size > 3*A53_MAX_CC_COUNT)
> + return AVERROR(EINVAL);
> +
> + ret = av_buffer_realloc(&s->a53_buf_ref, new_size);
> + if (ret >= 0)
> + memcpy(s->a53_buf_ref->data + old_size, p + 7, cc_count *
> UINT64_C(3));
> +
> + avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS;
> + }
> + return 1;
> + } else if (buf_size >= 2 && p[0] == 0x03 && (p[1]&0x7f) == 0x01) {
> + /* extract SCTE-20 CC data */
> + GetBitContext gb;
> + unsigned cc_count = 0;
> + int ret;
> +
> + init_get_bits8(&gb, p + 2, buf_size - 2);
> + cc_count = get_bits(&gb, 5);
> + if (cc_count > 0) {
> + uint64_t old_size = s->a53_buf_ref ? s->a53_buf_ref->size : 0;
> + uint64_t new_size = (old_size + cc_count * UINT64_C(3));
> + if (new_size > 3 * A53_MAX_CC_COUNT)
> + return AVERROR(EINVAL);
> +
> + ret = av_buffer_realloc(&s->a53_buf_ref, new_size);
> + if (ret >= 0) {
> + uint8_t field, cc1, cc2;
> + uint8_t *cap = s->a53_buf_ref->data;
> +
> + memset(s->a53_buf_ref->data + old_size, 0, cc_count * 3);
> + for (unsigned i = 0; i < cc_count && get_bits_left(&gb) >=
> 26; i++) {
> + skip_bits(&gb, 2); // priority
> + field = get_bits(&gb, 2);
> + skip_bits(&gb, 5); // line_offset
> + cc1 = get_bits(&gb, 8);
> + cc2 = get_bits(&gb, 8);
> + skip_bits(&gb, 1); // marker
> +
> + if (!field) { // forbidden
> + cap[0] = cap[1] = cap[2] = 0x00;
> + } else {
> + field = (field == 2 ? 1 : 0);
> + ////if (!s1->mpeg_enc_ctx.top_field_first) field =
> !field;
> + cap[0] = 0x04 | field;
> + cap[1] = ff_reverse[cc1];
> + cap[2] = ff_reverse[cc2];
> + }
> + cap += 3;
> + }
> + }
> + avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS;
> + }
> + return 1;
> + } else if (buf_size >= 11 && p[0] == 'C' && p[1] == 'C' && p[2] == 0x01
> && p[3] == 0xf8) {
> + int cc_count = 0;
> + int i, ret;
> + // There is a caption count field in the data, but it is often
> + // incorrect. So count the number of captions present.
> + for (i = 5; i + 6 <= buf_size && ((p[i] & 0xfe) == 0xfe); i += 6)
> + cc_count++;
> + // Transform the DVD format into A53 Part 4 format
> + if (cc_count > 0) {
> + int old_size = s->a53_buf_ref ? s->a53_buf_ref->size : 0;
> + uint64_t new_size = (old_size + cc_count
> + * UINT64_C(6));
> + if (new_size > 3*A53_MAX_CC_COUNT)
> + return AVERROR(EINVAL);
> +
> + ret = av_buffer_realloc(&s->a53_buf_ref, new_size);
> + if (ret >= 0) {
> + uint8_t field1 = !!(p[4] & 0x80);
> + uint8_t *cap = s->a53_buf_ref->data;
> + p += 5;
> + for (i = 0; i < cc_count; i++) {
> + cap[0] = (p[0] == 0xff && field1) ? 0xfc : 0xfd;
> + cap[1] = p[1];
> + cap[2] = p[2];
> + cap[3] = (p[3] == 0xff && !field1) ? 0xfc : 0xfd;
> + cap[4] = p[4];
> + cap[5] = p[5];
> + cap += 6;
> + p += 6;
> + }
> + }
> + avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS;
> + }
> + return 1;
> + }
> + return 0;
> +}
> +
> +static int parse_sei_mpeg12(AVCodecContext* avctx, QSVContext* q, AVFrame*
> out)
> +{
> + mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize =
> sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
> + mfxU64 ts;
> + int ret;
> +
> + while (1) {
> + int start;
> +
> + memset(payload.Data, 0, payload.BufSize);
> + ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
> + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on
> GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer),
> payload.BufSize);
> + return 0;
> + }
> + if (ret != MFX_ERR_NONE)
> + return ret;
> +
> + if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
> + break;
> +
> + start = find_start_offset(payload.Data);
> +
> + start++;
> +
> + mpeg_decode_a53_cc(avctx, q, &payload.Data[start],
> (int)((payload.NumBit + 7) / 8) - start);
> +
> + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits %d start %d
> -> %.s\n", payload.Type, payload.NumBit, start, (char
> *)(&payload.Data[start]));
> + }
> +
> + if (!out)
> + return 0;
> +
> + if (q->a53_buf_ref) {
> +
> + AVFrameSideData *sd = av_frame_new_side_data_from_buf(out,
> AV_FRAME_DATA_A53_CC, q->a53_buf_ref);
> + if (!sd)
> + av_buffer_unref(&q->a53_buf_ref);
> + q->a53_buf_ref = NULL;
> + }
> +
> + return 0;
> +}
> +
> static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
> AVFrame *frame, int *got_frame,
> const AVPacket *avpkt)
> @@ -664,6 +966,8 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext
> *q,
> insurf, &outsurf, sync);
> if (ret == MFX_WRN_DEVICE_BUSY)
> av_usleep(500);
> + else if (avctx->codec_id == AV_CODEC_ID_MPEG2VIDEO)
> + parse_sei_mpeg12(avctx, q, NULL);
>
> } while (ret == MFX_WRN_DEVICE_BUSY || ret == MFX_ERR_MORE_SURFACE);
>
> @@ -705,6 +1009,23 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext
> *q,
> return AVERROR_BUG;
> }
>
> + switch (avctx->codec_id) {
> + case AV_CODEC_ID_MPEG2VIDEO:
> + ret = parse_sei_mpeg12(avctx, q, out_frame->frame);
> + break;
> + case AV_CODEC_ID_H264:
> + ret = parse_sei_h264(avctx, q, out_frame->frame);
> + break;
> + case AV_CODEC_ID_HEVC:
> + ret = parse_sei_hevc(avctx, q, out_frame);
> + break;
> + default:
> + ret = 0;
> + }
> +
> + if (ret < 0)
> + av_log(avctx, AV_LOG_ERROR, "Error parsing SEI data: %d\n", ret);
> +
> out_frame->queued += 1;
>
> aframe = (QSVAsyncFrame){ sync, out_frame };
More information about the ffmpeg-devel
mailing list