[FFmpeg-devel] [PATCH v6 3/3] avcodec/qsvdec: Implement SEI parsing for QSV decoders
Soft Works
softworkz at hotmail.com
Mon Nov 21 17:58:53 EET 2022
> -----Original Message-----
> From: Xiang, Haihao <haihao.xiang at intel.com>
> Sent: Monday, November 21, 2022 3:45 AM
> To: ffmpeg-devel at ffmpeg.org
> Cc: softworkz at hotmail.com; kierank at obe.tv; haihao.xiang-at-
> intel.com at ffmpeg.org; andreas.rheinhardt at outlook.com
> Subject: Re: [FFmpeg-devel] [PATCH v6 3/3] avcodec/qsvdec: Implement
> SEI parsing for QSV decoders
>
> On Tue, 2022-10-25 at 04:03 +0000, softworkz wrote:
> > From: softworkz <softworkz at hotmail.com>
> >
> > Signed-off-by: softworkz <softworkz at hotmail.com>
> > ---
> > libavcodec/Makefile | 2 +-
> > libavcodec/qsvdec.c | 321
> ++++++++++++++++++++++++++++++++++++++++++++
> > 2 files changed, 322 insertions(+), 1 deletion(-)
> >
> > diff --git a/libavcodec/Makefile b/libavcodec/Makefile
> > index 90c7f113a3..cbddbb0ace 100644
> > --- a/libavcodec/Makefile
> > +++ b/libavcodec/Makefile
> > @@ -146,7 +146,7 @@ OBJS-$(CONFIG_MSS34DSP) +=
> mss34dsp.o
> > OBJS-$(CONFIG_PIXBLOCKDSP) += pixblockdsp.o
> > OBJS-$(CONFIG_QPELDSP) += qpeldsp.o
> > OBJS-$(CONFIG_QSV) += qsv.o
> > -OBJS-$(CONFIG_QSVDEC) += qsvdec.o
> > +OBJS-$(CONFIG_QSVDEC) += qsvdec.o h264_sei.o
> hevc_sei.o
> > OBJS-$(CONFIG_QSVENC) += qsvenc.o
> > OBJS-$(CONFIG_RANGECODER) += rangecoder.o
> > OBJS-$(CONFIG_RDFT) += rdft.o
> > diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
> > index 73405b5747..467a248224 100644
> > --- a/libavcodec/qsvdec.c
> > +++ b/libavcodec/qsvdec.c
> > @@ -41,6 +41,7 @@
> > #include "libavutil/time.h"
> > #include "libavutil/imgutils.h"
> > #include "libavutil/film_grain_params.h"
> > +#include <libavutil/reverse.h>
> >
> > #include "avcodec.h"
> > #include "codec_internal.h"
> > @@ -49,6 +50,9 @@
> > #include "hwconfig.h"
> > #include "qsv.h"
> > #include "qsv_internal.h"
> > +#include "h264_sei.h"
> > +#include "hevc_ps.h"
> > +#include "hevc_sei.h"
> >
> > #if QSV_ONEVPL
> > #include <mfxdispatcher.h>
> > @@ -66,6 +70,8 @@ static const AVRational mfx_tb = { 1, 90000 };
> > AV_NOPTS_VALUE : pts_tb.num ? \
> > av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts)
> >
> > +#define PAYLOAD_BUFFER_SIZE 65535
> > +
> > typedef struct QSVAsyncFrame {
> > mfxSyncPoint *sync;
> > QSVFrame *frame;
> > @@ -107,6 +113,9 @@ typedef struct QSVContext {
> >
> > mfxExtBuffer **ext_buffers;
> > int nb_ext_buffers;
> > +
> > + mfxU8 payload_buffer[PAYLOAD_BUFFER_SIZE];
> > + AVBufferRef *a53_buf_ref;
> > } QSVContext;
> >
> > static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
> > @@ -628,6 +637,299 @@ static int
> qsv_export_film_grain(AVCodecContext *avctx,
> > mfxExtAV1FilmGrainParam
> > }
> > #endif
> >
> > +static int find_start_offset(mfxU8 data[4])
> > +{
> > + if (data[0] == 0 && data[1] == 0 && data[2] == 1)
> > + return 3;
> > +
> > + if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] ==
> 1)
> > + return 4;
> > +
> > + return 0;
> > +}
> > +
> > +static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q,
> AVFrame* out)
> > +{
> > + H264SEIContext sei = { 0 };
> > + GetBitContext gb = { 0 };
> > + mfxPayload payload = { 0, .Data = &q->payload_buffer[0],
> .BufSize =
> > sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
> > + mfxU64 ts;
> > + int ret;
> > +
> > + while (1) {
> > + int start;
> > + memset(payload.Data, 0, payload.BufSize);
> > +
> > + ret = MFXVideoDECODE_GetPayload(q->session, &ts,
> &payload);
> > + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> > + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient
> buffer on
> > GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q-
> >payload_buffer),
> > payload.BufSize);
> > + return 0;
> > + }
> > + if (ret != MFX_ERR_NONE)
> > + return ret;
> > +
> > + if (payload.NumBit == 0 || payload.NumBit >=
> payload.BufSize * 8)
> > + break;
> > +
> > + start = find_start_offset(payload.Data);
> > +
> > + switch (payload.Type) {
> > + case SEI_TYPE_BUFFERING_PERIOD:
> > + case SEI_TYPE_PIC_TIMING:
> > + continue;
> > + }
> > +
> > + if (init_get_bits(&gb, &payload.Data[start],
> payload.NumBit - start *
> > 8) < 0)
> > + av_log(avctx, AV_LOG_ERROR, "Error initializing
> bitstream reader
> > SEI type: %d Numbits %d error: %d\n", payload.Type,
> payload.NumBit, ret);
> > + else {
> > + ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx);
> > +
> > + if (ret < 0)
> > + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI
> type:
> > %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> > + else
> > + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d
> Numbits
> > %d\n", payload.Type, payload.NumBit);
> > + }
> > + }
> > +
> > + if (out)
> > + return ff_h264_set_sei_to_frame(avctx, &sei, out, NULL,
> 0);
> > +
> > + return 0;
> > +}
> > +
> > +static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q,
> QSVFrame*
> > out)
> > +{
> > + HEVCSEI sei = { 0 };
> > + HEVCParamSets ps = { 0 };
> > + GetBitContext gb = { 0 };
> > + mfxPayload payload = { 0, .Data = &q->payload_buffer[0],
> .BufSize =
> > sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
> > + mfxFrameSurface1 *surface = &out->surface;
> > + mfxU64 ts;
> > + int ret, has_logged = 0;
> > +
> > + while (1) {
> > + int start;
> > + memset(payload.Data, 0, payload.BufSize);
> > +
> > + ret = MFXVideoDECODE_GetPayload(q->session, &ts,
> &payload);
> > + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
> > + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient
> buffer on
> > GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q-
> >payload_buffer),
> > payload.BufSize);
> > + return 0;
> > + }
> > + if (ret != MFX_ERR_NONE)
> > + return ret;
> > +
> > + if (payload.NumBit == 0 || payload.NumBit >=
> payload.BufSize * 8)
> > + break;
> > +
> > + if (!has_logged) {
> > + has_logged = 1;
> > + av_log(avctx, AV_LOG_VERBOSE, "-----------------------
> ---------
> > ---------\n");
> > + av_log(avctx, AV_LOG_VERBOSE, "Start reading SEI -
> payload
> > timestamp: %llu - surface timestamp: %llu\n", ts, surface-
> >Data.TimeStamp);
> > + }
> > +
> > + if (ts != surface->Data.TimeStamp) {
> > + av_log(avctx, AV_LOG_WARNING, "GetPayload timestamp
> (%llu) does
> > not match surface timestamp: (%llu)\n", ts, surface-
> >Data.TimeStamp);
> > + }
> > +
> > + start = find_start_offset(payload.Data);
> > +
> > + av_log(avctx, AV_LOG_VERBOSE, "parsing SEI type: %3d
> Numbits
> > %3d Start: %d\n", payload.Type, payload.NumBit, start);
> > +
> > + switch (payload.Type) {
> > + case SEI_TYPE_BUFFERING_PERIOD:
> > + case SEI_TYPE_PIC_TIMING:
> > + continue;
> > + case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME:
> > + // There seems to be a bug in MSDK
> > + payload.NumBit -= 8;
> > +
> > + break;
> > + case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO:
> > + // There seems to be a bug in MSDK
> > + payload.NumBit = 48;
> > +
> > + break;
> > + case SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35:
> > + // There seems to be a bug in MSDK
> > + if (payload.NumBit == 552)
> > + payload.NumBit = 528;
> > + break;
> > + }
> > +
> > + if (init_get_bits(&gb, &payload.Data[start],
> payload.NumBit - start *
> > 8) < 0)
> > + av_log(avctx, AV_LOG_ERROR, "Error initializing
> bitstream reader
> > SEI type: %d Numbits %d error: %d\n", payload.Type,
> payload.NumBit, ret);
> > + else {
> > + ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps,
> > HEVC_NAL_SEI_PREFIX);
> > +
> > + if (ret < 0)
> > + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI
> type:
> > %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> > + else
> > + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d
> Numbits
> > %d\n", payload.Type, payload.NumBit);
> > + }
> > + }
> > +
> > + if (has_logged) {
> > + av_log(avctx, AV_LOG_VERBOSE, "End reading SEI\n");
> > + }
> > +
> > + if (out && out->frame)
> > + return ff_hevc_set_sei_to_frame(avctx, &sei, out->frame,
> avctx-
> > >framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps-
> >bit_depth_chroma);
>
> I got segfault when trying your patchset,
>
> Thread 1 "ffmpeg" received signal SIGSEGV, Segmentation fault.
> 0x00007ffff67c0497 in parse_sei_hevc
> (avctx=avctx at entry=0x5555555e4280, q=q at entry=0x555555625288,
> out=out at entry=0x5555559b6f80) at libavcodec/qsvdec.c:777
> 777 return ff_hevc_set_sei_to_frame(avctx, &sei, out-
> >frame, avctx->framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps-
> >bit_depth_chroma);
> (gdb) bt
> #0 0x00007ffff67c0497 in parse_sei_hevc
> (avctx=avctx at entry=0x5555555e4280, q=q at entry=0x555555625288,
> out=out at entry=0x5555559b6f80) at libavcodec/qsvdec.c:777
> #1 0x00007ffff67c1afe in qsv_decode
> (avctx=avctx at entry=0x5555555e4280, q=q at entry=0x555555625288,
> frame=frame at entry=0x5555556df740,
> got_frame=got_frame at entry=0x7fffffffd6bc,
> avpkt=avpkt at entry=0x555555635398) at libavcodec/qsvdec.c:1020
> BTW the SDK provides support for hevc HDR metadata, we needn't parse
> SEI payload
> in qsvdec and may get the corresponding info from the SDK, see
> https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html
I know. I was the one who had requested this to be added to MSDK :-)
But it's just one small part of SEI information, it's limited to
the latest MSDK versions and I'm not sure whether it's working
as reliably as this implementation. This would need to be tested.
You should still have access to the repo with the test files for
demoing the offset problems (which my patchset is working around).
But it's also about dynamic HDR data, dovi data, etc. - which
MSDK doesn't provide, so this single bit of SEI data doesn't
help much.
Best regards,
softworkz
More information about the ffmpeg-devel
mailing list