[FFmpeg-devel] [PATCH v6 3/3] avcodec/qsvdec: Implement SEI parsing for QSV decoders
softworkz
ffmpegagent at gmail.com
Tue Oct 25 07:03:36 EEST 2022
From: softworkz <softworkz at hotmail.com>
Signed-off-by: softworkz <softworkz at hotmail.com>
---
libavcodec/Makefile | 2 +-
libavcodec/qsvdec.c | 321 ++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 322 insertions(+), 1 deletion(-)
diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index 90c7f113a3..cbddbb0ace 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -146,7 +146,7 @@ OBJS-$(CONFIG_MSS34DSP) += mss34dsp.o
OBJS-$(CONFIG_PIXBLOCKDSP) += pixblockdsp.o
OBJS-$(CONFIG_QPELDSP) += qpeldsp.o
OBJS-$(CONFIG_QSV) += qsv.o
-OBJS-$(CONFIG_QSVDEC) += qsvdec.o
+OBJS-$(CONFIG_QSVDEC) += qsvdec.o h264_sei.o hevc_sei.o
OBJS-$(CONFIG_QSVENC) += qsvenc.o
OBJS-$(CONFIG_RANGECODER) += rangecoder.o
OBJS-$(CONFIG_RDFT) += rdft.o
diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
index 73405b5747..467a248224 100644
--- a/libavcodec/qsvdec.c
+++ b/libavcodec/qsvdec.c
@@ -41,6 +41,7 @@
#include "libavutil/time.h"
#include "libavutil/imgutils.h"
#include "libavutil/film_grain_params.h"
+#include <libavutil/reverse.h>
#include "avcodec.h"
#include "codec_internal.h"
@@ -49,6 +50,9 @@
#include "hwconfig.h"
#include "qsv.h"
#include "qsv_internal.h"
+#include "h264_sei.h"
+#include "hevc_ps.h"
+#include "hevc_sei.h"
#if QSV_ONEVPL
#include <mfxdispatcher.h>
@@ -66,6 +70,8 @@ static const AVRational mfx_tb = { 1, 90000 };
AV_NOPTS_VALUE : pts_tb.num ? \
av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts)
+#define PAYLOAD_BUFFER_SIZE 65535
+
typedef struct QSVAsyncFrame {
mfxSyncPoint *sync;
QSVFrame *frame;
@@ -107,6 +113,9 @@ typedef struct QSVContext {
mfxExtBuffer **ext_buffers;
int nb_ext_buffers;
+
+ mfxU8 payload_buffer[PAYLOAD_BUFFER_SIZE];
+ AVBufferRef *a53_buf_ref;
} QSVContext;
static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
@@ -628,6 +637,299 @@ static int qsv_export_film_grain(AVCodecContext *avctx, mfxExtAV1FilmGrainParam
}
#endif
+static int find_start_offset(mfxU8 data[4])
+{
+ if (data[0] == 0 && data[1] == 0 && data[2] == 1)
+ return 3;
+
+ if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1)
+ return 4;
+
+ return 0;
+}
+
+static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q, AVFrame* out)
+{
+ H264SEIContext sei = { 0 };
+ GetBitContext gb = { 0 };
+ mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize = sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
+ mfxU64 ts;
+ int ret;
+
+ while (1) {
+ int start;
+ memset(payload.Data, 0, payload.BufSize);
+
+ ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
+ if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
+ av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer), payload.BufSize);
+ return 0;
+ }
+ if (ret != MFX_ERR_NONE)
+ return ret;
+
+ if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
+ break;
+
+ start = find_start_offset(payload.Data);
+
+ switch (payload.Type) {
+ case SEI_TYPE_BUFFERING_PERIOD:
+ case SEI_TYPE_PIC_TIMING:
+ continue;
+ }
+
+ if (init_get_bits(&gb, &payload.Data[start], payload.NumBit - start * 8) < 0)
+ av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
+ else {
+ ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx);
+
+ if (ret < 0)
+ av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
+ else
+ av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits %d\n", payload.Type, payload.NumBit);
+ }
+ }
+
+ if (out)
+ return ff_h264_set_sei_to_frame(avctx, &sei, out, NULL, 0);
+
+ return 0;
+}
+
+static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q, QSVFrame* out)
+{
+ HEVCSEI sei = { 0 };
+ HEVCParamSets ps = { 0 };
+ GetBitContext gb = { 0 };
+ mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize = sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
+ mfxFrameSurface1 *surface = &out->surface;
+ mfxU64 ts;
+ int ret, has_logged = 0;
+
+ while (1) {
+ int start;
+ memset(payload.Data, 0, payload.BufSize);
+
+ ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
+ if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
+ av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer), payload.BufSize);
+ return 0;
+ }
+ if (ret != MFX_ERR_NONE)
+ return ret;
+
+ if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
+ break;
+
+ if (!has_logged) {
+ has_logged = 1;
+ av_log(avctx, AV_LOG_VERBOSE, "-----------------------------------------\n");
+ av_log(avctx, AV_LOG_VERBOSE, "Start reading SEI - payload timestamp: %llu - surface timestamp: %llu\n", ts, surface->Data.TimeStamp);
+ }
+
+ if (ts != surface->Data.TimeStamp) {
+ av_log(avctx, AV_LOG_WARNING, "GetPayload timestamp (%llu) does not match surface timestamp: (%llu)\n", ts, surface->Data.TimeStamp);
+ }
+
+ start = find_start_offset(payload.Data);
+
+ av_log(avctx, AV_LOG_VERBOSE, "parsing SEI type: %3d Numbits %3d Start: %d\n", payload.Type, payload.NumBit, start);
+
+ switch (payload.Type) {
+ case SEI_TYPE_BUFFERING_PERIOD:
+ case SEI_TYPE_PIC_TIMING:
+ continue;
+ case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME:
+ // There seems to be a bug in MSDK
+ payload.NumBit -= 8;
+
+ break;
+ case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO:
+ // There seems to be a bug in MSDK
+ payload.NumBit = 48;
+
+ break;
+ case SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35:
+ // There seems to be a bug in MSDK
+ if (payload.NumBit == 552)
+ payload.NumBit = 528;
+ break;
+ }
+
+ if (init_get_bits(&gb, &payload.Data[start], payload.NumBit - start * 8) < 0)
+ av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
+ else {
+ ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps, HEVC_NAL_SEI_PREFIX);
+
+ if (ret < 0)
+ av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
+ else
+ av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits %d\n", payload.Type, payload.NumBit);
+ }
+ }
+
+ if (has_logged) {
+ av_log(avctx, AV_LOG_VERBOSE, "End reading SEI\n");
+ }
+
+ if (out && out->frame)
+ return ff_hevc_set_sei_to_frame(avctx, &sei, out->frame, avctx->framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps->bit_depth_chroma);
+
+ return 0;
+}
+
+#define A53_MAX_CC_COUNT 2000
+
+static int mpeg_decode_a53_cc(AVCodecContext *avctx, QSVContext *s,
+ const uint8_t *p, int buf_size)
+{
+ if (buf_size >= 6 &&
+ p[0] == 'G' && p[1] == 'A' && p[2] == '9' && p[3] == '4' &&
+ p[4] == 3 && (p[5] & 0x40)) {
+ /* extract A53 Part 4 CC data */
+ unsigned cc_count = p[5] & 0x1f;
+ if (cc_count > 0 && buf_size >= 7 + cc_count * 3) {
+ const uint64_t old_size = s->a53_buf_ref ? s->a53_buf_ref->size : 0;
+ const uint64_t new_size = (old_size + cc_count
+ * UINT64_C(3));
+ int ret;
+
+ if (new_size > 3*A53_MAX_CC_COUNT)
+ return AVERROR(EINVAL);
+
+ ret = av_buffer_realloc(&s->a53_buf_ref, new_size);
+ if (ret >= 0)
+ memcpy(s->a53_buf_ref->data + old_size, p + 7, cc_count * UINT64_C(3));
+
+ avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS;
+ }
+ return 1;
+ } else if (buf_size >= 2 && p[0] == 0x03 && (p[1]&0x7f) == 0x01) {
+ /* extract SCTE-20 CC data */
+ GetBitContext gb;
+ unsigned cc_count = 0;
+ int ret;
+
+ init_get_bits8(&gb, p + 2, buf_size - 2);
+ cc_count = get_bits(&gb, 5);
+ if (cc_count > 0) {
+ uint64_t old_size = s->a53_buf_ref ? s->a53_buf_ref->size : 0;
+ uint64_t new_size = (old_size + cc_count * UINT64_C(3));
+ if (new_size > 3 * A53_MAX_CC_COUNT)
+ return AVERROR(EINVAL);
+
+ ret = av_buffer_realloc(&s->a53_buf_ref, new_size);
+ if (ret >= 0) {
+ uint8_t field, cc1, cc2;
+ uint8_t *cap = s->a53_buf_ref->data;
+
+ memset(s->a53_buf_ref->data + old_size, 0, cc_count * 3);
+ for (unsigned i = 0; i < cc_count && get_bits_left(&gb) >= 26; i++) {
+ skip_bits(&gb, 2); // priority
+ field = get_bits(&gb, 2);
+ skip_bits(&gb, 5); // line_offset
+ cc1 = get_bits(&gb, 8);
+ cc2 = get_bits(&gb, 8);
+ skip_bits(&gb, 1); // marker
+
+ if (!field) { // forbidden
+ cap[0] = cap[1] = cap[2] = 0x00;
+ } else {
+ field = (field == 2 ? 1 : 0);
+ ////if (!s1->mpeg_enc_ctx.top_field_first) field = !field;
+ cap[0] = 0x04 | field;
+ cap[1] = ff_reverse[cc1];
+ cap[2] = ff_reverse[cc2];
+ }
+ cap += 3;
+ }
+ }
+ avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS;
+ }
+ return 1;
+ } else if (buf_size >= 11 && p[0] == 'C' && p[1] == 'C' && p[2] == 0x01 && p[3] == 0xf8) {
+ int cc_count = 0;
+ int i, ret;
+ // There is a caption count field in the data, but it is often
+ // incorrect. So count the number of captions present.
+ for (i = 5; i + 6 <= buf_size && ((p[i] & 0xfe) == 0xfe); i += 6)
+ cc_count++;
+ // Transform the DVD format into A53 Part 4 format
+ if (cc_count > 0) {
+ int old_size = s->a53_buf_ref ? s->a53_buf_ref->size : 0;
+ uint64_t new_size = (old_size + cc_count
+ * UINT64_C(6));
+ if (new_size > 3*A53_MAX_CC_COUNT)
+ return AVERROR(EINVAL);
+
+ ret = av_buffer_realloc(&s->a53_buf_ref, new_size);
+ if (ret >= 0) {
+ uint8_t field1 = !!(p[4] & 0x80);
+ uint8_t *cap = s->a53_buf_ref->data;
+ p += 5;
+ for (i = 0; i < cc_count; i++) {
+ cap[0] = (p[0] == 0xff && field1) ? 0xfc : 0xfd;
+ cap[1] = p[1];
+ cap[2] = p[2];
+ cap[3] = (p[3] == 0xff && !field1) ? 0xfc : 0xfd;
+ cap[4] = p[4];
+ cap[5] = p[5];
+ cap += 6;
+ p += 6;
+ }
+ }
+ avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS;
+ }
+ return 1;
+ }
+ return 0;
+}
+
+static int parse_sei_mpeg12(AVCodecContext* avctx, QSVContext* q, AVFrame* out)
+{
+ mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize = sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
+ mfxU64 ts;
+ int ret;
+
+ while (1) {
+ int start;
+
+ memset(payload.Data, 0, payload.BufSize);
+ ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
+ if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
+ av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer), payload.BufSize);
+ return 0;
+ }
+ if (ret != MFX_ERR_NONE)
+ return ret;
+
+ if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
+ break;
+
+ start = find_start_offset(payload.Data);
+
+ start++;
+
+ mpeg_decode_a53_cc(avctx, q, &payload.Data[start], (int)((payload.NumBit + 7) / 8) - start);
+
+ av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits %d start %d -> %.s\n", payload.Type, payload.NumBit, start, (char *)(&payload.Data[start]));
+ }
+
+ if (!out)
+ return 0;
+
+ if (q->a53_buf_ref) {
+
+ AVFrameSideData *sd = av_frame_new_side_data_from_buf(out, AV_FRAME_DATA_A53_CC, q->a53_buf_ref);
+ if (!sd)
+ av_buffer_unref(&q->a53_buf_ref);
+ q->a53_buf_ref = NULL;
+ }
+
+ return 0;
+}
+
static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
AVFrame *frame, int *got_frame,
const AVPacket *avpkt)
@@ -664,6 +966,8 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
insurf, &outsurf, sync);
if (ret == MFX_WRN_DEVICE_BUSY)
av_usleep(500);
+ else if (avctx->codec_id == AV_CODEC_ID_MPEG2VIDEO)
+ parse_sei_mpeg12(avctx, q, NULL);
} while (ret == MFX_WRN_DEVICE_BUSY || ret == MFX_ERR_MORE_SURFACE);
@@ -705,6 +1009,23 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
return AVERROR_BUG;
}
+ switch (avctx->codec_id) {
+ case AV_CODEC_ID_MPEG2VIDEO:
+ ret = parse_sei_mpeg12(avctx, q, out_frame->frame);
+ break;
+ case AV_CODEC_ID_H264:
+ ret = parse_sei_h264(avctx, q, out_frame->frame);
+ break;
+ case AV_CODEC_ID_HEVC:
+ ret = parse_sei_hevc(avctx, q, out_frame);
+ break;
+ default:
+ ret = 0;
+ }
+
+ if (ret < 0)
+ av_log(avctx, AV_LOG_ERROR, "Error parsing SEI data: %d\n", ret);
+
out_frame->queued += 1;
aframe = (QSVAsyncFrame){ sync, out_frame };
--
ffmpeg-codebot
More information about the ffmpeg-devel
mailing list