[FFmpeg-devel] [PATCH 2/2] avcodec/mediacodecenc: add async mode support
Zhao Zhili
quinkblack at foxmail.com
Thu Jan 4 18:52:09 EET 2024
From: Zhao Zhili <zhilizhao at tencent.com>
---
configure | 2 +-
libavcodec/mediacodecenc.c | 289 +++++++++++++++++++++++++++++++++----
2 files changed, 258 insertions(+), 33 deletions(-)
diff --git a/configure b/configure
index d15cfa4703..4f0bdd8c7b 100755
--- a/configure
+++ b/configure
@@ -3062,7 +3062,7 @@ d3d11va_deps="dxva_h ID3D11VideoDecoder ID3D11VideoContext"
d3d12va_deps="dxva_h ID3D12Device ID3D12VideoDecoder"
dxva2_deps="dxva2api_h DXVA2_ConfigPictureDecode ole32 user32"
ffnvcodec_deps_any="libdl LoadLibrary"
-mediacodec_deps="android"
+mediacodec_deps="android pthreads"
nvdec_deps="ffnvcodec"
vaapi_x11_deps="xlib_x11"
videotoolbox_hwaccel_deps="videotoolbox pthreads"
diff --git a/libavcodec/mediacodecenc.c b/libavcodec/mediacodecenc.c
index cab0189a9f..4729860d9c 100644
--- a/libavcodec/mediacodecenc.c
+++ b/libavcodec/mediacodecenc.c
@@ -20,9 +20,11 @@
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
+#include <pthread.h>
#include "config_components.h"
#include "libavutil/avassert.h"
+#include "libavutil/fifo.h"
#include "libavutil/hwcontext_mediacodec.h"
#include "libavutil/imgutils.h"
#include "libavutil/opt.h"
@@ -56,6 +58,7 @@ typedef struct MediaCodecEncContext {
AVClass *avclass;
FFAMediaCodec *codec;
int use_ndk_codec;
+ int async_mode;
const char *name;
FFANativeWindow *window;
@@ -73,6 +76,16 @@ typedef struct MediaCodecEncContext {
int bitrate_mode;
int level;
int pts_as_dts;
+
+ pthread_mutex_t input_mutex;
+ pthread_cond_t input_cond;
+ AVFifo *input_index;
+
+ pthread_mutex_t output_mutex;
+ pthread_cond_t output_cond;
+ int encode_status;
+ AVFifo *output_index;
+ AVFifo *output_buf_info;
} MediaCodecEncContext;
enum {
@@ -97,17 +110,25 @@ static const enum AVPixelFormat avc_pix_fmts[] = {
AV_PIX_FMT_NONE
};
-static void mediacodec_output_format(AVCodecContext *avctx)
+static void mediacodec_dump_format(AVCodecContext *avctx, FFAMediaFormat *out_format)
{
MediaCodecEncContext *s = avctx->priv_data;
- char *name = ff_AMediaCodec_getName(s->codec);
- FFAMediaFormat *out_format = ff_AMediaCodec_getOutputFormat(s->codec);
+ const char *name = s->name;
char *str = ff_AMediaFormat_toString(out_format);
av_log(avctx, AV_LOG_DEBUG, "MediaCodec encoder %s output format %s\n",
name ? name : "unknown", str);
- av_free(name);
av_free(str);
+}
+
+static void mediacodec_output_format(AVCodecContext *avctx)
+{
+ MediaCodecEncContext *s = avctx->priv_data;
+ FFAMediaFormat *out_format = ff_AMediaCodec_getOutputFormat(s->codec);
+
+ if (!s->name)
+ s->name = ff_AMediaCodec_getName(s->codec);
+ mediacodec_dump_format(avctx, out_format);
ff_AMediaFormat_delete(out_format);
}
@@ -147,6 +168,131 @@ static int mediacodec_init_bsf(AVCodecContext *avctx)
return ret;
}
+static void copy_frame_to_buffer(AVCodecContext *avctx, const AVFrame *frame, uint8_t *dst, size_t size)
+{
+ MediaCodecEncContext *s = avctx->priv_data;
+ uint8_t *dst_data[4] = {};
+ int dst_linesize[4] = {};
+
+ if (avctx->pix_fmt == AV_PIX_FMT_YUV420P) {
+ dst_data[0] = dst;
+ dst_data[1] = dst + s->width * s->height;
+ dst_data[2] = dst_data[1] + s->width * s->height / 4;
+
+ dst_linesize[0] = s->width;
+ dst_linesize[1] = dst_linesize[2] = s->width / 2;
+ } else if (avctx->pix_fmt == AV_PIX_FMT_NV12) {
+ dst_data[0] = dst;
+ dst_data[1] = dst + s->width * s->height;
+
+ dst_linesize[0] = s->width;
+ dst_linesize[1] = s->width;
+ } else {
+ av_assert0(0);
+ }
+
+ av_image_copy2(dst_data, dst_linesize, frame->data, frame->linesize,
+ avctx->pix_fmt, avctx->width, avctx->height);
+}
+
+static void on_input_available(FFAMediaCodec *codec, void *userdata, int32_t index)
+{
+ AVCodecContext *avctx = userdata;
+ MediaCodecEncContext *s = avctx->priv_data;
+
+ pthread_mutex_lock(&s->input_mutex);
+
+ av_fifo_write(s->input_index, &index, 1);
+
+ pthread_mutex_unlock(&s->input_mutex);
+ pthread_cond_signal(&s->input_cond);
+}
+
+static void on_output_available(FFAMediaCodec *codec, void *userdata,
+ int32_t index,
+ FFAMediaCodecBufferInfo *out_info)
+{
+ AVCodecContext *avctx = userdata;
+ MediaCodecEncContext *s = avctx->priv_data;
+
+ pthread_mutex_lock(&s->output_mutex);
+
+ av_fifo_write(s->output_index, &index, 1);
+ av_fifo_write(s->output_buf_info, out_info, 1);
+
+ pthread_mutex_unlock(&s->output_mutex);
+ pthread_cond_signal(&s->output_cond);
+}
+
+static void on_format_changed(FFAMediaCodec *codec, void *userdata, FFAMediaFormat *format)
+{
+ mediacodec_dump_format(userdata, format);
+}
+
+static void on_error(FFAMediaCodec *codec, void *userdata, int error, const char *detail)
+{
+ AVCodecContext *avctx = userdata;
+ MediaCodecEncContext *s = avctx->priv_data;
+
+ if (error == AVERROR(EAGAIN))
+ return;
+
+ av_log(avctx, AV_LOG_ERROR, "On error, %s, %s\n", av_err2str(error), detail);
+
+ pthread_mutex_lock(&s->input_mutex);
+ pthread_mutex_lock(&s->output_mutex);
+ s->encode_status = error;
+ pthread_mutex_unlock(&s->output_mutex);
+ pthread_mutex_unlock(&s->input_mutex);
+
+ pthread_cond_signal(&s->output_cond);
+ pthread_cond_signal(&s->input_cond);
+}
+
+static int mediacodec_init_async_state(AVCodecContext *avctx)
+{
+ MediaCodecEncContext *s = avctx->priv_data;
+ size_t fifo_size = 16;
+
+ if (!s->async_mode)
+ return 0;
+
+ pthread_mutex_init(&s->input_mutex, NULL);
+ pthread_cond_init(&s->input_cond, NULL);
+
+ pthread_mutex_init(&s->output_mutex, NULL);
+ pthread_cond_init(&s->output_cond, NULL);
+
+ s->input_index = av_fifo_alloc2(fifo_size, sizeof(int32_t), AV_FIFO_FLAG_AUTO_GROW);
+ s->output_index = av_fifo_alloc2(fifo_size, sizeof(int32_t), AV_FIFO_FLAG_AUTO_GROW);
+ s->output_buf_info = av_fifo_alloc2(fifo_size, sizeof(FFAMediaCodecBufferInfo), AV_FIFO_FLAG_AUTO_GROW);
+
+ if (!s->input_index || !s->output_index || !s->output_buf_info)
+ return AVERROR(ENOMEM);
+
+ return 0;
+}
+
+static void mediacodec_uninit_async_state(AVCodecContext *avctx)
+{
+ MediaCodecEncContext *s = avctx->priv_data;
+
+ if (!s->async_mode)
+ return;
+
+ pthread_mutex_destroy(&s->input_mutex);
+ pthread_cond_destroy(&s->input_cond);
+
+ pthread_mutex_destroy(&s->output_mutex);
+ pthread_cond_destroy(&s->output_cond);
+
+ av_fifo_freep2(&s->input_index);
+ av_fifo_freep2(&s->output_index);
+ av_fifo_freep2(&s->output_buf_info);
+
+ s->async_mode = 0;
+}
+
static av_cold int mediacodec_init(AVCodecContext *avctx)
{
const char *codec_mime = NULL;
@@ -155,6 +301,11 @@ static av_cold int mediacodec_init(AVCodecContext *avctx)
int ret;
int gop;
+ // Init async state first, so we can do cleanup safely on error path.
+ ret = mediacodec_init_async_state(avctx);
+ if (ret < 0)
+ return ret;
+
if (s->use_ndk_codec < 0)
s->use_ndk_codec = !av_jni_get_java_vm(avctx);
@@ -322,10 +473,21 @@ static av_cold int mediacodec_init(AVCodecContext *avctx)
goto bailout;
}
- ret = ff_AMediaCodec_start(s->codec);
- if (ret) {
- av_log(avctx, AV_LOG_ERROR, "MediaCodec failed to start, %s\n", av_err2str(ret));
- goto bailout;
+ if (s->async_mode) {
+ FFAMediaCodecOnAsyncNotifyCallback cb = {
+ .onAsyncInputAvailable = on_input_available,
+ .onAsyncOutputAvailable = on_output_available,
+ .onAsyncFormatChanged = on_format_changed,
+ .onAsyncError = on_error,
+ };
+
+ ret = ff_AMediaCodec_setAsyncNotifyCallback(s->codec, &cb, avctx);
+ if (ret < 0) {
+ av_log(avctx, AV_LOG_WARNING,
+ "Try MediaCodec async mode failed, %s, switch to sync mode\n",
+ av_err2str(ret));
+ mediacodec_uninit_async_state(avctx);
+ }
}
ret = mediacodec_init_bsf(avctx);
@@ -339,8 +501,16 @@ static av_cold int mediacodec_init(AVCodecContext *avctx)
"Use extract_extradata bsf when necessary.\n");
s->frame = av_frame_alloc();
- if (!s->frame)
+ if (!s->frame) {
ret = AVERROR(ENOMEM);
+ goto bailout;
+ }
+
+ ret = ff_AMediaCodec_start(s->codec);
+ if (ret) {
+ av_log(avctx, AV_LOG_ERROR, "MediaCodec failed to start, %s\n", av_err2str(ret));
+ goto bailout;
+ }
bailout:
if (format)
@@ -348,19 +518,63 @@ bailout:
return ret;
}
+static int mediacodec_get_output_index(AVCodecContext *avctx, ssize_t *index,
+ FFAMediaCodecBufferInfo *out_info)
+{
+ MediaCodecEncContext *s = avctx->priv_data;
+ FFAMediaCodec *codec = s->codec;
+ int64_t timeout_us = s->eof_sent ? OUTPUT_DEQUEUE_TIMEOUT_US : 0;
+ int n;
+ int ret;
+
+ if (!s->async_mode) {
+ *index = ff_AMediaCodec_dequeueOutputBuffer(codec, out_info, timeout_us);
+ return 0;
+ }
+
+ pthread_mutex_lock(&s->output_mutex);
+
+ n = -1;
+ while (n < 0 && !s->encode_status) {
+ if (av_fifo_can_read(s->output_index)) {
+ av_fifo_read(s->output_index, &n, 1);
+ av_fifo_read(s->output_buf_info, out_info, 1);
+ break;
+ }
+
+ if (n < 0 && s->eof_sent && !s->encode_status)
+ pthread_cond_wait(&s->output_cond, &s->output_mutex);
+ else
+ break;
+ }
+
+ ret = s->encode_status;
+ *index = n;
+ pthread_mutex_unlock(&s->output_mutex);
+
+ // Get output index success
+ if (*index >= 0)
+ return 0;
+
+ return ret ? ret : AVERROR(EAGAIN);
+}
+
static int mediacodec_receive(AVCodecContext *avctx,
AVPacket *pkt,
int *got_packet)
{
MediaCodecEncContext *s = avctx->priv_data;
FFAMediaCodec *codec = s->codec;
+ ssize_t index;
FFAMediaCodecBufferInfo out_info = {0};
uint8_t *out_buf;
size_t out_size = 0;
int ret;
int extradata_size = 0;
- int64_t timeout_us = s->eof_sent ? OUTPUT_DEQUEUE_TIMEOUT_US : 0;
- ssize_t index = ff_AMediaCodec_dequeueOutputBuffer(codec, &out_info, timeout_us);
+
+ ret = mediacodec_get_output_index(avctx, &index, &out_info);
+ if (ret < 0)
+ return ret;
if (ff_AMediaCodec_infoTryAgainLater(codec, index))
return AVERROR(EAGAIN);
@@ -426,31 +640,36 @@ bailout:
return ret;
}
-static void copy_frame_to_buffer(AVCodecContext *avctx, const AVFrame *frame, uint8_t *dst, size_t size)
+static int mediacodec_get_input_index(AVCodecContext *avctx, ssize_t *index)
{
MediaCodecEncContext *s = avctx->priv_data;
- uint8_t *dst_data[4] = {};
- int dst_linesize[4] = {};
+ FFAMediaCodec *codec = s->codec;
+ int ret = 0;
+ int32_t n;
- if (avctx->pix_fmt == AV_PIX_FMT_YUV420P) {
- dst_data[0] = dst;
- dst_data[1] = dst + s->width * s->height;
- dst_data[2] = dst_data[1] + s->width * s->height / 4;
+ if (!s->async_mode) {
+ *index = ff_AMediaCodec_dequeueInputBuffer(codec, INPUT_DEQUEUE_TIMEOUT_US);
+ return 0;
+ }
- dst_linesize[0] = s->width;
- dst_linesize[1] = dst_linesize[2] = s->width / 2;
- } else if (avctx->pix_fmt == AV_PIX_FMT_NV12) {
- dst_data[0] = dst;
- dst_data[1] = dst + s->width * s->height;
+ pthread_mutex_lock(&s->input_mutex);
- dst_linesize[0] = s->width;
- dst_linesize[1] = s->width;
- } else {
- av_assert0(0);
+ n = -1;
+ while (n < 0 && !s->encode_status) {
+ if (av_fifo_can_read(s->input_index) > 0) {
+ av_fifo_read(s->input_index, &n, 1);
+ break;
+ }
+
+ if (n < 0 && !s->encode_status)
+ pthread_cond_wait(&s->input_cond, &s->input_mutex);
}
- av_image_copy2(dst_data, dst_linesize, frame->data, frame->linesize,
- avctx->pix_fmt, avctx->width, avctx->height);
+ ret = s->encode_status;
+ *index = n;
+ pthread_mutex_unlock(&s->input_mutex);
+
+ return ret;
}
static int mediacodec_send(AVCodecContext *avctx,
@@ -462,7 +681,7 @@ static int mediacodec_send(AVCodecContext *avctx,
size_t input_size = 0;
int64_t pts = 0;
uint32_t flags = 0;
- int64_t timeout_us;
+ int ret;
if (s->eof_sent)
return 0;
@@ -478,8 +697,10 @@ static int mediacodec_send(AVCodecContext *avctx,
return 0;
}
- timeout_us = INPUT_DEQUEUE_TIMEOUT_US;
- index = ff_AMediaCodec_dequeueInputBuffer(codec, timeout_us);
+ ret = mediacodec_get_input_index(avctx, &index);
+ if (ret < 0)
+ return ret;
+
if (ff_AMediaCodec_infoTryAgainLater(codec, index))
return AVERROR(EAGAIN);
@@ -566,6 +787,8 @@ static av_cold int mediacodec_close(AVCodecContext *avctx)
av_bsf_free(&s->bsf);
av_frame_free(&s->frame);
+ mediacodec_uninit_async_state(avctx);
+
return 0;
}
@@ -587,6 +810,8 @@ static const AVCodecHWConfigInternal *const mediacodec_hw_configs[] = {
#define COMMON_OPTION \
{ "ndk_codec", "Use MediaCodec from NDK", \
OFFSET(use_ndk_codec), AV_OPT_TYPE_BOOL, {.i64 = -1}, -1, 1, VE }, \
+ { "ndk_async", "Try NDK MediaCodec in async mode", \
+ OFFSET(async_mode), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, VE }, \
{ "codec_name", "Select codec by name", \
OFFSET(name), AV_OPT_TYPE_STRING, {0}, 0, 0, VE }, \
{ "bitrate_mode", "Bitrate control method", \
--
2.25.1
More information about the ffmpeg-devel
mailing list