[FFmpeg-devel] [PATCH]Avoid an assertion failure in ff_init_vlc_sparse
Carl Eugen Hoyos
cehoyos at ag.or.at
Fri Jul 12 21:13:43 CEST 2013
Hi!
Attached patch fixes an assertion failure on oom while initializing tables,
an example is in ticket #2724.
Please comment, Carl Eugen
-------------- next part --------------
diff --git a/libavcodec/bitstream.c b/libavcodec/bitstream.c
index bbd9491..70801a5 100644
--- a/libavcodec/bitstream.c
+++ b/libavcodec/bitstream.c
@@ -282,7 +282,8 @@ int ff_init_vlc_sparse(VLC *vlc, int nb_bits, int nb_codes,
codes, codes_wrap, codes_size,
symbols, symbols_wrap, symbols_size,
flags & ~INIT_VLC_USE_NEW_STATIC);
- av_assert0(ret >= 0);
+ if (ret < 0)
+ return ret;
av_assert0(dyn_vlc.table_size <= vlc->table_allocated);
if (dyn_vlc.table_size < vlc->table_allocated)
av_log(NULL, AV_LOG_ERROR, "needed %d had %d\n", dyn_vlc.table_size, vlc->table_allocated);
diff --git a/libavcodec/faxcompr.c b/libavcodec/faxcompr.c
index 900851b..9fc6651 100644
--- a/libavcodec/faxcompr.c
+++ b/libavcodec/faxcompr.c
@@ -96,30 +96,33 @@ static const uint8_t ccitt_group3_2d_lens[11] = {
static VLC ccitt_vlc[2], ccitt_group3_2d_vlc;
-av_cold void ff_ccitt_unpack_init(void)
+av_cold int ff_ccitt_unpack_init(void)
{
static VLC_TYPE code_table1[528][2];
static VLC_TYPE code_table2[648][2];
- int i;
+ int i, ret;
static int initialized = 0;
if (initialized)
- return;
+ return 0;
ccitt_vlc[0].table = code_table1;
ccitt_vlc[0].table_allocated = 528;
ccitt_vlc[1].table = code_table2;
ccitt_vlc[1].table_allocated = 648;
for (i = 0; i < 2; i++) {
- ff_init_vlc_sparse(&ccitt_vlc[i], 9, CCITT_SYMS,
+ ret = ff_init_vlc_sparse(&ccitt_vlc[i], 9, CCITT_SYMS,
ccitt_codes_lens[i], 1, 1,
ccitt_codes_bits[i], 1, 1,
ccitt_syms, 2, 2,
INIT_VLC_USE_NEW_STATIC);
+ if (ret < 0)
+ return ret;
}
INIT_VLC_STATIC(&ccitt_group3_2d_vlc, 9, 11,
ccitt_group3_2d_lens, 1, 1,
ccitt_group3_2d_bits, 1, 1, 512);
initialized = 1;
+ return 0;
}
diff --git a/libavcodec/faxcompr.h b/libavcodec/faxcompr.h
index 53d1168..18f96dd 100644
--- a/libavcodec/faxcompr.h
+++ b/libavcodec/faxcompr.h
@@ -33,7 +33,7 @@
/**
* initialize upacker code
*/
-void ff_ccitt_unpack_init(void);
+int ff_ccitt_unpack_init(void);
/**
* unpack data compressed with CCITT Group 3 1/2-D or Group 4 method
diff --git a/libavcodec/mpc8.c b/libavcodec/mpc8.c
index 4d6ca89..239de61 100644
--- a/libavcodec/mpc8.c
+++ b/libavcodec/mpc8.c
@@ -96,7 +96,7 @@ static const uint16_t vlc_offsets[13] = {
static av_cold int mpc8_decode_init(AVCodecContext * avctx)
{
- int i;
+ int i, ret;
MPCContext *c = avctx->priv_data;
GetBitContext gb;
static int vlc_initialized = 0;
@@ -188,16 +188,20 @@ static av_cold int mpc8_decode_init(AVCodecContext * avctx)
q3_vlc[0].table = q3_0_table;
q3_vlc[0].table_allocated = 512;
- ff_init_vlc_sparse(&q3_vlc[0], MPC8_Q3_BITS, MPC8_Q3_SIZE,
+ ret = ff_init_vlc_sparse(&q3_vlc[0], MPC8_Q3_BITS, MPC8_Q3_SIZE,
mpc8_q3_bits, 1, 1,
mpc8_q3_codes, 1, 1,
mpc8_q3_syms, 1, 1, INIT_VLC_USE_NEW_STATIC);
+ if (ret < 0)
+ return ret;
q3_vlc[1].table = q3_1_table;
q3_vlc[1].table_allocated = 516;
- ff_init_vlc_sparse(&q3_vlc[1], MPC8_Q4_BITS, MPC8_Q4_SIZE,
+ ret = ff_init_vlc_sparse(&q3_vlc[1], MPC8_Q4_BITS, MPC8_Q4_SIZE,
mpc8_q4_bits, 1, 1,
mpc8_q4_codes, 1, 1,
mpc8_q4_syms, 1, 1, INIT_VLC_USE_NEW_STATIC);
+ if (ret < 0)
+ return ret;
for(i = 0; i < 2; i++){
res_vlc[i].table = &codes_table[vlc_offsets[0+i]];
diff --git a/libavcodec/rv34.c b/libavcodec/rv34.c
index 9b47d2e..f3fae7a 100644
--- a/libavcodec/rv34.c
+++ b/libavcodec/rv34.c
@@ -103,10 +103,10 @@ static VLC_TYPE table_data[117592][2];
* @param insyms symbols for input codes (NULL for default ones)
* @param num VLC table number (for static initialization)
*/
-static void rv34_gen_vlc(const uint8_t *bits, int size, VLC *vlc, const uint8_t *insyms,
+static int rv34_gen_vlc(const uint8_t *bits, int size, VLC *vlc, const uint8_t *insyms,
const int num)
{
- int i;
+ int i, ret;
int counts[17] = {0}, codes[17];
uint16_t cw[MAX_VLC_SIZE], syms[MAX_VLC_SIZE];
uint8_t bits2[MAX_VLC_SIZE];
@@ -130,46 +130,74 @@ static void rv34_gen_vlc(const uint8_t *bits, int size, VLC *vlc, const uint8_t
vlc->table = &table_data[table_offs[num]];
vlc->table_allocated = table_offs[num + 1] - table_offs[num];
- ff_init_vlc_sparse(vlc, FFMIN(maxbits, 9), realsize,
+ ret = ff_init_vlc_sparse(vlc, FFMIN(maxbits, 9), realsize,
bits2, 1, 1,
cw, 2, 2,
syms, 2, 2, INIT_VLC_USE_NEW_STATIC);
+ if (ret < 0)
+ return ret;
+ return 0;
}
/**
* Initialize all tables.
*/
-static av_cold void rv34_init_tables(void)
+static av_cold int rv34_init_tables(void)
{
- int i, j, k;
+ int i, j, k, ret;
for(i = 0; i < NUM_INTRA_TABLES; i++){
for(j = 0; j < 2; j++){
- rv34_gen_vlc(rv34_table_intra_cbppat [i][j], CBPPAT_VLC_SIZE, &intra_vlcs[i].cbppattern[j], NULL, 19*i + 0 + j);
- rv34_gen_vlc(rv34_table_intra_secondpat[i][j], OTHERBLK_VLC_SIZE, &intra_vlcs[i].second_pattern[j], NULL, 19*i + 2 + j);
- rv34_gen_vlc(rv34_table_intra_thirdpat [i][j], OTHERBLK_VLC_SIZE, &intra_vlcs[i].third_pattern[j], NULL, 19*i + 4 + j);
+ ret = rv34_gen_vlc(rv34_table_intra_cbppat [i][j], CBPPAT_VLC_SIZE, &intra_vlcs[i].cbppattern[j], NULL, 19*i + 0 + j);
+ if (ret < 0)
+ return ret;
+ ret = rv34_gen_vlc(rv34_table_intra_secondpat[i][j], OTHERBLK_VLC_SIZE, &intra_vlcs[i].second_pattern[j], NULL, 19*i + 2 + j);
+ if (ret < 0)
+ return ret;
+ ret = rv34_gen_vlc(rv34_table_intra_thirdpat [i][j], OTHERBLK_VLC_SIZE, &intra_vlcs[i].third_pattern[j], NULL, 19*i + 4 + j);
+ if (ret < 0)
+ return ret;
for(k = 0; k < 4; k++){
- rv34_gen_vlc(rv34_table_intra_cbp[i][j+k*2], CBP_VLC_SIZE, &intra_vlcs[i].cbp[j][k], rv34_cbp_code, 19*i + 6 + j*4 + k);
+ ret = rv34_gen_vlc(rv34_table_intra_cbp[i][j+k*2], CBP_VLC_SIZE, &intra_vlcs[i].cbp[j][k], rv34_cbp_code, 19*i + 6 + j*4 + k);
+ if (ret < 0)
+ return ret;
}
}
for(j = 0; j < 4; j++){
- rv34_gen_vlc(rv34_table_intra_firstpat[i][j], FIRSTBLK_VLC_SIZE, &intra_vlcs[i].first_pattern[j], NULL, 19*i + 14 + j);
+ ret = rv34_gen_vlc(rv34_table_intra_firstpat[i][j], FIRSTBLK_VLC_SIZE, &intra_vlcs[i].first_pattern[j], NULL, 19*i + 14 + j);
+ if (ret < 0)
+ return ret;
}
- rv34_gen_vlc(rv34_intra_coeff[i], COEFF_VLC_SIZE, &intra_vlcs[i].coefficient, NULL, 19*i + 18);
+ ret = rv34_gen_vlc(rv34_intra_coeff[i], COEFF_VLC_SIZE, &intra_vlcs[i].coefficient, NULL, 19*i + 18);
+ if (ret < 0)
+ return ret;
}
for(i = 0; i < NUM_INTER_TABLES; i++){
- rv34_gen_vlc(rv34_inter_cbppat[i], CBPPAT_VLC_SIZE, &inter_vlcs[i].cbppattern[0], NULL, i*12 + 95);
+ ret = rv34_gen_vlc(rv34_inter_cbppat[i], CBPPAT_VLC_SIZE, &inter_vlcs[i].cbppattern[0], NULL, i*12 + 95);
+ if (ret < 0)
+ return ret;
for(j = 0; j < 4; j++){
- rv34_gen_vlc(rv34_inter_cbp[i][j], CBP_VLC_SIZE, &inter_vlcs[i].cbp[0][j], rv34_cbp_code, i*12 + 96 + j);
+ ret = rv34_gen_vlc(rv34_inter_cbp[i][j], CBP_VLC_SIZE, &inter_vlcs[i].cbp[0][j], rv34_cbp_code, i*12 + 96 + j);
+ if (ret < 0)
+ return ret;
}
for(j = 0; j < 2; j++){
- rv34_gen_vlc(rv34_table_inter_firstpat [i][j], FIRSTBLK_VLC_SIZE, &inter_vlcs[i].first_pattern[j], NULL, i*12 + 100 + j);
- rv34_gen_vlc(rv34_table_inter_secondpat[i][j], OTHERBLK_VLC_SIZE, &inter_vlcs[i].second_pattern[j], NULL, i*12 + 102 + j);
- rv34_gen_vlc(rv34_table_inter_thirdpat [i][j], OTHERBLK_VLC_SIZE, &inter_vlcs[i].third_pattern[j], NULL, i*12 + 104 + j);
+ ret = rv34_gen_vlc(rv34_table_inter_firstpat [i][j], FIRSTBLK_VLC_SIZE, &inter_vlcs[i].first_pattern[j], NULL, i*12 + 100 + j);
+ if (ret < 0)
+ return ret;
+ ret = rv34_gen_vlc(rv34_table_inter_secondpat[i][j], OTHERBLK_VLC_SIZE, &inter_vlcs[i].second_pattern[j], NULL, i*12 + 102 + j);
+ if (ret < 0)
+ return ret;
+ ret = rv34_gen_vlc(rv34_table_inter_thirdpat [i][j], OTHERBLK_VLC_SIZE, &inter_vlcs[i].third_pattern[j], NULL, i*12 + 104 + j);
+ if (ret < 0)
+ return ret;
}
- rv34_gen_vlc(rv34_inter_coeff[i], COEFF_VLC_SIZE, &inter_vlcs[i].coefficient, NULL, i*12 + 106);
+ ret = rv34_gen_vlc(rv34_inter_coeff[i], COEFF_VLC_SIZE, &inter_vlcs[i].coefficient, NULL, i*12 + 106);
+ if (ret < 0)
+ return ret;
}
+ return 0;
}
/** @} */ // vlc group
@@ -1503,8 +1531,11 @@ av_cold int ff_rv34_decode_init(AVCodecContext *avctx)
if ((ret = rv34_decoder_alloc(r)) < 0)
return ret;
- if(!intra_vlcs[0].cbppattern[0].bits)
- rv34_init_tables();
+ if(!intra_vlcs[0].cbppattern[0].bits) {
+ ret = rv34_init_tables();
+ if (ret < 0)
+ return ret;
+ }
avctx->internal->allocate_progress = 1;
diff --git a/libavcodec/rv40.c b/libavcodec/rv40.c
index 17574ee..edf4f29 100644
--- a/libavcodec/rv40.c
+++ b/libavcodec/rv40.c
@@ -46,9 +46,9 @@ static const int16_t mode2_offs[] = {
/**
* Initialize all tables.
*/
-static av_cold void rv40_init_tables(void)
+static av_cold int rv40_init_tables(void)
{
- int i;
+ int i, ret;
static VLC_TYPE aic_table[1 << AIC_TOP_BITS][2];
static VLC_TYPE aic_mode1_table[AIC_MODE1_NUM << AIC_MODE1_BITS][2];
static VLC_TYPE aic_mode2_table[11814][2];
@@ -79,19 +79,24 @@ static av_cold void rv40_init_tables(void)
for(i = 0; i < NUM_PTYPE_VLCS; i++){
ptype_vlc[i].table = &ptype_table[i << PTYPE_VLC_BITS];
ptype_vlc[i].table_allocated = 1 << PTYPE_VLC_BITS;
- ff_init_vlc_sparse(&ptype_vlc[i], PTYPE_VLC_BITS, PTYPE_VLC_SIZE,
+ ret = ff_init_vlc_sparse(&ptype_vlc[i], PTYPE_VLC_BITS, PTYPE_VLC_SIZE,
ptype_vlc_bits[i], 1, 1,
ptype_vlc_codes[i], 1, 1,
ptype_vlc_syms, 1, 1, INIT_VLC_USE_NEW_STATIC);
+ if (ret < 0)
+ return ret;
}
for(i = 0; i < NUM_BTYPE_VLCS; i++){
btype_vlc[i].table = &btype_table[i << BTYPE_VLC_BITS];
btype_vlc[i].table_allocated = 1 << BTYPE_VLC_BITS;
- ff_init_vlc_sparse(&btype_vlc[i], BTYPE_VLC_BITS, BTYPE_VLC_SIZE,
+ ret = ff_init_vlc_sparse(&btype_vlc[i], BTYPE_VLC_BITS, BTYPE_VLC_SIZE,
btype_vlc_bits[i], 1, 1,
btype_vlc_codes[i], 1, 1,
btype_vlc_syms, 1, 1, INIT_VLC_USE_NEW_STATIC);
+ if (ret < 0)
+ return ret;
}
+ return 0;
}
/**
@@ -554,7 +559,9 @@ static av_cold int rv40_decode_init(AVCodecContext *avctx)
if (ret < 0)
return ret;
if(!aic_top_vlc.bits)
- rv40_init_tables();
+ ret = rv40_init_tables();
+ if (ret < 0)
+ return ret;
r->parse_slice_header = rv40_parse_slice_header;
r->decode_intra_types = rv40_decode_intra_types;
r->decode_mb_info = rv40_decode_mb_info;
diff --git a/libavcodec/tiff.c b/libavcodec/tiff.c
index fdfa8f2..754c84f 100644
--- a/libavcodec/tiff.c
+++ b/libavcodec/tiff.c
@@ -1259,14 +1259,15 @@ static int decode_frame(AVCodecContext *avctx,
static av_cold int tiff_init(AVCodecContext *avctx)
{
TiffContext *s = avctx->priv_data;
+ int ret;
s->width = 0;
s->height = 0;
s->avctx = avctx;
ff_lzw_decode_open(&s->lzw);
- ff_ccitt_unpack_init();
+ ret = ff_ccitt_unpack_init();
- return 0;
+ return ret;
}
static av_cold int tiff_end(AVCodecContext *avctx)
More information about the ffmpeg-devel
mailing list