[FFmpeg-devel] [PATCH]lavc/cfhddata: Reduce required stack size

[Carl Eugen Hoyos]

Hi!

Attached patch reduces the stack size of an init function, fixes part
of ticket #9399.

Please comment, Carl Eugen
-------------- next part --------------
From 0f034ebed2388e89e241d7e08bf59d335b5c4cee Mon Sep 17 00:00:00 2001
From: Carl Eugen Hoyos <ceffmpeg at gmail.com>
Date: Fri, 2 Sep 2022 23:36:29 +0200
Subject: [PATCH] lavc/cfhddata: Reduce required stack size.

Fixes part of ticket #9399.
---
 libavcodec/cfhddata.c | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/libavcodec/cfhddata.c b/libavcodec/cfhddata.c
index 55c8004bdd..17aa888c84 100644
--- a/libavcodec/cfhddata.c
+++ b/libavcodec/cfhddata.c
@@ -276,10 +276,20 @@ static const uint8_t table_18_vlc_level[NB_VLC_TABLE_18] = {
 av_cold int ff_cfhd_init_vlcs(CFHDContext *s)
 {
     int i, j, ret = 0;
-    uint32_t new_cfhd_vlc_bits[NB_VLC_TABLE_18 * 2];
-    uint8_t  new_cfhd_vlc_len[NB_VLC_TABLE_18 * 2];
-    uint16_t new_cfhd_vlc_run[NB_VLC_TABLE_18 * 2];
-    int16_t  new_cfhd_vlc_level[NB_VLC_TABLE_18 * 2];
+    uint32_t *new_cfhd_vlc_bits;
+    uint8_t  *new_cfhd_vlc_len;
+    uint16_t *new_cfhd_vlc_run;
+    int16_t  *new_cfhd_vlc_level;
+
+    new_cfhd_vlc_bits  = av_malloc(NB_VLC_TABLE_18 * 2 * 4);
+    new_cfhd_vlc_len   = av_malloc(NB_VLC_TABLE_18 * 2);
+    new_cfhd_vlc_run   = av_malloc(NB_VLC_TABLE_18 * 2 * 2);
+    new_cfhd_vlc_level = av_malloc(NB_VLC_TABLE_18 * 2 * 2);
+
+    if (!new_cfhd_vlc_bits || !new_cfhd_vlc_len || !new_cfhd_vlc_run || !new_cfhd_vlc_level) {
+        ret = AVERROR(ENOMEM);
+        goto end;
+    }
 
     /** Similar to dv.c, generate signed VLC tables **/
 
@@ -306,7 +316,7 @@ av_cold int ff_cfhd_init_vlcs(CFHDContext *s)
     ret = init_vlc(&s->vlc_9, VLC_BITS, j, new_cfhd_vlc_len,
                    1, 1, new_cfhd_vlc_bits, 4, 4, 0);
     if (ret < 0)
-        return ret;
+        goto end;
     for (i = 0; i < s->vlc_9.table_size; i++) {
         int code = s->vlc_9.table[i].sym;
         int len  = s->vlc_9.table[i].len;
@@ -347,7 +357,7 @@ av_cold int ff_cfhd_init_vlcs(CFHDContext *s)
     ret = init_vlc(&s->vlc_18, VLC_BITS, j, new_cfhd_vlc_len,
                    1, 1, new_cfhd_vlc_bits, 4, 4, 0);
     if (ret < 0)
-        return ret;
+        goto end;
     av_assert0(s->vlc_18.table_size == 4572);
 
     for (i = 0; i < s->vlc_18.table_size; i++) {
@@ -367,5 +377,11 @@ av_cold int ff_cfhd_init_vlcs(CFHDContext *s)
         s->table_18_rl_vlc[i].run   = run;
     }
 
+end:
+    av_free(new_cfhd_vlc_bits);
+    av_free(new_cfhd_vlc_len);
+    av_free(new_cfhd_vlc_run);
+    av_free(new_cfhd_vlc_level);
+
     return ret;
 }
-- 
2.17.1