[FFmpeg-devel] [PATCH 3/3] avcodec/dts2pts_bsf: Eliminate some 64bit corner cases

[Michael Niedermayer]

Fixes: negation of -2147483648 cannot be represented in type 'int'; cast to an unsigned type to negate this value to itself
Fixes: 53364/clusterfuzz-testcase-minimized-ffmpeg_BSF_DTS2PTS_fuzzer-4693772269387776

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
---
 libavcodec/dts2pts_bsf.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libavcodec/dts2pts_bsf.c b/libavcodec/dts2pts_bsf.c
index 8142562d2c..522d5e1eb0 100644
--- a/libavcodec/dts2pts_bsf.c
+++ b/libavcodec/dts2pts_bsf.c
@@ -301,15 +301,15 @@ static int h264_filter(AVBSFContext *ctx)
 
             if (output_picture_number != h264->last_poc) {
                 if (h264->last_poc != INT_MIN) {
-                    int diff = FFABS(h264->last_poc - output_picture_number);
+                    int64_t diff = FFABS(h264->last_poc - (int64_t)output_picture_number);
 
                     if ((output_picture_number < 0) && !h264->last_poc)
                         h264->poc_diff = 0;
-                    else if (FFABS(output_picture_number) < h264->poc_diff) {
+                    else if (FFABS((int64_t)output_picture_number) < h264->poc_diff) {
                         diff = FFABS(output_picture_number);
                         h264->poc_diff = 0;
                     }
-                    if (!h264->poc_diff || (h264->poc_diff > diff)) {
+                    if ((!h264->poc_diff || (h264->poc_diff > diff)) && diff <= INT_MAX) {
                         h264->poc_diff = diff;
                         if (h264->poc_diff == 1 && h264->sps.frame_mbs_only_flag) {
                             av_tree_enumerate(s->root, &h264->poc_diff, NULL, dec_poc);
-- 
2.17.1