Fixed buffer overflow when using Video Toolbox (#3738)

pjsip · Oct 13, 2023 · 6aa5349 · 6aa5349
1 parent 5c5b328
commit 6aa5349
Showing 1 changed file with 12 additions and 5 deletions.
diff --git a/pjmedia/src/pjmedia-codec/vid_toolbox.m b/pjmedia/src/pjmedia-codec/vid_toolbox.m
@@ -164,7 +164,7 @@ static pj_status_t vtool_codec_decode(pjmedia_vid_codec *codec,
     pj_uint8_t                  *dec_buf;
     unsigned                     dec_buf_size;
     CMFormatDescriptionRef       dec_format;
-    OSStatus             dec_status;
+    OSStatus                     dec_status;
 
     unsigned                     dec_sps_size;
     unsigned                     dec_pps_size;
@@ -1042,7 +1042,7 @@ static void decode_cb(void *decompressionOutputRefCon,
                       CMTime presentationDuration)
 {
     struct vtool_codec_data *vtool_data;
-    pj_size_t width, height, len;
+    pj_size_t width, height, len = 0;
 
     /* This callback can be called from another, unregistered thread.
      * So do not call pjlib functions here.
@@ -1068,7 +1068,12 @@ static void decode_cb(void *decompressionOutputRefCon,
         vtool_data->dec_fmt_change = PJ_FALSE;
     }
 
-    len = process_i420(imageBuffer, (pj_uint8_t *)vtool_data->dec_frame->buf);
+    if (vtool_data->dec_frame->size >= width * height * 3 / 2) {
+        len = process_i420(imageBuffer,
+                           (pj_uint8_t *)vtool_data->dec_frame->buf);
+    } else {
+        vtool_data->dec_status = (OSStatus)PJMEDIA_CODEC_EFRMTOOSHORT;
+    }
     vtool_data->dec_frame->size = len;
 
     CVPixelBufferUnlockBaseAddress(imageBuffer,0);
@@ -1308,6 +1313,7 @@ static pj_status_t vtool_codec_decode(pjmedia_vid_codec *codec,
 
             if (ret == noErr) {
                 vtool_data->dec_frame = output;
+                vtool_data->dec_frame->size = out_size;
                 ret = VTDecompressionSessionDecodeFrame(
                           vtool_data->dec, sample_buf, 0,
                           NULL, NULL);
@@ -1345,8 +1351,9 @@ static pj_status_t vtool_codec_decode(pjmedia_vid_codec *codec,
                 }
 
                 if ((ret != noErr) || (vtool_data->dec_status != noErr)) {
-            char *ret_err = (ret != noErr)?"decode err":"cb err";
-            OSStatus err_code = (ret != noErr)?ret:vtool_data->dec_status;
+                    char *ret_err = (ret != noErr)?"decode err":"cb err";
+                    OSStatus err_code = (ret != noErr)? ret:
+                                        vtool_data->dec_status;
 
                     PJ_LOG(5,(THIS_FILE, "Failed to decode frame %d of size "
                                  "%d %s:%d", nalu_type, frm_size, ret_err,