diff --git a/libavcodec/h264_direct.c b/libavcodec/h264_direct.c
index ec9fca0350..a01d823e7a 100644
--- a/libavcodec/h264_direct.c
+++ b/libavcodec/h264_direct.c
@@ -156,8 +156,8 @@ void ff_h264_direct_ref_list_init(const H264Context *const h, H264SliceContext *
             av_log(h->avctx, AV_LOG_ERROR, "co located POCs unavailable\n");
             sl->col_parity = 1;
         } else
-        sl->col_parity = (FFABS(col_poc[0] - cur_poc) >=
-                          FFABS(col_poc[1] - cur_poc));
+            sl->col_parity = (FFABS(col_poc[0] - (int64_t)cur_poc) >=
+                              FFABS(col_poc[1] - (int64_t)cur_poc));
         ref1sidx =
         sidx     = sl->col_parity;
     // FL -> FL & differ parity