diff --git a/libavcodec/vdpau_mpeg12.c b/libavcodec/vdpau_mpeg12.c
index 354239cad5..79007aa1a8 100644
--- a/libavcodec/vdpau_mpeg12.c
+++ b/libavcodec/vdpau_mpeg12.c
@@ -75,8 +75,9 @@ static int vdpau_mpeg_start_frame(AVCodecContext *avctx,
     info->f_code[1][0]               = s->mpeg_f_code[1][0];
     info->f_code[1][1]               = s->mpeg_f_code[1][1];
     for (i = 0; i < 64; ++i) {
-        info->intra_quantizer_matrix[i]     = s->intra_matrix[i];
-        info->non_intra_quantizer_matrix[i] = s->inter_matrix[i];
+        int n = s->idsp.idct_permutation[i];
+        info->intra_quantizer_matrix[i]     = s->intra_matrix[n];
+        info->non_intra_quantizer_matrix[i] = s->inter_matrix[n];
     }
 
     return ff_vdpau_common_start_frame(pic_ctx, buffer, size);