diff --git a/jdhuff.c b/jdhuff.c
index f6b6e008..b795462f 100644
--- a/jdhuff.c
+++ b/jdhuff.c
@@ -740,9 +740,9 @@ decode_mcu_fast (j_decompress_ptr cinfo, JBLOCKROW *MCU_data)
     }
   }
 
-  if (cinfo->unread_marker != 0 && ! cinfo->entropy->insufficient_data) {
-    WARNMS(cinfo, JWRN_HIT_MARKER);
-    cinfo->entropy->insufficient_data = TRUE;
+  if (cinfo->unread_marker != 0) {
+    cinfo->unread_marker = 0;
+    return FALSE;
   }
 
   br_state.bytes_in_buffer -= (buffer - br_state.next_input_byte);
@@ -784,7 +784,8 @@ decode_mcu (j_decompress_ptr cinfo, JBLOCKROW *MCU_data)
     usefast = 0;
   }
 
-  if (cinfo->src->bytes_in_buffer < BUFSIZE * cinfo->blocks_in_MCU)
+  if (cinfo->src->bytes_in_buffer < BUFSIZE * cinfo->blocks_in_MCU
+    || cinfo->unread_marker != 0)
     usefast = 0;
 
   /* If we've run out of data, just leave the MCU set to zeroes.
@@ -793,9 +794,10 @@ decode_mcu (j_decompress_ptr cinfo, JBLOCKROW *MCU_data)
   if (! entropy->pub.insufficient_data) {
 
     if (usefast) {
-      if (!decode_mcu_fast(cinfo, MCU_data)) return FALSE;
+      if (!decode_mcu_fast(cinfo, MCU_data)) goto use_slow;
     }
     else {
+      use_slow:
       if (!decode_mcu_slow(cinfo, MCU_data)) return FALSE;
     }