diff --git a/cdjpeg.h b/cdjpeg.h
index 2d7ef377..b6980a35 100644
--- a/cdjpeg.h
+++ b/cdjpeg.h
@@ -164,6 +164,3 @@ EXTERN(FILE *) write_stdout(void);
 #ifndef EXIT_WARNING
 #define EXIT_WARNING  2
 #endif
-
-#define IsExtRGB(cs) \
-  (cs == JCS_RGB || (cs >= JCS_EXT_RGB && cs <= JCS_EXT_ARGB))
diff --git a/cjpeg.1 b/cjpeg.1
index b7ad8ce0..bce299c2 100644
--- a/cjpeg.1
+++ b/cjpeg.1
@@ -1,4 +1,4 @@
-.TH CJPEG 1 "14 November 2022"
+.TH CJPEG 1 "16 November 2022"
 .SH NAME
 cjpeg \- compress an image file to a JPEG file
 .SH SYNOPSIS
@@ -176,8 +176,8 @@ unavailable when compressing or decompressing a lossless JPEG file:
 .IP
 - Quality/quantization table selection
 .IP
-- Color conversion (the JPEG image will use the same color space as the input
-image)
+- Color space conversion (the JPEG image will use the same color space as the
+input image)
 .IP
 - DCT/IDCT algorithm selection
 .IP
diff --git a/jccolor.c b/jccolor.c
index c073b4a9..dbbcecd9 100644
--- a/jccolor.c
+++ b/jccolor.c
@@ -577,24 +577,20 @@ _jinit_color_converter(j_compress_ptr cinfo)
     break;
   }
 
-  /* Check num_components, set conversion method based on requested space */
+  /* Check num_components, set conversion method based on requested space.
+   * NOTE: We do not allow any lossy color conversion algorithms in lossless
+   * mode.
+   */
   switch (cinfo->jpeg_color_space) {
   case JCS_GRAYSCALE:
+    if (cinfo->master->lossless &&
+        cinfo->in_color_space != cinfo->jpeg_color_space)
+      ERREXIT(cinfo, JERR_CONVERSION_NOTIMPL);
     if (cinfo->num_components != 1)
       ERREXIT(cinfo, JERR_BAD_J_COLORSPACE);
     if (cinfo->in_color_space == JCS_GRAYSCALE)
       cconvert->pub._color_convert = grayscale_convert;
-    else if (cinfo->in_color_space == JCS_RGB ||
-             cinfo->in_color_space == JCS_EXT_RGB ||
-             cinfo->in_color_space == JCS_EXT_RGBX ||
-             cinfo->in_color_space == JCS_EXT_BGR ||
-             cinfo->in_color_space == JCS_EXT_BGRX ||
-             cinfo->in_color_space == JCS_EXT_XBGR ||
-             cinfo->in_color_space == JCS_EXT_XRGB ||
-             cinfo->in_color_space == JCS_EXT_RGBA ||
-             cinfo->in_color_space == JCS_EXT_BGRA ||
-             cinfo->in_color_space == JCS_EXT_ABGR ||
-             cinfo->in_color_space == JCS_EXT_ARGB) {
+    else if (IsExtRGB(cinfo->in_color_space)) {
 #ifdef WITH_SIMD
       if (jsimd_can_rgb_gray())
         cconvert->pub._color_convert = jsimd_rgb_gray_convert;
@@ -611,6 +607,8 @@ _jinit_color_converter(j_compress_ptr cinfo)
     break;
 
   case JCS_RGB:
+    if (cinfo->master->lossless && !IsExtRGB(cinfo->in_color_space))
+      ERREXIT(cinfo, JERR_CONVERSION_NOTIMPL);
     if (cinfo->num_components != 3)
       ERREXIT(cinfo, JERR_BAD_J_COLORSPACE);
     if (rgb_red[cinfo->in_color_space] == 0 &&
@@ -623,36 +621,19 @@ _jinit_color_converter(j_compress_ptr cinfo)
       else
 #endif
         cconvert->pub._color_convert = null_convert;
-    } else if (cinfo->in_color_space == JCS_RGB ||
-               cinfo->in_color_space == JCS_EXT_RGB ||
-               cinfo->in_color_space == JCS_EXT_RGBX ||
-               cinfo->in_color_space == JCS_EXT_BGR ||
-               cinfo->in_color_space == JCS_EXT_BGRX ||
-               cinfo->in_color_space == JCS_EXT_XBGR ||
-               cinfo->in_color_space == JCS_EXT_XRGB ||
-               cinfo->in_color_space == JCS_EXT_RGBA ||
-               cinfo->in_color_space == JCS_EXT_BGRA ||
-               cinfo->in_color_space == JCS_EXT_ABGR ||
-               cinfo->in_color_space == JCS_EXT_ARGB)
+    } else if (IsExtRGB(cinfo->in_color_space))
       cconvert->pub._color_convert = rgb_rgb_convert;
     else
       ERREXIT(cinfo, JERR_CONVERSION_NOTIMPL);
     break;
 
   case JCS_YCbCr:
+    if (cinfo->master->lossless &&
+        cinfo->in_color_space != cinfo->jpeg_color_space)
+      ERREXIT(cinfo, JERR_CONVERSION_NOTIMPL);
     if (cinfo->num_components != 3)
       ERREXIT(cinfo, JERR_BAD_J_COLORSPACE);
-    if (cinfo->in_color_space == JCS_RGB ||
-        cinfo->in_color_space == JCS_EXT_RGB ||
-        cinfo->in_color_space == JCS_EXT_RGBX ||
-        cinfo->in_color_space == JCS_EXT_BGR ||
-        cinfo->in_color_space == JCS_EXT_BGRX ||
-        cinfo->in_color_space == JCS_EXT_XBGR ||
-        cinfo->in_color_space == JCS_EXT_XRGB ||
-        cinfo->in_color_space == JCS_EXT_RGBA ||
-        cinfo->in_color_space == JCS_EXT_BGRA ||
-        cinfo->in_color_space == JCS_EXT_ABGR ||
-        cinfo->in_color_space == JCS_EXT_ARGB) {
+    if (IsExtRGB(cinfo->in_color_space)) {
 #ifdef WITH_SIMD
       if (jsimd_can_rgb_ycc())
         cconvert->pub._color_convert = jsimd_rgb_ycc_convert;
@@ -674,6 +655,9 @@ _jinit_color_converter(j_compress_ptr cinfo)
     break;
 
   case JCS_CMYK:
+    if (cinfo->master->lossless &&
+        cinfo->in_color_space != cinfo->jpeg_color_space)
+      ERREXIT(cinfo, JERR_CONVERSION_NOTIMPL);
     if (cinfo->num_components != 4)
       ERREXIT(cinfo, JERR_BAD_J_COLORSPACE);
     if (cinfo->in_color_space == JCS_CMYK) {
@@ -688,6 +672,9 @@ _jinit_color_converter(j_compress_ptr cinfo)
     break;
 
   case JCS_YCCK:
+    if (cinfo->master->lossless &&
+        cinfo->in_color_space != cinfo->jpeg_color_space)
+      ERREXIT(cinfo, JERR_CONVERSION_NOTIMPL);
     if (cinfo->num_components != 4)
       ERREXIT(cinfo, JERR_BAD_J_COLORSPACE);
     if (cinfo->in_color_space == JCS_CMYK) {
@@ -716,13 +703,4 @@ _jinit_color_converter(j_compress_ptr cinfo)
       cconvert->pub._color_convert = null_convert;
     break;
   }
-
-  /* Prevent lossy color conversion in lossless mode */
-  if (cinfo->master->lossless) {
-    if ((cinfo->jpeg_color_space == JCS_GRAYSCALE &&
-         cinfo->in_color_space != JCS_GRAYSCALE) ||
-        (cinfo->jpeg_color_space != JCS_GRAYSCALE &&
-         cconvert->pub._color_convert != null_convert))
-      ERREXIT(cinfo, JERR_CONVERSION_NOTIMPL);
-  }
 }
diff --git a/jdcolor.c b/jdcolor.c
index 01f2318e..132d8478 100644
--- a/jdcolor.c
+++ b/jdcolor.c
@@ -776,10 +776,15 @@ _jinit_color_deconverter(j_decompress_ptr cinfo)
   /* Set out_color_components and conversion method based on requested space.
    * Also clear the component_needed flags for any unused components,
    * so that earlier pipeline stages can avoid useless computation.
+   * NOTE: We do not allow any lossy color conversion algorithms in lossless
+   * mode.
    */
 
   switch (cinfo->out_color_space) {
   case JCS_GRAYSCALE:
+    if (cinfo->master->lossless &&
+        cinfo->jpeg_color_space != cinfo->out_color_space)
+      ERREXIT(cinfo, JERR_CONVERSION_NOTIMPL);
     cinfo->out_color_components = 1;
     if (cinfo->jpeg_color_space == JCS_GRAYSCALE ||
         cinfo->jpeg_color_space == JCS_YCbCr) {
@@ -805,6 +810,8 @@ _jinit_color_deconverter(j_decompress_ptr cinfo)
   case JCS_EXT_BGRA:
   case JCS_EXT_ABGR:
   case JCS_EXT_ARGB:
+    if (cinfo->master->lossless && cinfo->jpeg_color_space != JCS_RGB)
+      ERREXIT(cinfo, JERR_CONVERSION_NOTIMPL);
     cinfo->out_color_components = rgb_pixelsize[cinfo->out_color_space];
     if (cinfo->jpeg_color_space == JCS_YCbCr) {
 #ifdef WITH_SIMD
@@ -831,6 +838,8 @@ _jinit_color_deconverter(j_decompress_ptr cinfo)
     break;
 
   case JCS_RGB565:
+    if (cinfo->master->lossless)
+      ERREXIT(cinfo, JERR_CONVERSION_NOTIMPL);
     cinfo->out_color_components = 3;
     if (cinfo->dither_mode == JDITHER_NONE) {
       if (cinfo->jpeg_color_space == JCS_YCbCr) {
@@ -864,6 +873,9 @@ _jinit_color_deconverter(j_decompress_ptr cinfo)
     break;
 
   case JCS_CMYK:
+    if (cinfo->master->lossless &&
+        cinfo->jpeg_color_space != cinfo->out_color_space)
+      ERREXIT(cinfo, JERR_CONVERSION_NOTIMPL);
     cinfo->out_color_components = 4;
     if (cinfo->jpeg_color_space == JCS_YCCK) {
       cconvert->pub._color_convert = ycck_cmyk_convert;
@@ -884,15 +896,6 @@ _jinit_color_deconverter(j_decompress_ptr cinfo)
     break;
   }
 
-  /* Prevent lossy color conversion in lossless mode */
-  if (cinfo->master->lossless) {
-    if ((cinfo->out_color_space == JCS_GRAYSCALE &&
-         cinfo->jpeg_color_space != JCS_GRAYSCALE) ||
-        (cinfo->out_color_space != JCS_GRAYSCALE &&
-         cconvert->pub._color_convert != null_convert))
-      ERREXIT(cinfo, JERR_CONVERSION_NOTIMPL);
-  }
-
   if (cinfo->quantize_colors)
     cinfo->output_components = 1; /* single colormapped output component */
   else
diff --git a/jpegint.h b/jpegint.h
index decd5557..0854e8d3 100644
--- a/jpegint.h
+++ b/jpegint.h
@@ -7,7 +7,7 @@
  * Lossless JPEG Modifications:
  * Copyright (C) 1999, Ken Murchison.
  * libjpeg-turbo Modifications:
- * Copyright (C) 2015-2016, 2019, 2021-2022, D. R. Commander.
+ * Copyright (C) 2015-2017, 2019, 2021-2022, D. R. Commander.
  * Copyright (C) 2015, Google, Inc.
  * Copyright (C) 2021, Alex Richardson.
  * For conditions of distribution and use, see the accompanying README.ijg
@@ -74,6 +74,9 @@ typedef __UINTPTR_TYPE__ JUINTPTR;
 typedef size_t JUINTPTR;
 #endif
 
+#define IsExtRGB(cs) \
+  (cs == JCS_RGB || (cs >= JCS_EXT_RGB && cs <= JCS_EXT_ARGB))
+
 /*
  * Left shift macro that handles a negative operand without causing any
  * sanitizer warnings
diff --git a/libjpeg.txt b/libjpeg.txt
index f81037ee..62ef64e1 100644
--- a/libjpeg.txt
+++ b/libjpeg.txt
@@ -1010,8 +1010,8 @@ jpeg_enable_lossless (j_compress_ptr cinfo, int predictor_selection_value,
           * DCT/IDCT algorithm selection
           * Smoothing
           * Downsampling/upsampling
-          * Color conversion (the JPEG image will use the same color space as
-            the input image)
+          * Color space conversion (the JPEG image will use the same color
+            space as the input image)
           * IDCT scaling
           * Raw (downsampled) data input/output
           * Transcoding of DCT coefficients
diff --git a/usage.txt b/usage.txt
index ed3ea47f..6251c3fa 100644
--- a/usage.txt
+++ b/usage.txt
@@ -177,8 +177,8 @@ Switches for advanced users:
                         following features will be unavailable when compressing
                         or decompressing a lossless JPEG file:
                           * Quality/quantization table selection
-                          * Color conversion (the JPEG image will use the same
-                            color space as the input image)
+                          * Color space conversion (the JPEG image will use the
+                            same color space as the input image)
                           * DCT/IDCT algorithm selection
                           * Smoothing
                           * Downsampling/upsampling