diff --git a/lib/extras/enc/jpegli.cc b/lib/extras/enc/jpegli.cc
index 280b8661..28c4499c 100644
--- a/lib/extras/enc/jpegli.cc
+++ b/lib/extras/enc/jpegli.cc
@@ -414,6 +414,8 @@ Status EncodeJpeg(const PackedPixelFile& ppf, const JpegSettings& jpeg_settings,
     }
     jpegli_set_cicp_transfer_function(&cinfo, cicp_tf);
     jpegli_set_defaults(&cinfo);
+    // All factors need to be specified to subsample the blue channel
+    // for XYB. H and V are swapped between YCbCr and XYB.
     if (!jpeg_settings.chroma_subsampling.empty()) {
       if (jpeg_settings.chroma_subsampling == "444") {
         cinfo.comp_info[0].h_samp_factor = 1;
@@ -421,23 +423,39 @@ Status EncodeJpeg(const PackedPixelFile& ppf, const JpegSettings& jpeg_settings,
       } else if (jpeg_settings.chroma_subsampling == "440") {
         cinfo.comp_info[0].h_samp_factor = 1;
         cinfo.comp_info[0].v_samp_factor = 2;
+        if (jpeg_settings.xyb) {
+          cinfo.comp_info[0].h_samp_factor = 2;
+          cinfo.comp_info[0].v_samp_factor = 2;
+          cinfo.comp_info[1].h_samp_factor = 2;
+          cinfo.comp_info[1].v_samp_factor = 2;
+          cinfo.comp_info[2].h_samp_factor = 2;
+          cinfo.comp_info[2].v_samp_factor = 1;
+        }
       } else if (jpeg_settings.chroma_subsampling == "422") {
         cinfo.comp_info[0].h_samp_factor = 2;
         cinfo.comp_info[0].v_samp_factor = 1;
+        if (jpeg_settings.xyb) {
+          cinfo.comp_info[0].h_samp_factor = 2;
+          cinfo.comp_info[0].v_samp_factor = 2;
+          cinfo.comp_info[1].h_samp_factor = 2;
+          cinfo.comp_info[1].v_samp_factor = 2;
+          cinfo.comp_info[2].h_samp_factor = 1;
+          cinfo.comp_info[2].v_samp_factor = 2;
+        }
       } else if (jpeg_settings.chroma_subsampling == "420") {
         cinfo.comp_info[0].h_samp_factor = 2;
         cinfo.comp_info[0].v_samp_factor = 2;
+        if (jpeg_settings.xyb) {
+          cinfo.comp_info[0].h_samp_factor = 2;
+          cinfo.comp_info[0].v_samp_factor = 2;
+          cinfo.comp_info[1].h_samp_factor = 2;
+          cinfo.comp_info[1].v_samp_factor = 2;
+          cinfo.comp_info[2].h_samp_factor = 1;
+          cinfo.comp_info[2].v_samp_factor = 1;
+        }
       } else {
         return false;
       }
-      for (int i = 1; i < cinfo.num_components; ++i) {
-        cinfo.comp_info[i].h_samp_factor = 1;
-        cinfo.comp_info[i].v_samp_factor = 1;
-      }
-    } else if (!jpeg_settings.xyb) {
-      // Default is no chroma subsampling.
-      cinfo.comp_info[0].h_samp_factor = 1;
-      cinfo.comp_info[0].v_samp_factor = 1;
     }
     jpegli_enable_adaptive_quantization(
         &cinfo, TO_JPEGLI_BOOL(jpeg_settings.use_adaptive_quantization));
diff --git a/lib/jpegli/encode.cc b/lib/jpegli/encode.cc
index 5a5e320a..4f207e7e 100644
--- a/lib/jpegli/encode.cc
+++ b/lib/jpegli/encode.cc
@@ -800,6 +800,7 @@ void jpegli_set_colorspace(j_compress_ptr cinfo, J_COLOR_SPACE colorspace) {
     jpeg_component_info* comp = &cinfo->comp_info[c];
     comp->component_index = c;
     comp->component_id = c + 1;
+    // Default is no chroma subsampling.
     comp->h_samp_factor = 1;
     comp->v_samp_factor = 1;
     comp->quant_tbl_no = 0;
@@ -811,10 +812,6 @@ void jpegli_set_colorspace(j_compress_ptr cinfo, J_COLOR_SPACE colorspace) {
     cinfo->comp_info[1].component_id = 'G';
     cinfo->comp_info[2].component_id = 'B';
     if (cinfo->master->xyb_mode) {
-      // Subsample blue channel.
-      cinfo->comp_info[0].h_samp_factor = cinfo->comp_info[0].v_samp_factor = 2;
-      cinfo->comp_info[1].h_samp_factor = cinfo->comp_info[1].v_samp_factor = 2;
-      cinfo->comp_info[2].h_samp_factor = cinfo->comp_info[2].v_samp_factor = 1;
       // Use separate quantization tables for each component
       cinfo->comp_info[1].quant_tbl_no = 1;
       cinfo->comp_info[2].quant_tbl_no = 2;
@@ -830,11 +827,6 @@ void jpegli_set_colorspace(j_compress_ptr cinfo, J_COLOR_SPACE colorspace) {
     cinfo->comp_info[2].quant_tbl_no = 1;
     cinfo->comp_info[1].dc_tbl_no = cinfo->comp_info[1].ac_tbl_no = 1;
     cinfo->comp_info[2].dc_tbl_no = cinfo->comp_info[2].ac_tbl_no = 1;
-    // Use chroma subsampling by default
-    cinfo->comp_info[0].h_samp_factor = cinfo->comp_info[0].v_samp_factor = 2;
-    if (colorspace == JCS_YCCK) {
-      cinfo->comp_info[3].h_samp_factor = cinfo->comp_info[3].v_samp_factor = 2;
-    }
   }
 }