fix: enforce 5MB image size limit with progressive compression (#630)

Co-authored-by: Letta <noreply@letta.com>

src/cli/helpers/imageResize.ts

@@ -7,6 +7,10 @@ import sharp from "sharp";
 export const MAX_IMAGE_WIDTH = 2000;
 export const MAX_IMAGE_HEIGHT = 2000;
 
+// Anthropic's API enforces a 5MB limit on image bytes (not base64 string)
+// We enforce this in the client to avoid API errors
+export const MAX_IMAGE_BYTES = 5 * 1024 * 1024; // 5MB = 5,242,880 bytes
+
 export interface ResizeResult {
   data: string; // base64 encoded
   mediaType: string;
@@ -15,6 +19,67 @@ export interface ResizeResult {
   resized: boolean;
 }
 
+/**
+ * Compress an image to fit within MAX_IMAGE_BYTES using progressive JPEG quality reduction.
+ * If quality reduction alone isn't enough, also reduces dimensions.
+ * Returns null if compression is not needed (image already under limit).
+ */
+async function compressToFitByteLimit(
+  buffer: Buffer,
+  currentWidth: number,
+  currentHeight: number,
+): Promise<ResizeResult | null> {
+  // Check if compression is needed
+  if (buffer.length <= MAX_IMAGE_BYTES) {
+    return null; // No compression needed
+  }
+
+  // Try progressive JPEG quality reduction
+  const qualities = [85, 70, 55, 40];
+  for (const quality of qualities) {
+    const compressed = await sharp(buffer).jpeg({ quality }).toBuffer();
+    if (compressed.length <= MAX_IMAGE_BYTES) {
+      const meta = await sharp(compressed).metadata();
+      return {
+        data: compressed.toString("base64"),
+        mediaType: "image/jpeg",
+        width: meta.width ?? currentWidth,
+        height: meta.height ?? currentHeight,
+        resized: true,
+      };
+    }
+  }
+
+  // Quality reduction wasn't enough - also reduce dimensions
+  const scales = [0.75, 0.5, 0.25];
+  for (const scale of scales) {
+    const scaledWidth = Math.floor(currentWidth * scale);
+    const scaledHeight = Math.floor(currentHeight * scale);
+    const reduced = await sharp(buffer)
+      .resize(scaledWidth, scaledHeight, {
+        fit: "inside",
+        withoutEnlargement: true,
+      })
+      .jpeg({ quality: 70 })
+      .toBuffer();
+    if (reduced.length <= MAX_IMAGE_BYTES) {
+      const meta = await sharp(reduced).metadata();
+      return {
+        data: reduced.toString("base64"),
+        mediaType: "image/jpeg",
+        width: meta.width ?? scaledWidth,
+        height: meta.height ?? scaledHeight,
+        resized: true,
+      };
+    }
+  }
+
+  // Extremely rare: even 25% scale at q70 doesn't fit
+  throw new Error(
+    `Image too large: ${(buffer.length / 1024 / 1024).toFixed(1)}MB exceeds 5MB limit even after compression`,
+  );
+}
+
 /**
  * Resize image if it exceeds MAX_IMAGE_WIDTH or MAX_IMAGE_HEIGHT.
  * Uses 'inside' fit to preserve aspect ratio (like Codex's resize behavior).
@@ -36,7 +101,11 @@ export async function resizeImageIfNeeded(
   const isPassthroughFormat = format === "png" || format === "jpeg";
 
   if (!needsResize && isPassthroughFormat) {
-    // No resize needed and format is supported - return original bytes
+    // No resize needed and format is supported - but check byte limit
+    const compressed = await compressToFitByteLimit(buffer, width, height);
+    if (compressed) {
+      return compressed;
+    }
     return {
       data: buffer.toString("base64"),
       mediaType: inputMediaType,
@@ -69,17 +138,37 @@ export async function resizeImageIfNeeded(
     }
 
     const resizedMeta = await sharp(outputBuffer).metadata();
+    const resizedWidth = resizedMeta.width ?? 0;
+    const resizedHeight = resizedMeta.height ?? 0;
+
+    // Check byte limit after dimension resize
+    const compressed = await compressToFitByteLimit(
+      outputBuffer,
+      resizedWidth,
+      resizedHeight,
+    );
+    if (compressed) {
+      return compressed;
+    }
+
     return {
       data: outputBuffer.toString("base64"),
       mediaType: outputMediaType,
-      width: resizedMeta.width ?? 0,
-      height: resizedMeta.height ?? 0,
+      width: resizedWidth,
+      height: resizedHeight,
       resized: true,
     };
   }
 
   // No resize needed but format needs conversion (e.g., HEIC, TIFF, etc.)
   const outputBuffer = await image.png().toBuffer();
+
+  // Check byte limit after format conversion
+  const compressed = await compressToFitByteLimit(outputBuffer, width, height);
+  if (compressed) {
+    return compressed;
+  }
+
   return {
     data: outputBuffer.toString("base64"),
     mediaType: "image/png",