From 271289a00b500d1fa625e156b5d4f8eacdd4fe78 Mon Sep 17 00:00:00 2001 From: Charles Packer Date: Tue, 20 Jan 2026 19:52:46 -0800 Subject: [PATCH] fix: streaming flicker aggressive static promotion (#608) Co-authored-by: Letta --- src/cli/App.tsx | 5 + src/cli/components/AssistantMessageRich.tsx | 5 +- src/cli/components/ReasoningMessageRich.tsx | 17 ++- src/cli/helpers/accumulator.ts | 87 +++++++++++++-- src/cli/helpers/markdownSplit.ts | 116 ++++++++++++++++++++ 5 files changed, 220 insertions(+), 10 deletions(-) create mode 100644 src/cli/helpers/markdownSplit.ts diff --git a/src/cli/App.tsx b/src/cli/App.tsx index 2a5b85e..eec2231 100644 --- a/src/cli/App.tsx +++ b/src/cli/App.tsx @@ -1236,6 +1236,11 @@ export default function App({ // Track whether we've already backfilled history (should only happen once) const hasBackfilledRef = useRef(false); + // Keep buffers in sync with tokenStreamingEnabled state for aggressive static promotion + useEffect(() => { + buffersRef.current.tokenStreamingEnabled = tokenStreamingEnabled; + }, [tokenStreamingEnabled]); + // Cache precomputed diffs from approval dialogs for tool return rendering // Key: toolCallId or "toolCallId:filePath" for Patch operations const precomputedDiffsRef = useRef>( diff --git a/src/cli/components/AssistantMessageRich.tsx b/src/cli/components/AssistantMessageRich.tsx index 6d81013..b4f50d3 100644 --- a/src/cli/components/AssistantMessageRich.tsx +++ b/src/cli/components/AssistantMessageRich.tsx @@ -16,6 +16,7 @@ type AssistantLine = { id: string; text: string; phase: "streaming" | "finished"; + isContinuation?: boolean; }; /** @@ -23,7 +24,7 @@ type AssistantLine = { * This is a direct port from the old letta-code codebase to preserve the exact styling * * Features: - * - Left column (2 chars wide) with bullet point marker + * - Left column (2 chars wide) with bullet point marker (unless continuation) * - Right column with wrapped text content * - Proper text normalization * - Support for markdown rendering (when MarkdownDisplay is available) @@ -37,7 +38,7 @@ export const AssistantMessage = memo(({ line }: { line: AssistantLine }) => { return ( - ● + {line.isContinuation ? " " : "●"} diff --git a/src/cli/components/ReasoningMessageRich.tsx b/src/cli/components/ReasoningMessageRich.tsx index eb5e6da..258134f 100644 --- a/src/cli/components/ReasoningMessageRich.tsx +++ b/src/cli/components/ReasoningMessageRich.tsx @@ -16,6 +16,7 @@ type ReasoningLine = { id: string; text: string; phase: "streaming" | "finished"; + isContinuation?: boolean; }; /** @@ -23,7 +24,7 @@ type ReasoningLine = { * This is a direct port from the old letta-code codebase to preserve the exact styling * * Features: - * - Header row with "✻" symbol and "Thinking…" text + * - Header row with "✻" symbol and "Thinking…" text (unless continuation) * - Reasoning content indented with 2 spaces * - Full markdown rendering with dimmed colors * - Proper text normalization @@ -34,6 +35,20 @@ export const ReasoningMessage = memo(({ line }: { line: ReasoningLine }) => { const normalizedText = normalize(line.text); + // Continuation lines skip the header, just show content + if (line.isContinuation) { + return ( + + + + + + + + + ); + } + return ( diff --git a/src/cli/helpers/accumulator.ts b/src/cli/helpers/accumulator.ts index c78338e..eb557d1 100644 --- a/src/cli/helpers/accumulator.ts +++ b/src/cli/helpers/accumulator.ts @@ -6,6 +6,7 @@ import type { LettaStreamingResponse } from "@letta-ai/letta-client/resources/agents/messages"; import { INTERRUPTED_BY_USER } from "../../constants"; +import { findLastSafeSplitPoint } from "./markdownSplit"; import { isShellTool } from "./toolNameMapping"; // Constants for streaming output @@ -104,12 +105,14 @@ export type Line = id: string; text: string; phase: "streaming" | "finished"; + isContinuation?: boolean; // true for split continuation lines (no header) } | { kind: "assistant"; id: string; text: string; phase: "streaming" | "finished"; + isContinuation?: boolean; // true for split continuation lines (no bullet) } | { kind: "tool_call"; @@ -174,6 +177,9 @@ export type Buffers = { reasoningTokens: number; stepCount: number; }; + // Aggressive static promotion: split streaming content at paragraph boundaries + tokenStreamingEnabled?: boolean; + splitCounters: Map; // tracks split count per original otid }; export function createBuffers(): Buffers { @@ -194,6 +200,8 @@ export function createBuffers(): Buffers { reasoningTokens: 0, stepCount: 0, }, + tokenStreamingEnabled: false, + splitCounters: new Map(), }; } @@ -337,6 +345,63 @@ function extractTextPart(v: unknown): string { return ""; } +/** + * Attempts to split content at a paragraph boundary for aggressive static promotion. + * If split found, creates a committed line for "before" and updates original with "after". + * Returns true if split occurred, false otherwise. + */ +function trySplitContent( + b: Buffers, + id: string, + kind: "assistant" | "reasoning", + newText: string, +): boolean { + if (!b.tokenStreamingEnabled) return false; + + const splitPoint = findLastSafeSplitPoint(newText); + if (splitPoint >= newText.length) return false; // No safe split point + + const beforeText = newText.substring(0, splitPoint); + const afterText = newText.substring(splitPoint); + + // Get or initialize split counter for this original ID + const counter = b.splitCounters.get(id) ?? 0; + b.splitCounters.set(id, counter + 1); + + // Create committed line for "before" content + // Only the first split (counter=0) shows the bullet/header; subsequent splits are continuations + const commitId = `${id}-split-${counter}`; + const committedLine = { + kind, + id: commitId, + text: beforeText, + phase: "finished" as const, + isContinuation: counter > 0, // First split shows bullet, subsequent don't + }; + b.byId.set(commitId, committedLine); + + // Insert committed line BEFORE the original in order array + const originalIndex = b.order.indexOf(id); + if (originalIndex !== -1) { + b.order.splice(originalIndex, 0, commitId); + } else { + // Should not happen, but handle gracefully + b.order.push(commitId); + } + + // Update original line with just the "after" content (keep streaming) + // Mark it as a continuation so it doesn't show bullet/header + const originalLine = b.byId.get(id); + if ( + originalLine && + (originalLine.kind === "assistant" || originalLine.kind === "reasoning") + ) { + b.byId.set(id, { ...originalLine, text: afterText, isContinuation: true }); + } + + return true; +} + // Feed one SDK chunk; mutate buffers in place. export function onChunk(b: Buffers, chunk: LettaStreamingResponse) { // Skip processing if stream was interrupted mid-turn. handleInterrupt already @@ -371,11 +436,15 @@ export function onChunk(b: Buffers, chunk: LettaStreamingResponse) { phase: "streaming", })); if (delta) { - // Immutable update: create new object with updated text - const updatedLine = { ...line, text: line.text + delta }; - b.byId.set(id, updatedLine); + const newText = line.text + delta; b.tokenCount += delta.length; - // console.log(`[REASONING] Updated ${id}, phase=${updatedLine.phase}, textLen=${updatedLine.text.length}`); + + // Try to split at paragraph boundary (only if streaming enabled) + if (!trySplitContent(b, id, "reasoning", newText)) { + // No split - normal accumulation + b.byId.set(id, { ...line, text: newText }); + } + // console.log(`[REASONING] Updated ${id}, textLen=${newText.length}`); } break; } @@ -395,10 +464,14 @@ export function onChunk(b: Buffers, chunk: LettaStreamingResponse) { phase: "streaming", })); if (delta) { - // Immutable update: create new object with updated text - const updatedLine = { ...line, text: line.text + delta }; - b.byId.set(id, updatedLine); + const newText = line.text + delta; b.tokenCount += delta.length; + + // Try to split at paragraph boundary (only if streaming enabled) + if (!trySplitContent(b, id, "assistant", newText)) { + // No split - normal accumulation + b.byId.set(id, { ...line, text: newText }); + } } break; } diff --git a/src/cli/helpers/markdownSplit.ts b/src/cli/helpers/markdownSplit.ts new file mode 100644 index 0000000..6de542d --- /dev/null +++ b/src/cli/helpers/markdownSplit.ts @@ -0,0 +1,116 @@ +// src/cli/helpers/markdownSplit.ts +// Markdown-aware content splitting for aggressive static promotion. +// Ported from Gemini CLI: packages/cli/src/ui/utils/markdownUtilities.ts + +/** + * Checks if a given character index is inside a fenced code block (```). + * Counts fence markers before the index - odd count means inside a block. + * Only counts ``` at the start of a line (real markdown fences), not ones + * embedded in code like: content.indexOf("```") + */ +function isIndexInsideCodeBlock(content: string, indexToTest: number): boolean { + let fenceCount = 0; + let searchPos = 0; + while (searchPos < content.length) { + const nextFence = content.indexOf("```", searchPos); + if (nextFence === -1 || nextFence >= indexToTest) { + break; + } + // Only count as fence if at start of content or after a newline + if (nextFence === 0 || content[nextFence - 1] === "\n") { + fenceCount++; + } + searchPos = nextFence + 3; + } + return fenceCount % 2 === 1; +} + +/** + * Finds the next fence marker (``` at start of line) starting from pos. + * Returns -1 if not found. + */ +function findNextLineFence(content: string, startPos: number): number { + let pos = startPos; + while (pos < content.length) { + const nextFence = content.indexOf("```", pos); + if (nextFence === -1) return -1; + // Only count as fence if at start of content or after a newline + if (nextFence === 0 || content[nextFence - 1] === "\n") { + return nextFence; + } + pos = nextFence + 3; + } + return -1; +} + +/** + * Finds the starting index of the code block that encloses the given index. + * Returns -1 if the index is not inside a code block. + */ +function findEnclosingCodeBlockStart(content: string, index: number): number { + if (!isIndexInsideCodeBlock(content, index)) { + return -1; + } + let currentSearchPos = 0; + while (currentSearchPos < index) { + const blockStartIndex = findNextLineFence(content, currentSearchPos); + if (blockStartIndex === -1 || blockStartIndex >= index) { + break; + } + const blockEndIndex = findNextLineFence(content, blockStartIndex + 3); + if (blockStartIndex < index) { + if (blockEndIndex === -1 || index < blockEndIndex + 3) { + return blockStartIndex; + } + } + if (blockEndIndex === -1) break; + currentSearchPos = blockEndIndex + 3; + } + return -1; +} + +// Minimum content length before we consider splitting +// This prevents creating many tiny chunks which causes spacing issues +// Higher value = fewer splits = cleaner output but more content re-rendering +const MIN_SPLIT_LENGTH = 1500; + +/** + * Finds the last safe split point in content (paragraph boundary not inside code block). + * Returns content.length if no safe split point found (meaning don't split). + * + * Used for aggressive static promotion during streaming - completed paragraphs + * can be committed to Ink's component to reduce flicker. + */ +export function findLastSafeSplitPoint(content: string): number { + // Don't split if content is too short - prevents excessive chunking + if (content.length < MIN_SPLIT_LENGTH) { + return content.length; + } + // If end of content is inside a code block, split before that block + const enclosingBlockStart = findEnclosingCodeBlockStart( + content, + content.length, + ); + if (enclosingBlockStart !== -1) { + return enclosingBlockStart; + } + + // Search for the last double newline (\n\n) not in a code block + let searchStartIndex = content.length; + while (searchStartIndex >= 0) { + const dnlIndex = content.lastIndexOf("\n\n", searchStartIndex); + if (dnlIndex === -1) { + break; + } + + const potentialSplitPoint = dnlIndex + 2; // Split AFTER the \n\n + if (!isIndexInsideCodeBlock(content, potentialSplitPoint)) { + return potentialSplitPoint; + } + + searchStartIndex = dnlIndex - 1; + } + + // No safe split point found - don't split + return content.length; +}