fix: gate skills reminders in headless sessions (#987)

2026-02-16 19:08:55 -08:00
parent 4cd1c5e22e
commit e074d0bfb7
3 changed files with 203 additions and 46 deletions
--- a/src/cli/App.tsx
+++ b/src/cli/App.tsx
@@ -8161,37 +8161,49 @@ ${SYSTEM_REMINDER_CLOSE}
      pushReminder(sessionContextReminder);

      // Inject available skills as system-reminder (LET-7353)
-      // Lazy-discover on first message, reinject after compaction
+      // Discover each turn so on-disk skill changes can trigger reinjection.
      {
-        if (!discoveredSkillsRef.current) {
-          try {
-            const { discoverSkills: discover, SKILLS_DIR: defaultDir } =
-              await import("../agent/skills");
-            const { getSkillsDirectory, getNoSkills } = await import(
-              "../agent/context"
-            );
-            const skillsDir =
-              getSkillsDirectory() || join(process.cwd(), defaultDir);
-            const { skills } = await discover(skillsDir, agentId, {
-              skipBundled: getNoSkills(),
-            });
-            discoveredSkillsRef.current = skills;
-          } catch {
-            discoveredSkillsRef.current = [];
-          }
+        const {
+          discoverSkills: discover,
+          SKILLS_DIR: defaultDir,
+          formatSkillsAsSystemReminder,
+        } = await import("../agent/skills");
+        const { getSkillsDirectory, getNoSkills } = await import(
+          "../agent/context"
+        );
+
+        const previousSkillsReminder = discoveredSkillsRef.current
+          ? formatSkillsAsSystemReminder(discoveredSkillsRef.current)
+          : null;
+
+        let latestSkills = discoveredSkillsRef.current ?? [];
+        try {
+          const skillsDir =
+            getSkillsDirectory() || join(process.cwd(), defaultDir);
+          const { skills } = await discover(skillsDir, agentId, {
+            skipBundled: getNoSkills(),
+          });
+          latestSkills = skills;
+        } catch {
+          // Keep the previous snapshot when discovery fails.
+        }
+
+        discoveredSkillsRef.current = latestSkills;
+        const latestSkillsReminder = formatSkillsAsSystemReminder(
+          discoveredSkillsRef.current,
+        );
+        if (
+          previousSkillsReminder !== null &&
+          previousSkillsReminder !== latestSkillsReminder
+        ) {
+          contextTrackerRef.current.pendingSkillsReinject = true;
        }

        const needsSkillsReinject =
          contextTrackerRef.current.pendingSkillsReinject;
        if (!hasInjectedSkillsRef.current || needsSkillsReinject) {
-          const { formatSkillsAsSystemReminder } = await import(
-            "../agent/skills"
-          );
-          const skillsReminder = formatSkillsAsSystemReminder(
-            discoveredSkillsRef.current,
-          );
-          if (skillsReminder) {
-            pushReminder(skillsReminder);
+          if (latestSkillsReminder) {
+            pushReminder(latestSkillsReminder);
          }
          hasInjectedSkillsRef.current = true;
          contextTrackerRef.current.pendingSkillsReinject = false;
--- a/src/headless.ts
+++ b/src/headless.ts
@@ -78,6 +78,41 @@ const LLM_API_ERROR_MAX_RETRIES = 3;
 const CONVERSATION_BUSY_MAX_RETRIES = 1; // Only retry once, fail on 2nd 409
 const CONVERSATION_BUSY_RETRY_DELAY_MS = 2500; // 2.5 seconds

+export function prependSkillsReminderToContent(
+  content: MessageCreate["content"],
+  skillsReminder: string,
+): MessageCreate["content"] {
+  if (!skillsReminder) {
+    return content;
+  }
+
+  if (typeof content === "string") {
+    return `${skillsReminder}\n\n${content}`;
+  }
+
+  if (Array.isArray(content)) {
+    return [
+      {
+        type: "text",
+        text: `${skillsReminder}\n\n`,
+      },
+      ...content,
+    ] as MessageCreate["content"];
+  }
+
+  return content;
+}
+
+export function shouldReinjectSkillsAfterCompaction(lines: Line[]): boolean {
+  return lines.some(
+    (line) =>
+      line.kind === "event" &&
+      line.eventType === "compaction" &&
+      line.phase === "finished" &&
+      (line.summary !== undefined || line.stats !== undefined),
+  );
+}
+
 export async function handleHeadlessCommand(
  argv: string[],
  model?: string,
@@ -2013,6 +2048,12 @@ async function runBidirectionalMode(

  // Track current operation for interrupt support
  let currentAbortController: AbortController | null = null;
+  // Skills reminder lifecycle in bidirectional mode:
+  // - Inject once on first user turn
+  // - Reinject only after compaction completion or skills diff
+  let hasInjectedSkillsReminder = false;
+  let pendingSkillsReinject = false;
+  let cachedSkillsReminder: string | null = null;

  // Resolve pending approvals for this conversation before retrying user input.
  const resolveAllPendingApprovals = async () => {
@@ -2278,7 +2319,7 @@ async function runBidirectionalMode(

    let message: {
      type: string;
-      message?: { role: string; content: string };
+      message?: { role: string; content: MessageCreate["content"] };
      request_id?: string;
      request?: { subtype: string };
      session_id?: string;
@@ -2417,7 +2458,7 @@ async function runBidirectionalMode(
    }

    // Handle user messages
-    if (message.type === "user" && message.message?.content) {
+    if (message.type === "user" && message.message?.content !== undefined) {
      const userContent = message.message.content;

      // Create abort controller for this operation
@@ -2431,27 +2472,45 @@ async function runBidirectionalMode(
        let sawStreamError = false; // Track if we emitted an error during streaming
        let preStreamTransientRetries = 0;

-        // Inject available skills as system-reminder for bidirectional mode (LET-7353)
+        // Inject available skills as system-reminder for bidirectional mode (LET-7353).
+        // Discover each turn so skill file changes are naturally picked up.
        let enrichedContent = userContent;
-        if (typeof enrichedContent === "string") {
-          try {
-            const {
-              discoverSkills: discover,
-              SKILLS_DIR: defaultDir,
-              formatSkillsAsSystemReminder,
-            } = await import("./agent/skills");
-            const { getSkillsDirectory } = await import("./agent/context");
-            const { join } = await import("node:path");
-            const skillsDir =
-              getSkillsDirectory() || join(process.cwd(), defaultDir);
-            const { skills } = await discover(skillsDir, agent.id);
-            const skillsReminder = formatSkillsAsSystemReminder(skills);
-            if (skillsReminder) {
-              enrichedContent = `${skillsReminder}\n\n${enrichedContent}`;
-            }
-          } catch {
-            // Skills discovery failed, skip
+        try {
+          const {
+            discoverSkills: discover,
+            SKILLS_DIR: defaultDir,
+            formatSkillsAsSystemReminder,
+          } = await import("./agent/skills");
+          const { getSkillsDirectory } = await import("./agent/context");
+          const { join } = await import("node:path");
+          const skillsDir =
+            getSkillsDirectory() || join(process.cwd(), defaultDir);
+          const { skills } = await discover(skillsDir, agent.id);
+          const latestSkillsReminder = formatSkillsAsSystemReminder(skills);
+
+          // Trigger reinjection when the available-skills block changed on disk.
+          if (
+            cachedSkillsReminder !== null &&
+            latestSkillsReminder !== cachedSkillsReminder
+          ) {
+            pendingSkillsReinject = true;
          }
+          cachedSkillsReminder = latestSkillsReminder;
+
+          const shouldInjectSkillsReminder =
+            !hasInjectedSkillsReminder || pendingSkillsReinject;
+          if (shouldInjectSkillsReminder && latestSkillsReminder) {
+            enrichedContent = prependSkillsReminderToContent(
+              enrichedContent,
+              latestSkillsReminder,
+            );
+          }
+          if (shouldInjectSkillsReminder) {
+            hasInjectedSkillsReminder = true;
+            pendingSkillsReinject = false;
+          }
+        } catch {
+          // Skills discovery failed, skip
        }

        // Initial input is the user message
@@ -2792,6 +2851,9 @@ async function runBidirectionalMode(
        // Emit result
        const durationMs = performance.now() - startTime;
        const lines = toLines(buffers);
+        if (shouldReinjectSkillsAfterCompaction(lines)) {
+          pendingSkillsReinject = true;
+        }
        const reversed = [...lines].reverse();
        const lastAssistant = reversed.find(
          (line) =>
--- a/src/tests/headless/skills-reminder.test.ts
+++ b/src/tests/headless/skills-reminder.test.ts
@@ -0,0 +1,83 @@
+import { describe, expect, test } from "bun:test";
+import type { MessageCreate } from "@letta-ai/letta-client/resources/agents/agents";
+import type { Line } from "../../cli/helpers/accumulator";
+import {
+  prependSkillsReminderToContent,
+  shouldReinjectSkillsAfterCompaction,
+} from "../../headless";
+
+describe("headless skills reminder helpers", () => {
+  test("prepends reminder to string user content", () => {
+    const result = prependSkillsReminderToContent(
+      "hello",
+      "<skills>demo</skills>",
+    );
+    expect(result).toBe("<skills>demo</skills>\n\nhello");
+  });
+
+  test("prepends reminder as a text part for multimodal user content", () => {
+    const multimodal = [
+      { type: "text", text: "what is in this image?" },
+      {
+        type: "image",
+        source: { type: "base64", media_type: "image/png", data: "abc" },
+      },
+    ] as unknown as Exclude<MessageCreate["content"], string>;
+
+    const result = prependSkillsReminderToContent(
+      multimodal as MessageCreate["content"],
+      "<skills>demo</skills>",
+    );
+
+    expect(Array.isArray(result)).toBe(true);
+    if (!Array.isArray(result)) return;
+    expect(result[0]).toEqual({
+      type: "text",
+      text: "<skills>demo</skills>\n\n",
+    });
+    expect(result[1]).toEqual(multimodal[0]);
+    expect(result[2]).toEqual(multimodal[1]);
+  });
+
+  test("does not reinject on compaction start event", () => {
+    const lines: Line[] = [
+      {
+        kind: "event",
+        id: "evt-1",
+        eventType: "compaction",
+        eventData: {},
+        phase: "running",
+      },
+    ];
+    expect(shouldReinjectSkillsAfterCompaction(lines)).toBe(false);
+  });
+
+  test("reinjection triggers after compaction completion", () => {
+    const withSummary: Line[] = [
+      {
+        kind: "event",
+        id: "evt-2",
+        eventType: "compaction",
+        eventData: {},
+        phase: "finished",
+        summary: "Compacted old messages",
+      },
+    ];
+    expect(shouldReinjectSkillsAfterCompaction(withSummary)).toBe(true);
+
+    const withStatsOnly: Line[] = [
+      {
+        kind: "event",
+        id: "evt-3",
+        eventType: "compaction",
+        eventData: {},
+        phase: "finished",
+        stats: {
+          contextTokensBefore: 12000,
+          contextTokensAfter: 7000,
+        },
+      },
+    ];
+    expect(shouldReinjectSkillsAfterCompaction(withStatsOnly)).toBe(true);
+  });
+});