diff --git a/letta/server/rest_api/routers/v1/agents.py b/letta/server/rest_api/routers/v1/agents.py
index 3d0a32f1..d170c423 100644
--- a/letta/server/rest_api/routers/v1/agents.py
+++ b/letta/server/rest_api/routers/v1/agents.py
@@ -2419,7 +2419,11 @@ async def summarize_messages(
 
         # If mode changed from agent's original settings and prompt not explicitly set in request, then use the default prompt for the new mode
         # Ex: previously was sliding_window, now is all, so we need to use the default prompt for all mode
-        if "mode" in changed_fields and agent.compaction_settings.mode != request.compaction_settings.mode:
+        if (
+            "mode" in changed_fields
+            and "prompt" not in changed_fields
+            and agent.compaction_settings.mode != request.compaction_settings.mode
+        ):
             from letta.services.summarizer.summarizer_config import get_default_prompt_for_mode
 
             compaction_settings.prompt = get_default_prompt_for_mode(compaction_settings.mode)
diff --git a/letta/server/rest_api/routers/v1/conversations.py b/letta/server/rest_api/routers/v1/conversations.py
index 9c4067ec..a3258e7c 100644
--- a/letta/server/rest_api/routers/v1/conversations.py
+++ b/letta/server/rest_api/routers/v1/conversations.py
@@ -712,7 +712,11 @@ async def compact_conversation(
 
         # If mode changed from agent's original settings and prompt not explicitly set in request, then use the default prompt for the new mode
         # Ex: previously was sliding_window, now is all, so we need to use the default prompt for all mode
-        if "mode" in changed_fields and agent.compaction_settings.mode != request.compaction_settings.mode:
+        if (
+            "mode" in changed_fields
+            and "prompt" not in changed_fields
+            and agent.compaction_settings.mode != request.compaction_settings.mode
+        ):
             from letta.services.summarizer.summarizer_config import get_default_prompt_for_mode
 
             compaction_settings.prompt = get_default_prompt_for_mode(compaction_settings.mode)
diff --git a/letta/services/agent_manager.py b/letta/services/agent_manager.py
index 7a51947e..2f9769e6 100644
--- a/letta/services/agent_manager.py
+++ b/letta/services/agent_manager.py
@@ -791,14 +791,18 @@ class AgentManager:
             # Upsert compaction_settings: merge incoming partial update with existing settings
             if agent_update.compaction_settings is not None:
                 # If mode changed, update the prompt to the default for the new mode
-                if agent.compaction_settings is not None and agent_update.compaction_settings.mode != agent.compaction_settings.mode:
+                changed_fields = agent_update.compaction_settings.model_fields_set
+                if (
+                    agent.compaction_settings is not None
+                    and "mode" in changed_fields
+                    and agent_update.compaction_settings.mode != agent.compaction_settings.mode
+                ):
                     from letta.services.summarizer.summarizer_config import get_default_prompt_for_mode
 
                     agent_update.compaction_settings.prompt = get_default_prompt_for_mode(agent_update.compaction_settings.mode)
 
                 # Fill in unchanged fields from existing settings
                 if agent.compaction_settings is not None:
-                    changed_fields = agent_update.compaction_settings.model_fields_set
                     for field in agent.compaction_settings.model_fields:
                         if field not in changed_fields:
                             setattr(agent_update.compaction_settings, field, getattr(agent.compaction_settings, field))