Fimeg/letta-server
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: prompt tune rethink with current default tools without anthropic view / str edit (#1709)

Browse Source 
Co-authored-by: cpacker <packercharles@gmail.com>
Co-authored-by: Caren Thomas <carenthomas@gmail.com>
This commit is contained in:
Kevin Lin
2025-04-17 11:53:18 -07:00
committed by GitHub
parent bf2c37a18e
commit 1cd26d0199
9 changed files with 241 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
letta/agent.py
View File

@@ -36,7 +36,7 @@ from letta.log import get_logger
from letta.memory import summarize_messages
from letta.orm import User
from letta.orm.enums import ToolType
from letta.schemas.agent import AgentState, AgentStepResponse, UpdateAgent
from letta.schemas.agent import AgentState, AgentStepResponse, UpdateAgent, get_prompt_template_for_agent_type
from letta.schemas.block import BlockUpdate
from letta.schemas.embedding_config import EmbeddingConfig
from letta.schemas.enums import MessageRole
@@ -52,11 +52,7 @@ from letta.schemas.tool_rule import TerminalToolRule
from letta.schemas.usage import LettaUsageStatistics
from letta.services.agent_manager import AgentManager
from letta.services.block_manager import BlockManager
from letta.services.helpers.agent_manager_helper import (
check_supports_structured_output,
compile_memory_metadata_block,
compile_system_message,
)
from letta.services.helpers.agent_manager_helper import check_supports_structured_output, compile_memory_metadata_block
from letta.services.job_manager import JobManager
from letta.services.message_manager import MessageManager
from letta.services.passage_manager import PassageManager
@@ -204,7 +200,8 @@ class Agent(BaseAgent):
# refresh memory from DB (using block ids)
self.agent_state.memory = Memory(
blocks=[self.block_manager.get_block_by_id(block.id, actor=self.user) for block in self.agent_state.memory.get_blocks()]
blocks=[self.block_manager.get_block_by_id(block.id, actor=self.user) for block in self.agent_state.memory.get_blocks()],
prompt_template=get_prompt_template_for_agent_type(self.agent_state.agent_type),
)
# NOTE: don't do this since re-buildin the memory is handled at the start of the step
@@ -306,29 +303,6 @@ class Agent(BaseAgent):
elif step_count is not None and step_count > 0 and len(allowed_tool_names) == 1:
force_tool_call = allowed_tool_names[0]
if force_tool_call == "core_memory_insert":
current_system_message = message_sequence[0]
new_memory = Memory(
blocks=self.agent_state.memory.blocks,
prompt_template=(
"{% for block in blocks %}"
'<{{ block.label }} characters="{{ block.value|length }}/{{ block.limit }}">\n'
"{% for line in block.value.splitlines() %}"
"{{ loop.index0 }}: {{ line }}\n"
"{% endfor %}"
"</{{ block.label }}>"
"{% if not loop.last %}\n{% endif %}"
"{% endfor %}"
),
)
new_system_message_str = compile_system_message(
system_prompt=self.agent_state.system,
in_context_memory=new_memory,
in_context_memory_last_edit=current_system_message.created_at,
previous_message_count=len(message_sequence),
)
message_sequence[0].content = [TextContent(text=new_system_message_str)]
for attempt in range(1, empty_response_retry_limit + 1):
try:
log_telemetry(self.logger, "_get_ai_reply create start")
@@ -834,7 +808,8 @@ class Agent(BaseAgent):
# Step 0: update core memory
# only pulling latest block data if shared memory is being used
current_persisted_memory = Memory(
blocks=[self.block_manager.get_block_by_id(block.id, actor=self.user) for block in self.agent_state.memory.get_blocks()]
blocks=[self.block_manager.get_block_by_id(block.id, actor=self.user) for block in self.agent_state.memory.get_blocks()],
prompt_template=get_prompt_template_for_agent_type(self.agent_state.agent_type),
) # read blocks from DB
self.update_memory_if_changed(current_persisted_memory)

8
letta/constants.py
View File

@@ -56,10 +56,10 @@ BASE_MEMORY_TOOLS = ["core_memory_append", "core_memory_replace"]
BASE_SLEEPTIME_CHAT_TOOLS = ["send_message", "conversation_search", "archival_memory_search"]
# Base memory tools for sleeptime agent
BASE_SLEEPTIME_TOOLS = [
"rethink_memory",
"finish_rethinking_memory",
"view_core_memory_with_line_numbers",
"core_memory_insert",
"memory_replace",
"memory_insert",
"memory_rethink",
"memory_finish_edits",
"archival_memory_insert",
"archival_memory_search",
"conversation_search",

180
letta/functions/function_sets/base.py
View File

@@ -195,40 +195,174 @@ def finish_rethinking_memory(agent_state: "AgentState") -> None: # type: ignore
return None
def view_core_memory_with_line_numbers(agent_state: "AgentState", target_block_label: str) -> None: # type: ignore
## Attempted v2 of sleep-time function set, meant to work better across all types
SNIPPET_LINES: int = 4
# Based off of: https://github.com/anthropics/anthropic-quickstarts/blob/main/computer-use-demo/computer_use_demo/tools/edit.py?ref=musings.yasyf.com#L154
def memory_replace(agent_state: "AgentState", label: str, old_str: str, new_str: Optional[str] = None) -> str: # type: ignore
"""
View the contents of core memory in editor mode with line numbers. Called before `core_memory_insert` to see line numbers of memory block.
The memory_replace command allows you to replace a specific string in a memory block with a new string. This is used for making precise edits.
Args:
target_block_label (str): The name of the block to view.
label (str): Section of the memory to be edited, identified by its label.
old_str (str): The text to replace (must match exactly, including whitespace and indentation).
new_str (Optional[str]): The new text to insert in place of the old text. Omit this argument to delete the old_str.
Returns:
str: The success message
"""
import re
if bool(re.search(r"\nLine \d+: ", old_str)):
raise ValueError(
"old_str contains a line number prefix, which is not allowed. Do not include line numbers when calling memory tools (line numbers are for display purposes only)."
)
if bool(re.search(r"\nLine \d+: ", new_str)):
raise ValueError(
"new_str contains a line number prefix, which is not allowed. Do not include line numbers when calling memory tools (line numbers are for display purposes only)."
)
old_str = str(old_str).expandtabs()
new_str = str(new_str).expandtabs()
current_value = str(agent_state.memory.get_block(label).value).expandtabs()
# Check if old_str is unique in the block
occurences = current_value.count(old_str)
if occurences == 0:
raise ValueError(f"No replacement was performed, old_str `{old_str}` did not appear verbatim in memory block with label `{label}`.")
elif occurences > 1:
content_value_lines = current_value.split("\n")
lines = [idx + 1 for idx, line in enumerate(content_value_lines) if old_str in line]
raise ValueError(
f"No replacement was performed. Multiple occurrences of old_str `{old_str}` in lines {lines}. Please ensure it is unique."
)
# Replace old_str with new_str
new_value = current_value.replace(str(old_str), str(new_str))
# Write the new content to the block
agent_state.memory.update_block_value(label=label, value=new_value)
# Create a snippet of the edited section
SNIPPET_LINES = 3
replacement_line = current_value.split(old_str)[0].count("\n")
start_line = max(0, replacement_line - SNIPPET_LINES)
end_line = replacement_line + SNIPPET_LINES + new_str.count("\n")
snippet = "\n".join(new_value.split("\n")[start_line : end_line + 1])
# Prepare the success message
success_msg = f"The core memory block with label `{label}` has been edited. "
# success_msg += self._make_output(
# snippet, f"a snippet of {path}", start_line + 1
# )
# success_msg += f"A snippet of core memory block `{label}`:\n{snippet}\n"
success_msg += "Review the changes and make sure they are as expected (correct indentation, no duplicate lines, etc). Edit the memory block again if necessary."
# return None
return success_msg
def memory_insert(agent_state: "AgentState", label: str, new_str: str, insert_line: int = -1) -> Optional[str]: # type: ignore
"""
The memory_insert command allows you to insert text at a specific location in a memory block.
Args:
label (str): Section of the memory to be edited, identified by its label.
new_str (str): The text to insert.
insert_line (int): The line number after which to insert the text (0 for beginning of file). Defaults to -1 (end of the file).
Returns:
Optional[str]: None is always returned as this function does not produce a response.
"""
import re
if bool(re.search(r"\nLine \d+: ", new_str)):
raise ValueError(
"new_str contains a line number prefix, which is not allowed. Do not include line numbers when calling memory tools (line numbers are for display purposes only)."
)
current_value = str(agent_state.memory.get_block(label).value).expandtabs()
new_str = str(new_str).expandtabs()
current_value_lines = current_value.split("\n")
n_lines = len(current_value_lines)
# Check if we're in range, from 0 (pre-line), to 1 (first line), to n_lines (last line)
if insert_line < 0 or insert_line > n_lines:
raise ValueError(
f"Invalid `insert_line` parameter: {insert_line}. It should be within the range of lines of the memory block: {[0, n_lines]}, or -1 to append to the end of the memory block."
)
# Insert the new string as a line
new_str_lines = new_str.split("\n")
new_value_lines = current_value_lines[:insert_line] + new_str_lines + current_value_lines[insert_line:]
snippet_lines = (
current_value_lines[max(0, insert_line - SNIPPET_LINES) : insert_line]
+ new_str_lines
+ current_value_lines[insert_line : insert_line + SNIPPET_LINES]
)
# Collate into the new value to update
new_value = "\n".join(new_value_lines)
snippet = "\n".join(snippet_lines)
# Write into the block
agent_state.memory.update_block_value(label=label, value=new_value)
# Prepare the success message
success_msg = f"The core memory block with label `{label}` has been edited. "
# success_msg += self._make_output(
# snippet,
# "a snippet of the edited file",
# max(1, insert_line - SNIPPET_LINES + 1),
# )
# success_msg += f"A snippet of core memory block `{label}`:\n{snippet}\n"
success_msg += "Review the changes and make sure they are as expected (correct indentation, no duplicate lines, etc). Edit the memory block again if necessary."
return success_msg
def memory_rethink(agent_state: "AgentState", label: str, new_memory: str) -> None:
"""
The memory_rethink command allows you to completely rewrite the contents of a memory block. Use this tool to make large sweeping changes (e.g. when you want to condense or reorganize the memory blocks), do NOT use this tool to make small precise edits (e.g. add or remove a line, replace a specific string, etc).
Args:
label (str): The memory block to be rewritten, identified by its label.
new_memory (str): The new memory contents with information integrated from existing memory blocks and the conversation context.
Returns:
None: None is always returned as this function does not produce a response.
"""
return None
import re
if bool(re.search(r"\nLine \d+: ", new_memory)):
raise ValueError(
"new_memory contains a line number prefix, which is not allowed. Do not include line numbers when calling memory tools (line numbers are for display purposes only)."
)
if agent_state.memory.get_block(label) is None:
agent_state.memory.create_block(label=label, value=new_memory)
agent_state.memory.update_block_value(label=label, value=new_memory)
# Prepare the success message
success_msg = f"The core memory block with label `{label}` has been edited. "
# success_msg += self._make_output(
# snippet, f"a snippet of {path}", start_line + 1
# )
# success_msg += f"A snippet of core memory block `{label}`:\n{snippet}\n"
success_msg += "Review the changes and make sure they are as expected (correct indentation, no duplicate lines, etc). Edit the memory block again if necessary."
# return None
return success_msg
def core_memory_insert(agent_state: "AgentState", target_block_label: str, new_memory: str, line_number: Optional[int] = None, replace: bool = False) -> None: # type: ignore
def memory_finish_edits(agent_state: "AgentState") -> None: # type: ignore
"""
Insert new memory content into a core memory block at a specific line number. Call `view_core_memory_with_line_numbers` to see line numbers of the memory block before using this tool.
Args:
target_block_label (str): The name of the block to write to.
new_memory (str): The new memory content to insert.
line_number (Optional[int]): Line number to insert content into, 0 indexed (None for end of file).
replace (bool): Whether to overwrite the content at the specified line number.
Call the memory_finish_edits command when you are finished making edits (integrating all new information) into the memory blocks. This function is called when the agent is done rethinking the memory.
Returns:
None: None is always returned as this function does not produce a response.
Optional[str]: None is always returned as this function does not produce a response.
"""
current_value = str(agent_state.memory.get_block(target_block_label).value)
current_value_list = current_value.split("\n")
if line_number is None:
line_number = len(current_value_list)
if replace:
current_value_list[line_number - 1] = new_memory
else:
current_value_list.insert(line_number, new_memory)
new_value = "\n".join(current_value_list)
agent_state.memory.update_block_value(label=target_block_label, value=new_value)
return None

7
letta/orm/agent.py
View File

@@ -12,7 +12,7 @@ from letta.orm.mixins import OrganizationMixin
from letta.orm.organization import Organization
from letta.orm.sqlalchemy_base import SqlalchemyBase
from letta.schemas.agent import AgentState as PydanticAgentState
from letta.schemas.agent import AgentType
from letta.schemas.agent import AgentType, get_prompt_template_for_agent_type
from letta.schemas.embedding_config import EmbeddingConfig
from letta.schemas.llm_config import LLMConfig
from letta.schemas.memory import Memory
@@ -202,7 +202,10 @@ class Agent(SqlalchemyBase, OrganizationMixin):
"tags": lambda: [t.tag for t in self.tags],
"tools": lambda: self.tools,
"sources": lambda: [s.to_pydantic() for s in self.sources],
"memory": lambda: Memory(blocks=[b.to_pydantic() for b in self.core_memory]),
"memory": lambda: Memory(
blocks=[b.to_pydantic() for b in self.core_memory],
prompt_template=get_prompt_template_for_agent_type(self.agent_type),
),
"identity_ids": lambda: [i.id for i in self.identities],
"multi_agent_group": lambda: self.multi_agent_group,
"tool_exec_environment_variables": lambda: self.tool_exec_environment_variables,

31
letta/prompts/system/sleeptime.txt
View File

@@ -6,21 +6,30 @@ Your core memory unit is held inside the initial system instructions file, and i
Your core memory contains the essential, foundational context for keeping track of your own persona, and the persona of the agent that is conversing with the user.
Your core memory is made up of read-only blocks and read-write blocks.
Read-Only Blocks:
Memory Persona Sub-Block: Stores details about your current persona, guiding how you organize the memory. This helps you understand what aspects of the memory is important.
Access as a source block with the label `memory_persona` when calling `rethink_memory`.
Memory Persona Sub-Block: Stores details about your current persona (the memory management agent), guiding how you organize the memory. This helps you understand what aspects of the memory is important.
Read-Write Blocks:
Persona Sub-Block: Stores details about the assistant's persona, guiding how they behave and respond. This helps them to maintain consistency and personality in their interactions.
Access as a source or target block with the label `persona` when calling `rethink_memory`, `view_core_memory_with_line_numbers`, or `core_memory_insert`.
Access as a target block with the label `persona` when calling your memory editing tools.
Human Sub-Block: Stores key details about the person the assistant is conversing with, allowing for more personalized and friend-like conversation.
Access as a source block or target block with the label `human` when calling `rethink_memory`, `view_core_memory_with_line_numbers`, or `core_memory_insert`.
Any additional blocks that you are given access to are also read-write blocks.
Access as a target block with the label `human` when calling your memory editing tools. Any additional blocks that you are given access to are also read-write blocks.
Memory editing:
You have the ability to make edits to the memory by calling `core_memory_insert` and `rethink_memory`.
You call `view_core_memory_with_line_numbers` to view the line numbers of a memory block, before calling `core_memory_insert`.
You call `core_memory_insert` when there is new information to add or overwrite to the memory. Use the replace flag when you want to perform a targeted edit.
To keep the memory blocks organized and readable, you call `rethink_memory` to reorganize the entire memory block so that it is comprehensive, readable, and up to date.
You continue memory editing until the blocks are organized and readable, and do not contain redundant and outdate information, then call `finish_rethinking_memory`.
If there are no meaningful updates to make to the memory, you call `finish_rethinking_memory` directly.
You have the ability to make edits to the memory memory blocks.
Use your precise tools to make narrow edits, as well as broad tools to make larger comprehensive edits.
To keep the memory blocks organized and readable, you can use your precise tools to make narrow edits (additions, deletions, and replacements), and you can use your `rethink` tool to reorganize the entire memory block at a single time.
You goal is to make sure the memory blocks are comprehensive, readable, and up to date.
When writing to memory blocks, make sure to be precise when referencing dates and times (for example, do not write "today" or "recently", instead write specific dates and times, because "today" and "recently" are relative, and the memory is persisted indefinitely).
Multi-step editing:
You should continue memory editing until the blocks are organized and readable, and do not contain redundant and outdate information, then you can call a tool to finish your edits.
You can chain together multiple precise edits, or use the `rethink` tool to reorganize the entire memory block at a single time.
Skipping memory edits:
If there are no meaningful updates to make to the memory, you call the finish tool directly.
Not every observation warrants a memory edit, be selective in your memory editing, but also aim to have high recall.
Line numbers:
Line numbers are shown to you when viewing the memory blocks to help you make precise edits when needed. The line numbers are for viewing only, do NOT under any circumstances actually include the line numbers when using your memory editing tools, or they will not work properly.

24
letta/prompts/system/sleeptime_doc_ingest.txt
View File

@@ -12,14 +12,24 @@ Persona Sub-Block: Stores details about your persona, guiding how you behave.
Instructions Sub-Block: Stores instructions on how to ingest the document.
Read-Write Blocks:
all other memory blocks correspond to data sources, which you will write to for your task. Access the target block using its label when calling `rethink_memory`.
All other memory blocks correspond to data sources, which you will write to for your task. Access the target block using its label when calling `memory_rethink`.
Memory editing:
You have the ability to make edits to the memory by calling `core_memory_insert` and `rethink_memory`.
You call `view_core_memory_with_line_numbers` to view the line numbers of a memory block, before calling `core_memory_insert`.
You call `core_memory_insert` when there is new information to add or overwrite to the memory. Use the replace flag when you want to perform a targeted edit.
To keep the memory blocks organized and readable, you call `rethink_memory` to reorganize the entire memory block so that it is comprehensive, readable, and up to date.
You continue memory editing until the blocks are organized and readable, and do not contain redundant and outdate information, then call `finish_rethinking_memory`.
If there are no meaningful updates to make to the memory, you call `finish_rethinking_memory` directly.
You have the ability to make edits to the memory blocks.
Use your precise tools to make narrow edits, as well as broad tools to make larger comprehensive edits.
To keep the memory blocks organized and readable, you can use your precise tools to make narrow edits (insertions, deletions, and replacements), and you can use your `memory_rethink` tool to reorganize the entire memory block at a single time.
You goal is to make sure the memory blocks are comprehensive, readable, and up to date.
When writing to memory blocks, make sure to be precise when referencing dates and times (for example, do not write "today" or "recently", instead write specific dates and times, because "today" and "recently" are relative, and the memory is persisted indefinitely).
Multi-step editing:
You should continue memory editing until the blocks are organized and readable, and do not contain redundant and outdate information, then you can call a tool to finish your edits.
You can chain together multiple precise edits, or use the `memory_rethink` tool to reorganize the entire memory block at a single time.
Skipping memory edits:
If there are no meaningful updates to make to the memory, you call the finish tool directly.
Not every observation warrants a memory edit, be selective in your memory editing, but also aim to have high recall.
Line numbers:
Line numbers are shown to you when viewing the memory blocks to help you make precise edits when needed. The line numbers are for viewing only, do NOT under any circumstances actually include the line numbers when using your memory editing tools, or they will not work properly.
You will be sent external context about the interaction, and your goal is to summarize the context and store it in the right memory blocks.

23
letta/schemas/agent.py
View File

@@ -277,3 +277,26 @@ class AgentStepResponse(BaseModel):
class AgentStepState(BaseModel):
step_number: int = Field(..., description="The current step number in the agent loop")
tool_rules_solver: ToolRulesSolver = Field(..., description="The current state of the ToolRulesSolver")
def get_prompt_template_for_agent_type(agent_type: Optional[AgentType] = None):
if agent_type == AgentType.sleeptime_agent:
return (
"{% for block in blocks %}"
'<{{ block.label }} characters="{{ block.value|length }}/{{ block.limit }}">\n'
"# NOTE: Line numbers shown below are to help during editing. Do NOT include line number prefixes in your memory edit tool calls."
"{% for line in block.value.split('\\n') %}"
"Line {{ loop.index }}: {{ line }}\n"
"{% endfor %}"
"</{{ block.label }}>"
"{% if not loop.last %}\n{% endif %}"
"{% endfor %}"
)
return (
"{% for block in blocks %}"
'<{{ block.label }} characters="{{ block.value|length }}/{{ block.limit }}">\n'
"{{ block.value }}\n"
"</{{ block.label }}>"
"{% if not loop.last %}\n{% endif %}"
"{% endfor %}"
)

11
letta/services/agent_manager.py
View File

@@ -30,7 +30,7 @@ from letta.orm.sandbox_config import AgentEnvironmentVariable as AgentEnvironmen
from letta.orm.sqlalchemy_base import AccessType
from letta.orm.sqlite_functions import adapt_array
from letta.schemas.agent import AgentState as PydanticAgentState
from letta.schemas.agent import AgentType, CreateAgent, UpdateAgent
from letta.schemas.agent import AgentType, CreateAgent, UpdateAgent, get_prompt_template_for_agent_type
from letta.schemas.block import Block as PydanticBlock
from letta.schemas.block import BlockUpdate
from letta.schemas.embedding_config import EmbeddingConfig
@@ -44,7 +44,6 @@ from letta.schemas.passage import Passage as PydanticPassage
from letta.schemas.source import Source as PydanticSource
from letta.schemas.tool import Tool as PydanticTool
from letta.schemas.tool_rule import ContinueToolRule as PydanticContinueToolRule
from letta.schemas.tool_rule import ParentToolRule as PydanticParentToolRule
from letta.schemas.tool_rule import TerminalToolRule as PydanticTerminalToolRule
from letta.schemas.tool_rule import ToolRule as PydanticToolRule
from letta.schemas.user import User as PydanticUser
@@ -159,14 +158,11 @@ class AgentManager:
if agent_create.include_base_tool_rules:
# apply default tool rules
for tool_name in tool_names:
if tool_name == "send_message" or tool_name == "send_message_to_agent_async" or tool_name == "finish_rethinking_memory":
if tool_name == "send_message" or tool_name == "send_message_to_agent_async" or tool_name == "memory_finish_edits":
tool_rules.append(PydanticTerminalToolRule(tool_name=tool_name))
elif tool_name in BASE_TOOLS + BASE_MEMORY_TOOLS + BASE_SLEEPTIME_TOOLS:
tool_rules.append(PydanticContinueToolRule(tool_name=tool_name))
if agent_create.agent_type == AgentType.sleeptime_agent:
tool_rules.append(PydanticParentToolRule(tool_name="view_core_memory_with_line_numbers", children=["core_memory_insert"]))
# if custom rules, check tool rules are valid
if agent_create.tool_rules:
check_supports_structured_output(model=agent_create.llm_config.model, tool_rules=agent_create.tool_rules)
@@ -843,7 +839,8 @@ class AgentManager:
# refresh memory from DB (using block ids)
agent_state.memory = Memory(
blocks=[self.block_manager.get_block_by_id(block.id, actor=actor) for block in agent_state.memory.get_blocks()]
blocks=[self.block_manager.get_block_by_id(block.id, actor=actor) for block in agent_state.memory.get_blocks()],
prompt_template=get_prompt_template_for_agent_type(agent_state.agent_type),
)
# NOTE: don't do this since re-buildin the memory is handled at the start of the step

10
tests/integration_test_sleeptime_agent.py
View File

@@ -111,12 +111,12 @@ async def test_sleeptime_group_chat(server, actor):
# 4 Verify sleeptime agent tools
sleeptime_agent = server.agent_manager.get_agent_by_id(agent_id=sleeptime_agent_id, actor=actor)
sleeptime_agent_tools = [tool.name for tool in sleeptime_agent.tools]
assert "rethink_memory" in sleeptime_agent_tools
assert "finish_rethinking_memory" in sleeptime_agent_tools
assert "view_core_memory_with_line_numbers" in sleeptime_agent_tools
assert "core_memory_insert" in sleeptime_agent_tools
assert "memory_rethink" in sleeptime_agent_tools
assert "memory_finish_edits" in sleeptime_agent_tools
assert "memory_replace" in sleeptime_agent_tools
assert "memory_insert" in sleeptime_agent_tools
assert len([rule for rule in sleeptime_agent.tool_rules if rule.type == ToolRuleType.parent_last_tool]) > 0
assert len([rule for rule in sleeptime_agent.tool_rules if rule.type == ToolRuleType.exit_loop]) > 0
# 5. Send messages and verify run ids
message_text = [