fix: sanitize null bytes to prevent PostgreSQL CharacterNotInRepertoireError (#8015)
This fixes the asyncpg.exceptions.CharacterNotInRepertoireError that occurs when tool returns contain null bytes (0x00), which PostgreSQL TEXT columns reject in UTF-8 encoding. Changes: - Add sanitize_null_bytes() function to recursively remove null bytes from strings - Update json_dumps() to sanitize data before serialization - Apply sanitization in converters.py for tool_calls, tool_returns, approvals, and message_content - Add comprehensive unit tests Fixes #8014 🤖 Generated with [Letta Code](https://letta.com) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Letta <noreply@letta.com> Co-authored-by: Kian Jones <11655409+kianjones9@users.noreply.github.com>
This commit is contained in:
committed by
Caren Thomas
parent
d5decc2a27
commit
dbdd1a40e4
@@ -678,3 +678,102 @@ def test_sdk_version_check():
|
||||
assert is_1_0_sdk_version(HeaderParams(sdk_version="v1.0.0-alpha.7"))
|
||||
assert is_1_0_sdk_version(HeaderParams(sdk_version="v1.0.0a7"))
|
||||
assert is_1_0_sdk_version(HeaderParams(sdk_version="v2.0.0"))
|
||||
|
||||
|
||||
# ---------------------- sanitize_null_bytes TESTS ---------------------- #
|
||||
|
||||
|
||||
def test_sanitize_null_bytes_string():
|
||||
"""Test that null bytes are removed from strings"""
|
||||
from letta.helpers.json_helpers import sanitize_null_bytes
|
||||
|
||||
# Test basic null byte removal
|
||||
assert sanitize_null_bytes("hello\x00world") == "helloworld"
|
||||
|
||||
# Test multiple null bytes
|
||||
assert sanitize_null_bytes("a\x00b\x00c") == "abc"
|
||||
|
||||
# Test null byte at beginning
|
||||
assert sanitize_null_bytes("\x00hello") == "hello"
|
||||
|
||||
# Test null byte at end
|
||||
assert sanitize_null_bytes("hello\x00") == "hello"
|
||||
|
||||
# Test string without null bytes
|
||||
assert sanitize_null_bytes("hello world") == "hello world"
|
||||
|
||||
# Test empty string
|
||||
assert sanitize_null_bytes("") == ""
|
||||
|
||||
|
||||
def test_sanitize_null_bytes_dict():
|
||||
"""Test that null bytes are removed from dictionary values"""
|
||||
from letta.helpers.json_helpers import sanitize_null_bytes
|
||||
|
||||
# Test nested dict with null bytes
|
||||
result = sanitize_null_bytes({
|
||||
"key1": "value\x00with\x00nulls",
|
||||
"key2": {"nested": "also\x00null"},
|
||||
"key3": 123, # non-string should be unchanged
|
||||
})
|
||||
assert result == {
|
||||
"key1": "valuewithnulls",
|
||||
"key2": {"nested": "alsonull"},
|
||||
"key3": 123,
|
||||
}
|
||||
|
||||
|
||||
def test_sanitize_null_bytes_list():
|
||||
"""Test that null bytes are removed from list elements"""
|
||||
from letta.helpers.json_helpers import sanitize_null_bytes
|
||||
|
||||
result = sanitize_null_bytes(["hello\x00world", "no nulls", {"nested\x00key": "value\x00"}])
|
||||
assert result == ["helloworld", "no nulls", {"nestedkey": "value"}]
|
||||
|
||||
|
||||
def test_sanitize_null_bytes_tuple():
|
||||
"""Test that null bytes are removed from tuple elements"""
|
||||
from letta.helpers.json_helpers import sanitize_null_bytes
|
||||
|
||||
result = sanitize_null_bytes(("hello\x00world", "no nulls"))
|
||||
assert result == ("helloworld", "no nulls")
|
||||
|
||||
|
||||
def test_sanitize_null_bytes_preserves_other_types():
|
||||
"""Test that non-string types are preserved unchanged"""
|
||||
from letta.helpers.json_helpers import sanitize_null_bytes
|
||||
|
||||
assert sanitize_null_bytes(123) == 123
|
||||
assert sanitize_null_bytes(3.14) == 3.14
|
||||
assert sanitize_null_bytes(True) is True
|
||||
assert sanitize_null_bytes(False) is False
|
||||
assert sanitize_null_bytes(None) is None
|
||||
|
||||
|
||||
def test_json_dumps_sanitizes_null_bytes():
|
||||
"""Test that json_dumps sanitizes null bytes before serialization"""
|
||||
from letta.helpers.json_helpers import json_dumps
|
||||
|
||||
# Test that null bytes are removed from the output
|
||||
result = json_dumps({"message": "hello\x00world"})
|
||||
assert "\x00" not in result
|
||||
assert "helloworld" in result
|
||||
|
||||
|
||||
def test_json_dumps_with_complex_nested_null_bytes():
|
||||
"""Test that json_dumps handles complex nested structures with null bytes"""
|
||||
from letta.helpers.json_helpers import json_dumps
|
||||
|
||||
data = {
|
||||
"tool_return": {
|
||||
"status": "success",
|
||||
"func_response": "Binary\x00data\x00here",
|
||||
},
|
||||
"content": [
|
||||
{"type": "text", "text": "Message\x00with\x00nulls"},
|
||||
],
|
||||
}
|
||||
result = json_dumps(data)
|
||||
assert "\x00" not in result
|
||||
assert "Binarydatahere" in result
|
||||
assert "Messagewithnulls" in result
|
||||
|
||||
Reference in New Issue
Block a user