feat: support anyOf for structured output tools [LET-5615] (#5556)

* base

* works?

* update tests

---------

Co-authored-by: Letta Bot <noreply@letta.com>

letta/llm_api/helpers.py

@@ -17,15 +17,65 @@ from letta.utils import count_tokens, printd
 def _convert_to_structured_output_helper(property: dict) -> dict:
     """Convert a single JSON schema property to structured output format (recursive)"""
 
-    if "type" not in property:
-        raise ValueError(f"Property {property} is missing a type")
-    param_type = property["type"]
+    # Handle anyOf structures
+    if "anyOf" in property and "type" not in property:
+        # Check if this is a simple anyOf that can be flattened to type array
+        types = []
+        has_complex = False
+        for option in property["anyOf"]:
+            if "type" in option:
+                opt_type = option["type"]
+                if opt_type in ["object", "array"]:
+                    has_complex = True
+                    break
+                types.append(opt_type)
+            elif "$ref" in option:
+                # Has unresolved $ref, treat as complex
+                has_complex = True
+                break
 
-    if "description" not in property:
-        # raise ValueError(f"Property {property} is missing a description")
-        param_description = None
-    else:
-        param_description = property["description"]
+        # If it's simple primitives only (string, null, integer, boolean, etc), flatten to type array
+        if not has_complex and types:
+            param_description = property.get("description")
+            property_dict = {"type": types}
+            if param_description is not None:
+                property_dict["description"] = param_description
+            if "default" in property:
+                property_dict["default"] = property["default"]
+            # Preserve other fields like enum, format, etc
+            for key in ["enum", "format", "pattern", "minLength", "maxLength", "minimum", "maximum"]:
+                if key in property:
+                    property_dict[key] = property[key]
+            return property_dict
+
+        # Otherwise, preserve anyOf and recursively process each option
+        property_dict = {"anyOf": [_convert_to_structured_output_helper(opt) for opt in property["anyOf"]]}
+        if "description" in property:
+            property_dict["description"] = property["description"]
+        if "default" in property:
+            property_dict["default"] = property["default"]
+        if "title" in property:
+            property_dict["title"] = property["title"]
+        return property_dict
+
+    if "type" not in property:
+        raise ValueError(f"Property {property} is missing a type and doesn't have anyOf")
+
+    param_type = property["type"]
+    param_description = property.get("description")
+
+    # Handle type arrays (e.g., ["string", "null"])
+    if isinstance(param_type, list):
+        property_dict = {"type": param_type}
+        if param_description is not None:
+            property_dict["description"] = param_description
+        if "default" in property:
+            property_dict["default"] = property["default"]
+        # Preserve other fields
+        for key in ["enum", "format", "pattern", "minLength", "maxLength", "minimum", "maximum", "title"]:
+            if key in property:
+                property_dict[key] = property[key]
+        return property_dict
 
     if param_type == "object":
         if "properties" not in property:
@@ -39,6 +89,8 @@ def _convert_to_structured_output_helper(property: dict) -> dict:
         }
         if param_description is not None:
             property_dict["description"] = param_description
+        if "title" in property:
+            property_dict["title"] = property["title"]
         return property_dict
 
     elif param_type == "array":
@@ -51,6 +103,8 @@ def _convert_to_structured_output_helper(property: dict) -> dict:
         }
         if param_description is not None:
             property_dict["description"] = param_description
+        if "title" in property:
+            property_dict["title"] = property["title"]
         return property_dict
 
     else:
@@ -59,6 +113,10 @@ def _convert_to_structured_output_helper(property: dict) -> dict:
         }
         if param_description is not None:
             property_dict["description"] = param_description
+        # Preserve other fields
+        for key in ["enum", "format", "pattern", "minLength", "maxLength", "minimum", "maximum", "default", "title"]:
+            if key in property:
+                property_dict[key] = property[key]
         return property_dict
 
 
@@ -66,6 +124,14 @@ def convert_to_structured_output(openai_function: dict, allow_optional: bool = F
     """Convert function call objects to structured output objects.
 
     See: https://platform.openai.com/docs/guides/structured-outputs/supported-schemas
+
+    Supports:
+    - Simple type arrays: type: ["string", "null"]
+    - anyOf with primitives (flattened to type array)
+    - anyOf with complex objects (preserved as anyOf)
+    - Nested structures with recursion
+
+    For OpenAI strict mode, optional fields (not in required) must have explicit default values.
     """
     description = openai_function.get("description", "")
 
@@ -82,57 +148,19 @@ def convert_to_structured_output(openai_function: dict, allow_optional: bool = F
     }
 
     for param, details in openai_function["parameters"]["properties"].items():
-        param_type = details["type"]
-        param_description = details.get("description", "")
-
-        if param_type == "object":
-            if "properties" not in details:
-                raise ValueError(f"Property {param} of type object is missing 'properties'")
-            structured_output["parameters"]["properties"][param] = {
-                "type": "object",
-                "description": param_description,
-                "properties": {k: _convert_to_structured_output_helper(v) for k, v in details["properties"].items()},
-                "additionalProperties": False,
-                "required": list(details["properties"].keys()),
-            }
-
-        elif param_type == "array":
-            items_schema = details.get("items")
-            prefix_items_schema = details.get("prefixItems")
-
-            if prefix_items_schema:
-                # assume fixed-length tuple — safe fallback to use first type for items
-                fallback_item = prefix_items_schema[0] if isinstance(prefix_items_schema, list) else prefix_items_schema
-                structured_output["parameters"]["properties"][param] = {
-                    "type": "array",
-                    "description": param_description,
-                    "prefixItems": [_convert_to_structured_output_helper(item) for item in prefix_items_schema],
-                    "items": _convert_to_structured_output_helper(fallback_item),
-                    "minItems": details.get("minItems", len(prefix_items_schema)),
-                    "maxItems": details.get("maxItems", len(prefix_items_schema)),
-                }
-            elif items_schema:
-                structured_output["parameters"]["properties"][param] = {
-                    "type": "array",
-                    "description": param_description,
-                    "items": _convert_to_structured_output_helper(items_schema),
-                }
-            else:
-                raise ValueError(f"Array param '{param}' is missing both 'items' and 'prefixItems'")
-
-        else:
-            prop = {
-                "type": param_type,
-                "description": param_description,
-            }
-            if "enum" in details:
-                prop["enum"] = details["enum"]
-            structured_output["parameters"]["properties"][param] = prop
+        # Use the helper for all parameter types - it now handles anyOf, type arrays, objects, arrays, etc.
+        structured_output["parameters"]["properties"][param] = _convert_to_structured_output_helper(details)
 
+    # Determine which fields are required
+    # For OpenAI strict mode, ALL fields must be in the required array
+    # This is a requirement for strict: true schemas
     if not allow_optional:
+        # All fields are required for strict mode
         structured_output["parameters"]["required"] = list(structured_output["parameters"]["properties"].keys())
     else:
-        raise NotImplementedError("Optional parameter handling is not implemented.")
+        # Use the input's required list if provided, otherwise empty
+        structured_output["parameters"]["required"] = openai_function["parameters"].get("required", [])
+
     return structured_output