import json from urllib.parse import urljoin from letta.local_llm.settings.settings import get_completions_settings from letta.local_llm.utils import post_json_auth_request LMSTUDIO_API_CHAT_SUFFIX = "/v1/chat/completions" LMSTUDIO_API_COMPLETIONS_SUFFIX = "/v1/completions" LMSTUDIO_API_CHAT_COMPLETIONS_SUFFIX = "/v1/chat/completions" def get_lmstudio_completion_chatcompletions(endpoint, auth_type, auth_key, model, messages): """ This is the request we need to send { "model": "deepseek-r1-distill-qwen-7b", "messages": [ { "role": "system", "content": "Always answer in rhymes. Today is Thursday" }, { "role": "user", "content": "What day is it today?" }, { "role": "user", "content": "What day is it today?" }], "temperature": 0.7, "max_tokens": -1, "stream": false """ from letta.utils import printd URI = endpoint + LMSTUDIO_API_CHAT_COMPLETIONS_SUFFIX request = {"model": model, "messages": messages} response = post_json_auth_request(uri=URI, json_payload=request, auth_type=auth_type, auth_key=auth_key) # Get the reasoning from the model if response.status_code == 200: result_full = response.json() result_reasoning = result_full["choices"][0]["message"].get("reasoning_content") result = result_full["choices"][0]["message"]["content"] usage = result_full["usage"] # See if result is json try: function_call = json.loads(result) if "function" in function_call and "params" in function_call: return result, usage, result_reasoning else: print("Did not get json on without json constraint, attempting with json decoding") except Exception as e: print(f"Did not get json on without json constraint, attempting with json decoding: {e}") request["messages"].append({"role": "assistant", "content": result_reasoning}) request["messages"].append({"role": "user", "content": ""}) # last message must be user # Now run with json decoding to get the function request["response_format"] = { "type": "json_schema", "json_schema": { "name": "function_call", "strict": "true", "schema": { "type": "object", "properties": {"function": {"type": "string"}, "params": {"type": "object"}}, "required": ["function", "params"], }, }, } response = post_json_auth_request(uri=URI, json_payload=request, auth_type=auth_type, auth_key=auth_key) if response.status_code == 200: result_full = response.json() printd(f"JSON API response:\n{result_full}") result = result_full["choices"][0]["message"]["content"] # add usage with previous call, merge with prev usage for key, value in result_full["usage"].items(): usage[key] += value return result, usage, result_reasoning def get_lmstudio_completion(endpoint, auth_type, auth_key, prompt, context_window, api="completions"): """Based on the example for using LM Studio as a backend from https://github.com/lmstudio-ai/examples/tree/main/Hello%2C%20world%20-%20OpenAI%20python%20client""" from letta.utils import printd # Approximate token count: bytes / 4 prompt_tokens = len(prompt.encode("utf-8")) // 4 if prompt_tokens > context_window: raise Exception(f"Request exceeds maximum context length ({prompt_tokens} > {context_window} tokens)") settings = get_completions_settings() settings.update( { "input_prefix": "", "input_suffix": "", # This controls how LM studio handles context overflow # In Letta we handle this ourselves, so this should be disabled # "context_overflow_policy": 0, # "lmstudio": {"context_overflow_policy": 0}, # 0 = stop at limit # "lmstudio": {"context_overflow_policy": "stopAtLimit"}, # https://github.com/letta-ai/letta/issues/1782 "stream": False, "model": "local model", } ) # Uses the ChatCompletions API style # Seems to work better, probably because it's applying some extra settings under-the-hood? if api == "chat": URI = urljoin(endpoint.strip("/") + "/", LMSTUDIO_API_CHAT_SUFFIX.strip("/")) # Settings for the generation, includes the prompt + stop tokens, max length, etc request = settings request["max_tokens"] = context_window # Put the entire completion string inside the first message message_structure = [{"role": "user", "content": prompt}] request["messages"] = message_structure # Uses basic string completions (string in, string out) # Does not work as well as ChatCompletions for some reason elif api == "completions": URI = urljoin(endpoint.strip("/") + "/", LMSTUDIO_API_COMPLETIONS_SUFFIX.strip("/")) # Settings for the generation, includes the prompt + stop tokens, max length, etc request = settings request["max_tokens"] = context_window # Standard completions format, formatted string goes in prompt request["prompt"] = prompt else: raise ValueError(api) if not endpoint.startswith(("http://", "https://")): raise ValueError(f"Provided OPENAI_API_BASE value ({endpoint}) must begin with http:// or https://") try: response = post_json_auth_request(uri=URI, json_payload=request, auth_type=auth_type, auth_key=auth_key) if response.status_code == 200: result_full = response.json() printd(f"JSON API response:\n{result_full}") if api == "chat": result = result_full["choices"][0]["message"]["content"] usage = result_full.get("usage", None) elif api == "completions": result = result_full["choices"][0]["text"] usage = result_full.get("usage", None) elif api == "chat/completions": result = result_full["choices"][0]["content"] result_full["choices"][0]["reasoning_content"] usage = result_full.get("usage", None) else: # Example error: msg={"error":"Context length exceeded. Tokens in context: 8000, Context length: 8000"} if "context length" in str(response.text).lower(): # "exceeds context length" is what appears in the LM Studio error message # raise an alternate exception that matches OpenAI's message, which is "maximum context length" raise Exception(f"Request exceeds maximum context length (code={response.status_code}, msg={response.text}, URI={URI})") else: raise Exception( f"API call got non-200 response code (code={response.status_code}, msg={response.text}) for address: {URI}." + f" Make sure that the LM Studio local inference server is running and reachable at {URI}." ) except: # TODO handle gracefully raise # Pass usage statistics back to main thread # These are used to compute memory warning messages completion_tokens = usage.get("completion_tokens", None) if usage is not None else None total_tokens = prompt_tokens + completion_tokens if completion_tokens is not None else None usage = { "prompt_tokens": prompt_tokens, # can grab from usage dict, but it's usually wrong (set to 0) "completion_tokens": completion_tokens, "total_tokens": total_tokens, } return result, usage