feat: add default context window (#995)

Co-authored-by: Charles Packer <packercharles@gmail.com>

letta/constants.py

@@ -27,6 +27,7 @@ TOOL_CALL_ID_MAX_LEN = 29
 
 # minimum context window size
 MIN_CONTEXT_WINDOW = 4096
+DEFAULT_CONTEXT_WINDOW_SIZE = 32000
 
 # embeddings
 MAX_EMBEDDING_DIM = 4096  # maximum supported embeding size - do NOT change or else DBs will need to be reset

letta/llm_api/google_vertex.py

@@ -1,7 +1,5 @@
 import uuid
-from typing import List, Optional, Tuple
-
-import requests
+from typing import List, Optional
 
 from letta.constants import NON_USER_MSG_PREFIX
 from letta.local_llm.json_parser import clean_json_string_extra_backslash

letta/server/server.py

@@ -1112,6 +1112,8 @@ class SyncServer(Server):
             if context_window_limit > llm_config.context_window:
                 raise ValueError(f"Context window limit ({context_window_limit}) is greater than maximum of ({llm_config.context_window})")
             llm_config.context_window = context_window_limit
+        else:
+            llm_config.context_window = min(llm_config.context_window, constants.DEFAULT_CONTEXT_WINDOW_SIZE)
 
         return llm_config