feat: Feature/global logging (#790)

memgpt/cli/cli.py

@@ -15,6 +15,7 @@ from enum import Enum
 from llama_index import set_global_service_context
 from llama_index import ServiceContext
 
+from memgpt.log import logger
 from memgpt.interface import CLIInterface as interface  # for printing to terminal
 from memgpt.cli.cli_config import configure
 import memgpt.presets.presets as presets
@@ -306,11 +307,14 @@ def run(
     """
 
     # setup logger
+    # TODO: remove Utils Debug after global logging is complete.
     utils.DEBUG = debug
-    logging.getLogger().setLevel(logging.CRITICAL)
-    if debug:
-        logging.getLogger().setLevel(logging.DEBUG)
+    # TODO: add logging command line options for runtime log level
 
+    if debug:
+        logger.setLevel(logging.DEBUG)
+    else:
+        logger.setLevel(logging.CRITICAL)
     if not MemGPTConfig.exists():
         # if no config, ask about quickstart
         # do you want to do:

memgpt/cli/cli_config.py

@@ -8,6 +8,9 @@ import shutil
 from typing import Annotated
 from enum import Enum
 
+# from global logging configuration
+from memgpt.log import logger
+
 # from memgpt.cli import app
 from memgpt import utils
 

memgpt/config.py

@@ -1,3 +1,4 @@
+from memgpt.log import logger
 import inspect
 import json
 import os
@@ -9,6 +10,7 @@ import memgpt
 import memgpt.utils as utils
 from memgpt.utils import printd, get_schema_diff
 from memgpt.functions.functions import load_all_function_sets
+
 from memgpt.constants import MEMGPT_DIR, LLM_MAX_TOKENS, DEFAULT_HUMAN, DEFAULT_PERSONA, DEFAULT_PRESET
 from memgpt.data_types import AgentState, User, LLMConfig, EmbeddingConfig
 
@@ -227,6 +229,8 @@ class MemGPTConfig:
         else:
             config_path = MemGPTConfig.config_path
 
+        # insure all configuration directories exist
+        cls.create_config_dir()
         if os.path.exists(config_path):
             # read existing config
             config.read(config_path)
@@ -265,11 +269,13 @@ class MemGPTConfig:
                 "memgpt_version": get_field(config, "version", "memgpt_version"),
             }
             config_dict = {k: v for k, v in config_dict.items() if v is not None}
+
             return cls(**config_dict)
 
         # create new config
         anon_clientid = MemGPTConfig.generate_uuid()
         config = cls(anon_clientid=anon_clientid, config_path=config_path)
+        config.create_config_dir()  # create dirs
         config.save()  # save updated config
         return config
 
@@ -331,10 +337,12 @@ class MemGPTConfig:
             self.anon_clientid = self.generate_uuid()
         set_field(config, "client", "anon_clientid", self.anon_clientid)
 
-        if not os.path.exists(MEMGPT_DIR):
-            os.makedirs(MEMGPT_DIR, exist_ok=True)
+        # always make sure all directories are present
+        self.create_config_dir()
+
         with open(self.config_path, "w") as f:
             config.write(f)
+        logger.debug(f"Saved Config:  {self.config_path}")
 
     @staticmethod
     def exists():
@@ -353,6 +361,7 @@ class MemGPTConfig:
             os.makedirs(MEMGPT_DIR, exist_ok=True)
 
         folders = ["personas", "humans", "archival", "agents", "functions", "system_prompts", "presets", "settings"]
+
         for folder in folders:
             if not os.path.exists(os.path.join(MEMGPT_DIR, folder)):
                 os.makedirs(os.path.join(MEMGPT_DIR, folder))

memgpt/constants.py

@@ -1,4 +1,5 @@
 import os
+from logging import CRITICAL, ERROR, WARN, WARNING, INFO, DEBUG, NOTSET
 
 MEMGPT_DIR = os.path.join(os.path.expanduser("~"), ".memgpt")
 
@@ -7,6 +8,20 @@ DEFAULT_PERSONA = "sam_pov"
 DEFAULT_HUMAN = "basic"
 DEFAULT_PRESET = "memgpt_chat"
 
+# Used to isolate MemGPT logger instance from Dependant Libraries logging
+LOGGER_NAME = "MemGPT"
+LOGGER_DEFAULT_LEVEL = CRITICAL
+# Where to store the logs
+LOGGER_DIR = os.path.join(MEMGPT_DIR, "logs")
+# filename of the log
+LOGGER_FILENAME = "MemGPT.log"
+# Number of log files to rotate
+LOGGER_FILE_BACKUP_COUNT = 3
+# Max Log file size in bytes
+LOGGER_MAX_FILE_SIZE = 10485760
+# LOGGER_LOG_LEVEL is use to convert Text to Logging level value for logging mostly for Cli input to setting level
+LOGGER_LOG_LEVELS = {"CRITICAL": CRITICAL, "ERROR": ERROR, "WARN": WARN, "WARNING": WARNING, "INFO": INFO, "DEBUG": DEBUG, "NOTSET": NOTSET}
+
 FIRST_MESSAGE_ATTEMPTS = 10
 
 INITIAL_BOOT_MESSAGE = "Boot sequence complete. Persona activated."

memgpt/log.py

@@ -0,0 +1,42 @@
+import os
+import os.path
+import logging
+from logging.handlers import RotatingFileHandler
+from memgpt.constants import (
+    LOGGER_NAME,
+    LOGGER_DEFAULT_LEVEL,
+    LOGGER_DIR,
+    LOGGER_FILENAME,
+    LOGGER_FILE_BACKUP_COUNT,
+    LOGGER_MAX_FILE_SIZE,
+)
+
+# Checking if log directory exists
+if not os.path.exists(LOGGER_DIR):
+    os.makedirs(LOGGER_DIR, exist_ok=True)
+
+# Create logger for MemGPT
+logger = logging.getLogger(LOGGER_NAME)
+logger.setLevel(LOGGER_DEFAULT_LEVEL)
+
+# create console handler and set level to debug
+console_handler = logging.StreamHandler()
+
+# create rotatating file handler
+file_handler = RotatingFileHandler(
+    os.path.join(LOGGER_DIR, LOGGER_FILENAME), maxBytes=LOGGER_MAX_FILE_SIZE, backupCount=LOGGER_FILE_BACKUP_COUNT
+)
+
+# create formatters
+console_formatter = logging.Formatter("%(name)s - %(levelname)s - %(message)s")  # not datetime
+file_formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+
+# add formatter to console handler
+console_handler.setFormatter(console_formatter)
+
+# add formatter for file handler
+file_handler.setFormatter(file_formatter)
+
+# add ch to logger
+logger.addHandler(console_handler)
+logger.addHandler(file_handler)

memgpt/main.py

@@ -17,6 +17,7 @@ from prettytable import PrettyTable
 
 console = Console()
 
+from memgpt.log import logger
 from memgpt.interface import CLIInterface as interface  # for printing to terminal
 from memgpt.config import MemGPTConfig
 import memgpt.agent as agent

tests/test_log.py

@@ -0,0 +1,16 @@
+import logging
+from memgpt.log import logger
+from memgpt.constants import LOGGER_LOG_LEVELS
+import pytest
+
+
+def test_log_debug():
+    # test setting logging level
+    assert logging.DEBUG == LOGGER_LOG_LEVELS["DEBUG"]
+    logger.setLevel(LOGGER_LOG_LEVELS["DEBUG"])
+    assert logger.isEnabledFor(logging.DEBUG)
+
+    # Assert that the message was logged
+    assert logger.hasHandlers()
+    logger.debug("This is a Debug message")
+    assert 1 == 1