From 6cafa9ae97374985fc3cd68728c47a97476715fb Mon Sep 17 00:00:00 2001 From: Max Blackmer Date: Mon, 18 Dec 2023 16:51:23 -0500 Subject: [PATCH 1/5] [#319] Global Logging Configuration with directory fixes at config load. --- memgpt/cli/cli.py | 10 ++-- memgpt/cli/cli_config.py | 3 +- memgpt/config.py | 100 +++++++++++++++++++++++++++++++++++++-- memgpt/constants.py | 2 + memgpt/main.py | 1 + memgpt/memgptlog.py | 12 +++++ memgpt/utils.py | 7 +++ 7 files changed, 126 insertions(+), 9 deletions(-) create mode 100644 memgpt/memgptlog.py diff --git a/memgpt/cli/cli.py b/memgpt/cli/cli.py index 71fea458..7a705cbb 100644 --- a/memgpt/cli/cli.py +++ b/memgpt/cli/cli.py @@ -8,6 +8,7 @@ import questionary from llama_index import set_global_service_context from llama_index import ServiceContext +from memgpt.memgptlog import logger from memgpt.interface import CLIInterface as interface # for printing to terminal from memgpt.cli.cli_config import configure import memgpt.presets.presets as presets @@ -50,11 +51,14 @@ def run( """ # setup logger + #TODO: remove Utils Debug after global logging is complete. utils.DEBUG = debug - logging.getLogger().setLevel(logging.CRITICAL) - if debug: - logging.getLogger().setLevel(logging.DEBUG) + # TODO: add logging command line options for runtime log level + if debug: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.CRITICAL) if not MemGPTConfig.exists(): # if no config, run configure if yes: # use defaults diff --git a/memgpt/cli/cli_config.py b/memgpt/cli/cli_config.py index 787d6bb5..a30612ce 100644 --- a/memgpt/cli/cli_config.py +++ b/memgpt/cli/cli_config.py @@ -4,7 +4,8 @@ from prettytable import PrettyTable import typer import os import shutil - +# from global logging configuration +from memgpt.memgptlog import logger # from memgpt.cli import app from memgpt import utils diff --git a/memgpt/config.py b/memgpt/config.py index 63c5fc1c..4b40747c 100644 --- a/memgpt/config.py +++ b/memgpt/config.py @@ -1,3 +1,6 @@ +import logging +import logging.config +from memgpt.memgptlog import logger,reload_logger import inspect import json import os @@ -7,10 +10,12 @@ import configparser import memgpt import memgpt.utils as utils -from memgpt.constants import MEMGPT_DIR, LLM_MAX_TOKENS, DEFAULT_HUMAN, DEFAULT_PERSONA +from memgpt.constants import MEMGPT_DIR, LLM_MAX_TOKENS, DEFAULT_HUMAN, DEFAULT_PERSONA, LOGGER_NAME from memgpt.presets.presets import DEFAULT_PRESET + + # helper functions for writing to configs def get_field(config, section, field): if section not in config: @@ -84,6 +89,14 @@ class MemGPTConfig: # version (for backcompat) memgpt_version: str = None + # logging (for logger) + logging_level: str = "CRITICAL" # default log level + logging_enable_logfile: bool = True + logging_backup_count: int = 3 + logging_max_file_bytes: int = 10 * 1024 * 1024 # 10 MB in bytes + logging_logdir: str = os.path.join(MEMGPT_DIR, "logs") + logging_logpathname: str = os.path.join(logging_logdir, "memgpt.log") + def __post_init__(self): # ensure types self.embedding_chunk_size = int(self.embedding_chunk_size) @@ -104,6 +117,8 @@ class MemGPTConfig: else: config_path = MemGPTConfig.config_path + # insure all configuration directories exist + cls.create_config_dir() if os.path.exists(config_path): # read existing config config.read(config_path) @@ -134,14 +149,44 @@ class MemGPTConfig: "anon_clientid": get_field(config, "client", "anon_clientid"), "config_path": config_path, "memgpt_version": get_field(config, "version", "memgpt_version"), + "logging_level": get_field(config, "logger_MemGPT", "level"), + "logging_enable_logfile": True if "consoleHandler,logfileHandler" == get_field(config, "logger_MemGPT", + "handlers") else False, + "logging_backup_count": get_field(config, "handler_logfileHandler", "backupcount"), + "logging_max_file_bytes": get_field(config, "handler_logfileHandler", "maxBytes"), + "logging_logdir": get_field(config, "logging_paths", "logdir"), + "logging_logpathname": get_field(config, "logging_paths", "logpathname"), } + # ensure logging is config is set correctly support for upgrades + force_save = False + if config_dict["logging_level"] is None or config_dict["logging_backup_count"] is None or config_dict[ + "logging_max_file_bytes"] is None or config_dict["logging_logdir"] is None or config_dict[ + "logging_logpathname"] is None: + # load loggind defaults if none + config_dict["logging_enable_logfile"] = MemGPTConfig.logging_enable_logfile + config_dict["logging_level"] = MemGPTConfig.logging_level + config_dict["logging_backup_count"] = MemGPTConfig.logging_backup_count + config_dict["logging_max_file_bytes"] = MemGPTConfig.logging_max_file_bytes + config_dict["logging_logdir"] = MemGPTConfig.logging_logdir + config_dict["logging_logpathname"] = MemGPTConfig.logging_logpathname + force_save = True config_dict = {k: v for k, v in config_dict.items() if v is not None} + + if force_save: + temp_config = cls(**config_dict) + temp_config.save() + logger = logging.getLogger(LOGGER_NAME) + logger.debug(f'Updated Missing Logging Configuration: {config_path}') + return cls(**config_dict) # create new config anon_clientid = MemGPTConfig.generate_uuid() config = cls(anon_clientid=anon_clientid, config_path=config_path) + config.create_config_dir() # create dirs config.save() # save updated config + logger = logging.getLogger(LOGGER_NAME) + logger.debug(f'Created New Configuration: {config_path}') return config def save(self): @@ -192,10 +237,54 @@ class MemGPTConfig: self.anon_clientid = self.generate_uuid() set_field(config, "client", "anon_clientid", self.anon_clientid) - if not os.path.exists(MEMGPT_DIR): - os.makedirs(MEMGPT_DIR, exist_ok=True) + # logging + set_field(config, "loggers", "keys", "root,MemGPT") + set_field(config, "handlers", "keys", "consoleHandler,logfileHandler") + set_field(config, "formatters", "keys", "consoleFormatter,logfileFormatter") + # logging root possibly used by other modules not using MemGPT logger + set_field(config, "logger_root", "level", "CRITICAL") + set_field(config, "logger_root", "handlers", "consoleHandler") + # logging MemGPT + set_field(config, "logger_MemGPT", "level", self.logging_level) + if self.logging_enable_logfile: + # this will enable logging to file + set_field(config, "logger_MemGPT", "handlers", "consoleHandler,logfileHandler") + else: + # this removes file logging if not enabled + set_field(config, "logger_MemGPT", "handlers", "consoleHandler") + set_field(config, "logger_MemGPT", "qualname", "MemGPT") + set_field(config, "logger_MemGPT", "propagate", "0") # do not propigate to root + # console logging handler + set_field(config, "handler_consoleHandler", "class", "StreamHandler") + set_field(config, "handler_consoleHandler", "level", self.logging_level) + set_field(config, "handler_consoleHandler", "formatter", "consoleFormatter") + set_field(config, "handler_consoleHandler", "args", "(sys.stdout,)") + # console logging formatter + set_field(config, "formatter_consoleFormatter", "format", "%(name)s - %(levelname)s - %(message)s") + if self.logging_enable_logfile: + set_field(config, "formatter_logfileFormatter", "format", + "%(asctime)s - %(name)s - %(levelname)s - %(message)s") + # logfile logging handler Rotating File Handler + set_field(config, "handler_logfileHandler", "class", "handlers.RotatingFileHandler") + set_field(config, "handler_logfileHandler", "level", self.logging_level) + fixed_logpathname = utils.fix_file_path(self.logging_logpathname) + set_field(config, "handler_logfileHandler", "args", + f"('{fixed_logpathname}', {self.logging_max_file_bytes}, {self.logging_backup_count})") + set_field(config, "handler_logfileHandler", "formatter", "logfileFormatter") + # logging paths + set_field(config, "logging_paths", "logdir", self.logging_logdir) + set_field(config, "logging_paths", "logpathname", self.logging_logpathname) + + # always make sure all directories are present + self.create_config_dir() + with open(self.config_path, "w") as f: config.write(f) + # reload logging config after write. + logging.config.fileConfig(self.config_path,disable_existing_loggers=False) + # reset the logger (global) logger is defined as global + reload_logger() + logger.debug(f'Saved Config: {self.config_path}') @staticmethod def exists(): @@ -213,7 +302,7 @@ class MemGPTConfig: if not os.path.exists(MEMGPT_DIR): os.makedirs(MEMGPT_DIR, exist_ok=True) - folders = ["personas", "humans", "archival", "agents", "functions", "system_prompts", "presets"] + folders = ["personas", "humans", "archival", "agents", "functions", "system_prompts", "presets", "logs"] for folder in folders: if not os.path.exists(os.path.join(MEMGPT_DIR, folder)): os.makedirs(os.path.join(MEMGPT_DIR, folder)) @@ -282,7 +371,8 @@ class AgentConfig: # save agent config self.agent_config_path = ( - os.path.join(MEMGPT_DIR, "agents", self.name, "config.json") if agent_config_path is None else agent_config_path + os.path.join(MEMGPT_DIR, "agents", self.name, + "config.json") if agent_config_path is None else agent_config_path ) def generate_agent_id(self, length=6): diff --git a/memgpt/constants.py b/memgpt/constants.py index a722a64c..6692d9db 100644 --- a/memgpt/constants.py +++ b/memgpt/constants.py @@ -6,6 +6,8 @@ DEFAULT_MEMGPT_MODEL = "gpt-4" DEFAULT_PERSONA = "sam_pov" DEFAULT_HUMAN = "basic" +LOGGER_NAME = "MemGPT" + FIRST_MESSAGE_ATTEMPTS = 10 INITIAL_BOOT_MESSAGE = "Boot sequence complete. Persona activated." diff --git a/memgpt/main.py b/memgpt/main.py index 9bc7fdbc..412c94b0 100644 --- a/memgpt/main.py +++ b/memgpt/main.py @@ -17,6 +17,7 @@ from prettytable import PrettyTable console = Console() +from memgpt.memgptlog import logger from memgpt.interface import CLIInterface as interface # for printing to terminal import memgpt.agent as agent import memgpt.system as system diff --git a/memgpt/memgptlog.py b/memgpt/memgptlog.py new file mode 100644 index 00000000..38ae260f --- /dev/null +++ b/memgpt/memgptlog.py @@ -0,0 +1,12 @@ +import os.path +import logging +from memgpt.constants import LOGGER_NAME + +# load the logger for global use +logger = logging.getLogger(LOGGER_NAME) + + +def reload_logger(): + global logger # This is required to modify the global 'logger' + # Reconfigure logger + logger = logging.getLogger(LOGGER_NAME) diff --git a/memgpt/utils.py b/memgpt/utils.py index 7b0ea1b8..71766562 100644 --- a/memgpt/utils.py +++ b/memgpt/utils.py @@ -171,3 +171,10 @@ def get_schema_diff(schema_a, schema_b): difference = [line for line in difference if line.startswith("+ ") or line.startswith("- ")] return "".join(difference) + +def fix_file_path(path): + """ + Converts backslashes to forward slashes in a file path. + This is useful for ensuring compatibility in file paths across different systems. + """ + return path.replace('\\', '/') \ No newline at end of file From f7df7d6d781fd11fa40d8e6e40864f0e4b310691 Mon Sep 17 00:00:00 2001 From: Max Blackmer Date: Tue, 19 Dec 2023 15:09:08 -0500 Subject: [PATCH 2/5] [cpacker#319] run Black Reformat on files. --- memgpt/cli/cli.py | 2 +- memgpt/cli/cli_config.py | 2 ++ memgpt/config.py | 41 ++++++++++++++++++++++------------------ memgpt/utils.py | 3 ++- 4 files changed, 28 insertions(+), 20 deletions(-) diff --git a/memgpt/cli/cli.py b/memgpt/cli/cli.py index 10328bfe..5f2463a6 100644 --- a/memgpt/cli/cli.py +++ b/memgpt/cli/cli.py @@ -139,7 +139,7 @@ def run( """ # setup logger - #TODO: remove Utils Debug after global logging is complete. + # TODO: remove Utils Debug after global logging is complete. utils.DEBUG = debug # TODO: add logging command line options for runtime log level diff --git a/memgpt/cli/cli_config.py b/memgpt/cli/cli_config.py index 132cbd8f..02eaf351 100644 --- a/memgpt/cli/cli_config.py +++ b/memgpt/cli/cli_config.py @@ -4,8 +4,10 @@ from prettytable import PrettyTable import typer import os import shutil + # from global logging configuration from memgpt.memgptlog import logger + # from memgpt.cli import app from memgpt import utils diff --git a/memgpt/config.py b/memgpt/config.py index 4b40747c..10758669 100644 --- a/memgpt/config.py +++ b/memgpt/config.py @@ -1,6 +1,6 @@ import logging import logging.config -from memgpt.memgptlog import logger,reload_logger +from memgpt.memgptlog import logger, reload_logger import inspect import json import os @@ -14,8 +14,6 @@ from memgpt.constants import MEMGPT_DIR, LLM_MAX_TOKENS, DEFAULT_HUMAN, DEFAULT_ from memgpt.presets.presets import DEFAULT_PRESET - - # helper functions for writing to configs def get_field(config, section, field): if section not in config: @@ -150,8 +148,9 @@ class MemGPTConfig: "config_path": config_path, "memgpt_version": get_field(config, "version", "memgpt_version"), "logging_level": get_field(config, "logger_MemGPT", "level"), - "logging_enable_logfile": True if "consoleHandler,logfileHandler" == get_field(config, "logger_MemGPT", - "handlers") else False, + "logging_enable_logfile": True + if "consoleHandler,logfileHandler" == get_field(config, "logger_MemGPT", "handlers") + else False, "logging_backup_count": get_field(config, "handler_logfileHandler", "backupcount"), "logging_max_file_bytes": get_field(config, "handler_logfileHandler", "maxBytes"), "logging_logdir": get_field(config, "logging_paths", "logdir"), @@ -159,9 +158,13 @@ class MemGPTConfig: } # ensure logging is config is set correctly support for upgrades force_save = False - if config_dict["logging_level"] is None or config_dict["logging_backup_count"] is None or config_dict[ - "logging_max_file_bytes"] is None or config_dict["logging_logdir"] is None or config_dict[ - "logging_logpathname"] is None: + if ( + config_dict["logging_level"] is None + or config_dict["logging_backup_count"] is None + or config_dict["logging_max_file_bytes"] is None + or config_dict["logging_logdir"] is None + or config_dict["logging_logpathname"] is None + ): # load loggind defaults if none config_dict["logging_enable_logfile"] = MemGPTConfig.logging_enable_logfile config_dict["logging_level"] = MemGPTConfig.logging_level @@ -176,7 +179,7 @@ class MemGPTConfig: temp_config = cls(**config_dict) temp_config.save() logger = logging.getLogger(LOGGER_NAME) - logger.debug(f'Updated Missing Logging Configuration: {config_path}') + logger.debug(f"Updated Missing Logging Configuration: {config_path}") return cls(**config_dict) @@ -186,7 +189,7 @@ class MemGPTConfig: config.create_config_dir() # create dirs config.save() # save updated config logger = logging.getLogger(LOGGER_NAME) - logger.debug(f'Created New Configuration: {config_path}') + logger.debug(f"Created New Configuration: {config_path}") return config def save(self): @@ -262,14 +265,17 @@ class MemGPTConfig: # console logging formatter set_field(config, "formatter_consoleFormatter", "format", "%(name)s - %(levelname)s - %(message)s") if self.logging_enable_logfile: - set_field(config, "formatter_logfileFormatter", "format", - "%(asctime)s - %(name)s - %(levelname)s - %(message)s") + set_field(config, "formatter_logfileFormatter", "format", "%(asctime)s - %(name)s - %(levelname)s - %(message)s") # logfile logging handler Rotating File Handler set_field(config, "handler_logfileHandler", "class", "handlers.RotatingFileHandler") set_field(config, "handler_logfileHandler", "level", self.logging_level) fixed_logpathname = utils.fix_file_path(self.logging_logpathname) - set_field(config, "handler_logfileHandler", "args", - f"('{fixed_logpathname}', {self.logging_max_file_bytes}, {self.logging_backup_count})") + set_field( + config, + "handler_logfileHandler", + "args", + f"('{fixed_logpathname}', {self.logging_max_file_bytes}, {self.logging_backup_count})", + ) set_field(config, "handler_logfileHandler", "formatter", "logfileFormatter") # logging paths set_field(config, "logging_paths", "logdir", self.logging_logdir) @@ -281,10 +287,10 @@ class MemGPTConfig: with open(self.config_path, "w") as f: config.write(f) # reload logging config after write. - logging.config.fileConfig(self.config_path,disable_existing_loggers=False) + logging.config.fileConfig(self.config_path, disable_existing_loggers=False) # reset the logger (global) logger is defined as global reload_logger() - logger.debug(f'Saved Config: {self.config_path}') + logger.debug(f"Saved Config: {self.config_path}") @staticmethod def exists(): @@ -371,8 +377,7 @@ class AgentConfig: # save agent config self.agent_config_path = ( - os.path.join(MEMGPT_DIR, "agents", self.name, - "config.json") if agent_config_path is None else agent_config_path + os.path.join(MEMGPT_DIR, "agents", self.name, "config.json") if agent_config_path is None else agent_config_path ) def generate_agent_id(self, length=6): diff --git a/memgpt/utils.py b/memgpt/utils.py index 27335cb5..efb9e24c 100644 --- a/memgpt/utils.py +++ b/memgpt/utils.py @@ -260,9 +260,10 @@ def get_schema_diff(schema_a, schema_b): return "".join(difference) + def fix_file_path(path): """ Converts backslashes to forward slashes in a file path. This is useful for ensuring compatibility in file paths across different systems. """ - return path.replace('\\', '/') \ No newline at end of file + return path.replace("\\", "/") From 5733f0418d87b2ebacfad2a3a776fe67e79d43b2 Mon Sep 17 00:00:00 2001 From: Max Blackmer Date: Tue, 26 Dec 2023 08:52:24 -0500 Subject: [PATCH 3/5] [cpacker#319] Refactor memgptlog.py to log.py and move a function to log.py --- memgpt/cli/cli.py | 2 +- memgpt/cli/cli_config.py | 2 +- memgpt/config.py | 2 +- memgpt/{memgptlog.py => log.py} | 8 ++++++++ memgpt/main.py | 2 +- memgpt/utils.py | 8 -------- 6 files changed, 12 insertions(+), 12 deletions(-) rename memgpt/{memgptlog.py => log.py} (57%) diff --git a/memgpt/cli/cli.py b/memgpt/cli/cli.py index 9196baec..93b9b305 100644 --- a/memgpt/cli/cli.py +++ b/memgpt/cli/cli.py @@ -13,7 +13,7 @@ from enum import Enum from llama_index import set_global_service_context from llama_index import ServiceContext -from memgpt.memgptlog import logger +from memgpt.log import logger from memgpt.interface import CLIInterface as interface # for printing to terminal from memgpt.cli.cli_config import configure import memgpt.presets.presets as presets diff --git a/memgpt/cli/cli_config.py b/memgpt/cli/cli_config.py index 4b2cc55f..8203a5e5 100644 --- a/memgpt/cli/cli_config.py +++ b/memgpt/cli/cli_config.py @@ -6,7 +6,7 @@ import os import shutil # from global logging configuration -from memgpt.memgptlog import logger +from memgpt.log import logger # from memgpt.cli import app from memgpt import utils diff --git a/memgpt/config.py b/memgpt/config.py index d81f76e2..804e7bf3 100644 --- a/memgpt/config.py +++ b/memgpt/config.py @@ -1,6 +1,6 @@ import logging import logging.config -from memgpt.memgptlog import logger, reload_logger +from memgpt.log import logger, reload_logger, fix_file_path import inspect import json import os diff --git a/memgpt/memgptlog.py b/memgpt/log.py similarity index 57% rename from memgpt/memgptlog.py rename to memgpt/log.py index 38ae260f..8151cd99 100644 --- a/memgpt/memgptlog.py +++ b/memgpt/log.py @@ -10,3 +10,11 @@ def reload_logger(): global logger # This is required to modify the global 'logger' # Reconfigure logger logger = logging.getLogger(LOGGER_NAME) + + +def fix_file_path(path): + """ + Converts backslashes to forward slashes in a file path. + This is useful for ensuring compatibility in file paths across different systems. + """ + return path.replace("\\", "/") diff --git a/memgpt/main.py b/memgpt/main.py index 19ff43c8..d5c2335e 100644 --- a/memgpt/main.py +++ b/memgpt/main.py @@ -17,7 +17,7 @@ from prettytable import PrettyTable console = Console() -from memgpt.memgptlog import logger +from memgpt.log import logger from memgpt.interface import CLIInterface as interface # for printing to terminal import memgpt.agent as agent import memgpt.system as system diff --git a/memgpt/utils.py b/memgpt/utils.py index 5b720d83..8e864a42 100644 --- a/memgpt/utils.py +++ b/memgpt/utils.py @@ -288,11 +288,3 @@ def get_schema_diff(schema_a, schema_b): difference = [line for line in difference if line.startswith("+ ") or line.startswith("- ")] return "".join(difference) - - -def fix_file_path(path): - """ - Converts backslashes to forward slashes in a file path. - This is useful for ensuring compatibility in file paths across different systems. - """ - return path.replace("\\", "/") From c43acc3ece05f6d61f49f690fa31895fc4d08afb Mon Sep 17 00:00:00 2001 From: Max Blackmer Date: Tue, 26 Dec 2023 09:03:35 -0500 Subject: [PATCH 4/5] [cpacker#319] fixed moved function call. --- memgpt/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/memgpt/config.py b/memgpt/config.py index 804e7bf3..f6d7aa8f 100644 --- a/memgpt/config.py +++ b/memgpt/config.py @@ -272,7 +272,7 @@ class MemGPTConfig: # logfile logging handler Rotating File Handler set_field(config, "handler_logfileHandler", "class", "handlers.RotatingFileHandler") set_field(config, "handler_logfileHandler", "level", self.logging_level) - fixed_logpathname = utils.fix_file_path(self.logging_logpathname) + fixed_logpathname = fix_file_path(self.logging_logpathname) set_field( config, "handler_logfileHandler", From abf4fb03bdbebaf930b6babfd3c14e0e71066fd5 Mon Sep 17 00:00:00 2001 From: Max Blackmer Date: Thu, 11 Jan 2024 10:44:20 -0500 Subject: [PATCH 5/5] [cpacker#319] fixed error with config with improper include in config.py added test_log.py. for testing global logging in log.py --- memgpt/config.py | 2 +- tests/test_log.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 tests/test_log.py diff --git a/memgpt/config.py b/memgpt/config.py index 5d61c72e..ced041bf 100644 --- a/memgpt/config.py +++ b/memgpt/config.py @@ -1,4 +1,4 @@ -from log import logger +from memgpt.log import logger import inspect import json import os diff --git a/tests/test_log.py b/tests/test_log.py new file mode 100644 index 00000000..5beb4530 --- /dev/null +++ b/tests/test_log.py @@ -0,0 +1,16 @@ +import logging +from memgpt.log import logger +from memgpt.constants import LOGGER_LOG_LEVELS +import pytest + + +def test_log_debug(): + # test setting logging level + assert logging.DEBUG == LOGGER_LOG_LEVELS["DEBUG"] + logger.setLevel(LOGGER_LOG_LEVELS["DEBUG"]) + assert logger.isEnabledFor(logging.DEBUG) + + # Assert that the message was logged + assert logger.hasHandlers() + logger.debug("This is a Debug message") + assert 1 == 1