From 9fa75e1752bec1901134d03979ee190ec4a0a98e Mon Sep 17 00:00:00 2001 From: sk Date: Tue, 31 Oct 2023 15:18:12 -0400 Subject: [PATCH] Added db load ability (#106) * Added db load ability, updated README * Fixed spacing on multiplication * Fixed double quotes * Added double quotes to .db * After reading the code, realized we can return here for .db extension, and let main handle loading the .db into archival memory. Issue is that the final else was treating the db as a file and reading it in raw, causing issues. * Removed .csv reference, not currently working. --- memgpt/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/memgpt/utils.py b/memgpt/utils.py index e2146bc7..860a459f 100644 --- a/memgpt/utils.py +++ b/memgpt/utils.py @@ -152,6 +152,10 @@ def total_bytes(pattern): def chunk_file(file, tkns_per_chunk=300, model="gpt-4"): encoding = tiktoken.encoding_for_model(model) + + if file.endswith(".db"): + return # can't read the sqlite db this way, will get handled in main.py + with open(file, "r") as f: if file.endswith(".pdf"): lines = [l for l in read_pdf_in_chunks(file, tkns_per_chunk * 8)]