From bccbf75520dedf3368219b26b125f26ac9ccd3cf Mon Sep 17 00:00:00 2001 From: Matthew Zhou Date: Fri, 27 Jun 2025 15:04:24 -0700 Subject: [PATCH] feat: Add csv support for files (#3077) --- letta/services/file_processor/file_types.py | 2 + tests/data/test.csv | 100 ++++++++++++++++++++ tests/test_sources.py | 1 + 3 files changed, 103 insertions(+) create mode 100644 tests/data/test.csv diff --git a/letta/services/file_processor/file_types.py b/letta/services/file_processor/file_types.py index 34be23f0..b311a180 100644 --- a/letta/services/file_processor/file_types.py +++ b/letta/services/file_processor/file_types.py @@ -49,6 +49,7 @@ class FileTypeRegistry: self.register(".markdown", "text/markdown", True, "Markdown document", ChunkingStrategy.DOCUMENTATION) self.register(".json", "application/json", True, "JSON data file", ChunkingStrategy.STRUCTURED_DATA) self.register(".jsonl", "application/jsonl", True, "JSON Lines file", ChunkingStrategy.STRUCTURED_DATA) + self.register(".csv", "text/csv", True, "CSV data file", ChunkingStrategy.STRUCTURED_DATA) # Programming languages self.register(".py", "text/x-python", True, "Python source code", ChunkingStrategy.CODE) @@ -137,6 +138,7 @@ class FileTypeRegistry: mimetypes.add_type("text/x-markdown", ".md") mimetypes.add_type("application/x-jsonlines", ".jsonl") mimetypes.add_type("text/xml", ".xml") + mimetypes.add_type("text/csv", ".csv") def get_allowed_media_types(self) -> Set[str]: """ diff --git a/tests/data/test.csv b/tests/data/test.csv new file mode 100644 index 00000000..3f3629d3 --- /dev/null +++ b/tests/data/test.csv @@ -0,0 +1,100 @@ +Index,Name,Description,Brand,Category,Price,Currency,Stock,EAN,Color,Size,Availability,Internal ID +1,Smart Fan Iron Cooker Go Wireless Portable,Catch enough role nearly.,Herman Ltd,Kids' Clothing,585,USD,194,3968600833473,Cornsilk,5x7 in,limited_stock,54 +2,Fan,All movement yeah tax me.,"Braun, King and Rollins",Grooming Tools,992,USD,724,0191126950284,Bisque,S,discontinued,49 +3,Smart Speakerphone Charger Eco Plus Clean,Quickly inside pull line lay start.,Peck-Coleman,Fishing & Hunting,940,USD,769,7569143820621,Blue,Extra Large,pre_order,42 +4,Premium Grill Trimmer Portable,Lawyer one than fire.,Hines Ltd,Skincare,324,USD,93,2705140928037,Ivory,50x70 cm,out_of_stock,93 +5,Keyboard Freezer,Remain Congress blood plan voice.,"Spence, Webster and Orr",Laptops & Computers,908,USD,614,9830391008108,FloralWhite,10x10 cm,discontinued,91 +6,Smart Fridge Plus,Tell tend provide government voice.,"Marks, Guzman and Cooley",Kitchen Appliances,627,USD,901,3039245054025,LightSalmon,XXL,pre_order,29 +7,Compact Dock One Portable Wireless,Push role risk must get performance color.,Newman-Barajas,Fishing & Hunting,385,USD,366,6194768427002,MediumPurple,Large,out_of_stock,23 +8,Premium Cooler Digital Portable X,Fall opportunity southern drive professor top.,"Gutierrez, Singleton and Downs",Skincare,296,USD,609,6978363826350,PaleGreen,8x10 in,in_stock,6 +9,Compact Fan Scooter Headphones,Medical myself his piece agree.,Weber and Sons,Clothing & Apparel,870,USD,930,2666733210512,DarkBlue,XS,limited_stock,6 +10,Eco Dock Air,Know hope enjoy budget have.,Humphrey and Sons,Beauty & Personal Care,602,USD,923,3726212774009,SandyBrown,50x70 cm,in_stock,11 +11,Thermostat Stove Lamp,Pull perhaps customer available.,Moreno-Weaver,Kids' Clothing,707,USD,257,6886778382210,Thistle,M,discontinued,33 +12,Mini Powerbank Lite Plus,Still can tell.,"Gentry, Vincent and Saunders",Makeup,499,USD,108,9632709420387,OldLace,M,pre_order,22 +13,Scooter Bicycle Oven,Deal event item ever financial home.,Mclean-Aguilar,Laptops & Computers,79,USD,203,0227867819631,LightSkyBlue,30x40 cm,in_stock,58 +14,Fan,It minute eight mother often focus ever.,Mcdaniel LLC,Toys & Games,897,USD,338,1326983446208,MediumBlue,50x70 cm,limited_stock,67 +15,Wireless Light Heater Clock X,Sing statement show future size hard hope.,Sosa LLC,Grooming Tools,581,USD,877,4749136042046,DarkTurquoise,8x10 in,limited_stock,84 +16,Ultra Powerbank Scale Automatic,Against or catch tough nor wait reason.,Macias Group,Fishing & Hunting,223,USD,62,6282374002499,Azure,30x40 cm,pre_order,93 +17,Projector,Party everybody bill work condition behind great share.,Ali Group,Men's Clothing,699,USD,762,3359907564212,DeepSkyBlue,Small,in_stock,89 +18,Blender,Bit dog upon sometimes trade.,Gray-Deleon,Camping & Hiking,881,USD,34,2045936905300,DarkSeaGreen,30x40 cm,discontinued,83 +19,Wireless Speakerphone Oven Advanced,Something the interview actually book white easy.,"Yates, Logan and Cervantes",Grooming Tools,601,USD,26,4270229099796,BlanchedAlmond,12x18 in,out_of_stock,76 +20,Iron Microphone,High see avoid draw check.,"Acosta, Sullivan and Chaney",Men's Clothing,261,USD,80,1846205234548,OldLace,S,discontinued,45 +21,Lock,Who seven meeting ball trouble ago cover.,Mullins-Dodson,Fishing & Hunting,350,USD,842,0651020229713,LavenderBlush,5x7 in,in_stock,22 +22,Automatic Speaker Wireless Plus Edge,Near physical blood number never and.,Farrell-Ramos,Cleaning Supplies,726,USD,215,8872341006131,LemonChiffon,15x20 cm,limited_stock,69 +23,Eco Treadmill Plus Smart,Despite truth develop cover move serve describe it.,Bautista-Allen,Headphones & Earbuds,756,USD,148,3581173655015,SaddleBrown,XS,discontinued,65 +24,Smart Microphone Clock Ultra One,Open section note all.,Larson LLC,Home & Kitchen,17,USD,687,8108499855216,CadetBlue,M,pre_order,1 +25,Digital Router Scanner One,Suffer event thought entire add.,"Bryan, Benitez and Barton",Haircare,257,USD,130,3287425624081,DarkSalmon,15x20 cm,in_stock,27 +26,Pro Stove Light,Mention nature whom discuss minute successful.,"Brock, Lawrence and Pennington",Fishing & Hunting,263,USD,867,3636002675846,HoneyDew,10x10 cm,pre_order,4 +27,Clean Mixer Ultra Eco,Serious source open threat.,"Luna, Davidson and Galvan",Grooming Tools,951,USD,190,7547662604151,DarkRed,15x20 cm,discontinued,33 +28,Ultra Fan,Structure training receive name collection stuff west.,Fleming PLC,Kids' Clothing,931,USD,31,6232473294448,Peru,S,discontinued,2 +29,Ultra Drone Smart,Conference imagine apply decade care return.,Aguirre-Mercado,Headphones & Earbuds,809,USD,250,0194012822726,AntiqueWhite,15x20 cm,backorder,25 +30,Smart Lock Mini Pro,Three development base reveal smile morning military.,Branch Ltd,Cleaning Supplies,989,USD,879,9292256940884,DarkCyan,10x10 cm,discontinued,91 +31,Rechargeable Keyboard Toaster Monitor Portable Fast,Parent these not.,Carter-Holland,Cameras & Accessories,962,USD,546,1884257734807,LightGray,XS,discontinued,24 +32,Mini Mixer Freezer Trimmer Smart,Impact prepare religious public everyone allow often.,Pollard-Branch,Bedding & Bath,399,USD,88,0358170616393,RoyalBlue,M,backorder,36 +33,Heater Cooker,Station shoulder get suggest tax.,"Preston, Bowen and Singh",Haircare,285,USD,840,4811472870604,CornflowerBlue,50x70 cm,pre_order,40 +34,Fast Shaver X One,Concern current war still sit a draw with.,"Cunningham, Barr and Ponce",Cleaning Supplies,894,USD,860,0631386626401,Red,Large,pre_order,92 +35,Wireless Webcam Bicycle Camera,Bank international into Republican something important mean.,"Houston, Newton and Terry",Men's Clothing,141,USD,704,0607700002091,SandyBrown,10x10 cm,discontinued,72 +36,Clean Scale Speaker Powerbank Clean Wireless,Science think fight piece front.,Galvan PLC,"Accessories (Bags, Hats, Belts)",308,USD,637,6988842313004,PaleGoldenRod,10x10 cm,limited_stock,35 +37,Advanced Fan Powerbank,White mission budget those pretty fly.,Sims LLC,Headphones & Earbuds,711,USD,892,5796764564586,IndianRed,L,out_of_stock,89 +38,Pro Thermostat Speaker Touch Eco,Republican vote drug produce.,Brewer-Day,Camping & Hiking,865,USD,593,7535096642818,LimeGreen,100x200 mm,backorder,57 +39,Fast Blender Treadmill Tablet One Air,True oil town like.,Stevenson-Cummings,Team Sports,409,USD,69,5314300563345,Aqua,Small,limited_stock,35 +40,Automatic Iron Toaster Projector,Hold dinner everything.,Foley-Thornton,Smartwatches,787,USD,805,9338080255944,RoyalBlue,8x10 in,out_of_stock,13 +41,Smart Scale Stove,Two toward direction memory account cost.,"Pace, Becker and Ibarra",Smartwatches,334,USD,38,9990041184672,AntiqueWhite,5x7 in,in_stock,10 +42,Smart Webcam Mouse Digital,Learn week rate hard.,Fernandez-Hendrix,Toys & Games,830,USD,120,8985787888893,Snow,S,out_of_stock,92 +43,Compact Drone Lite,There only southern.,Rowe-Robinson,"Accessories (Bags, Hats, Belts)",339,USD,220,0512595115903,SpringGreen,Extra Large,out_of_stock,20 +44,Digital Scale Wireless One Portable,Step system consider project point.,"Anthony, Monroe and Myers",Headphones & Earbuds,128,USD,727,6358706879234,Snow,Large,pre_order,86 +45,Clean Drone Thermostat Headphones,Several maintain most position street share room.,Ray-Monroe,Fragrances,716,USD,617,2864500397712,Teal,XXL,discontinued,40 +46,Wireless Scooter,Wish their determine character third would among.,Johns-Huber,Women's Clothing,625,USD,173,4771208434411,DarkOrchid,XL,pre_order,1 +47,Clean Light Advanced Smart Premium,Mouth others type as of same piece.,Robbins and Sons,Grooming Tools,485,USD,448,3353269497386,LightGreen,Small,discontinued,55 +48,Compact Fan Iron Clock Digital,Hotel admit pressure democratic television reason remember.,Novak-Hamilton,Home & Kitchen,526,USD,944,3674615714477,CadetBlue,15x20 cm,pre_order,11 +49,Ultra Watch Eco,Eye analysis require create community knowledge.,"Le, Sampson and Walsh",Shoes & Footwear,703,USD,572,5786160872088,LightSalmon,10x10 cm,discontinued,17 +50,Fast Lamp Scanner Vacuum,There respond common travel as language everything more.,Bautista-Green,Shoes & Footwear,363,USD,250,4598210880599,GreenYellow,10x10 cm,pre_order,60 +51,Eco Stove Edge,Hold wrong only language production.,"Gonzales, Best and Ponce",Furniture,52,USD,121,1925515729316,Plum,12x18 in,limited_stock,86 +52,Fast Powerbank Radio Automatic Touch Ultra,Education our science loss call.,Wolfe-Holden,Beauty & Personal Care,685,USD,178,1419539240556,DarkRed,50x70 cm,backorder,76 +53,Pro Oven Scale,Mention fish certain allow in environmental source leg.,Mcclure and Sons,Home Decor,150,USD,816,5047126002551,Blue,XL,backorder,24 +54,Smart Printer Charger Brush Smart Ultra Touch,Environmental authority accept.,Powers-Burgess,Grooming Tools,312,USD,752,9297742472047,MediumOrchid,M,out_of_stock,65 +55,Wireless Watch Premium Air,Little agree game thing type environmental.,Lynch PLC,Fishing & Hunting,673,USD,552,5327358796184,Fuchsia,30x40 cm,out_of_stock,39 +56,Advanced Thermostat Microphone Scooter Clean Smart,Miss special Republican team draw middle opportunity.,Meza-Roberson,Kitchen Appliances,425,USD,422,0673364369078,Aqua,30x40 cm,out_of_stock,28 +57,Pro Mouse Mini Rechargeable,Age possible Republican rest exist interesting source.,"Bernard, Bird and Eaton",Fitness Equipment,478,USD,693,3054066626701,Teal,XS,out_of_stock,68 +58,Silent Powerbank Radio Smart Plus,Listen maybe arrive other outside shake black west.,Lawrence LLC,Health & Wellness,871,USD,196,5772715035664,PaleGreen,XL,limited_stock,75 +59,Clean Speakerphone Edge Rechargeable Max,Yes many nation.,"Zimmerman, Esparza and Andrews",Smartphones,196,USD,937,9822542327121,MediumSpringGreen,M,pre_order,77 +60,Silent Heater Stove Scanner,Action positive skin Congress pressure begin return.,Chavez-Preston,"Accessories (Bags, Hats, Belts)",747,USD,646,1263475720975,PaleGoldenRod,M,discontinued,30 +61,Smart Mixer Bicycle Fridge,Dinner soldier name model cup physical.,Williams-Morrison,Men's Clothing,63,USD,535,8041785529098,LightBlue,XS,discontinued,82 +62,Smart Freezer Clean,Bank cover pattern since issue hear particularly bank.,"Costa, Mcneil and Stafford",Office Supplies,361,USD,340,5109263445780,Orchid,M,in_stock,23 +63,Automatic Blender Toaster Camera,Carry real agree outside major.,Conrad Group,Haircare,329,USD,416,5161284787010,BlanchedAlmond,5x7 in,limited_stock,66 +64,Premium Mouse Lamp Ultra Touch Compact,Someone raise lot let more list.,Page and Sons,Cycling,38,USD,29,5153484969624,MediumTurquoise,Large,out_of_stock,46 +65,Ultra Drone Wireless Go Clean,Project impact possible both.,Hurst Inc,Kitchen Appliances,119,USD,346,9613973317726,RosyBrown,S,pre_order,84 +66,Smart Monitor Cooler Dock Automatic Lite Prime,Skin quite serve kind themselves player so.,"Russell, Vance and Ortiz",Automotive,379,USD,787,6890202499702,Crimson,XXL,pre_order,47 +67,Automatic Fridge Freezer,While everybody war simply plan social.,Hogan Ltd,Furniture,918,USD,959,7998660897333,Tomato,XS,discontinued,1 +68,Rechargeable Router Keyboard,Chance start number accept.,Bautista Ltd,Fishing & Hunting,242,USD,974,2863748603180,LightGoldenRodYellow,50x70 cm,discontinued,14 +69,Digital Microphone Camera,Life your gas thus event.,"Davila, Bowman and Flynn",Home Decor,71,USD,100,3272423264049,LightSeaGreen,Extra Large,out_of_stock,3 +70,Smart Stove Iron,Think feel office pretty trade room.,"Harding, Frederick and Holloway",Toys & Games,689,USD,210,0761286910840,Blue,12x18 in,out_of_stock,4 +71,Ultra Treadmill,West against people general themselves young likely.,Alvarado Ltd,Toys & Games,65,USD,250,7977249259236,SlateBlue,50x70 cm,pre_order,18 +72,Mini Shaver Mouse Fast Mini X,Meet think hope per.,Robbins-Duffy,Toys & Games,84,USD,703,3570230765743,Cornsilk,8x10 in,pre_order,53 +73,Wireless Fridge Cooker Portable Edge,Church line feeling down maintain.,Henson-Ware,Clothing & Apparel,712,USD,966,6944904013299,HoneyDew,50x70 cm,discontinued,83 +74,Fan Iron Cooker,Process finish right star.,Avery-Meadows,Health & Wellness,795,USD,272,5139271989938,PowderBlue,M,pre_order,41 +75,Eco Fridge Printer Mouse,He effort though machine standard.,"Jones, Weber and Pitts",Cycling,367,USD,204,1631900897637,Salmon,Medium,limited_stock,73 +76,Wireless Scanner Tablet Radio,Plan mention fight focus research set health.,"Valenzuela, Mahoney and Lowery",Clothing & Apparel,594,USD,172,4697368151459,DeepSkyBlue,M,in_stock,99 +77,Pro Stove Light Air Mini,News design site able before candidate air.,Huffman-Washington,Camping & Hiking,976,USD,374,8439220109668,DarkRed,Medium,out_of_stock,18 +78,Shaver,Who someone training consumer Mr eight.,"Torres, Christensen and Edwards",Laptops & Computers,955,USD,466,7444904942852,MintCream,XL,limited_stock,11 +79,Smart Grill Trimmer,Both old letter national remain of.,"Pacheco, Villanueva and Hubbard",Smartwatches,787,USD,277,6855621726839,GoldenRod,50x70 cm,backorder,10 +80,Portable Vacuum Printer Go,Say minute type others.,Espinoza-Bowman,Makeup,321,USD,766,9824059449136,PapayaWhip,50x70 cm,in_stock,98 +81,Projector,City food wind tax by woman talk.,Haynes LLC,Automotive,556,USD,896,7629500644465,LightPink,S,in_stock,81 +82,Compact Powerbank Radio,Finish note quickly appear than do.,Espinoza-Bond,Skincare,551,USD,573,2986694250021,LightSkyBlue,Large,backorder,89 +83,Pro Freezer Prime,Within what question instead politics.,Perez Group,Kids' Clothing,713,USD,62,4547536554059,Indigo,5x7 in,pre_order,57 +84,Digital Printer Webcam Wireless Edge Air,Professor fish oil wait enter because.,"Wilkinson, Dawson and Berger",Fragrances,200,USD,51,9730310602717,LavenderBlush,L,pre_order,87 +85,Trimmer Cooler,Enter plant painting.,Roth-James,Smartphones,862,USD,216,2617594893505,Chocolate,8x10 in,pre_order,86 +86,Portable Projector Portable Air,He ball common attention visit.,Bradley Inc,Bedding & Bath,560,USD,637,0533350984435,YellowGreen,XXL,out_of_stock,63 +87,Portable Powerbank,Exactly eight entire mean stuff already.,Weeks and Sons,Haircare,766,USD,284,9090165135243,OliveDrab,Extra Large,pre_order,87 +88,Smart Lock Premium,Such accept shoulder medical bill world.,Hebert Group,Headphones & Earbuds,452,USD,96,4520822883545,MidnightBlue,Medium,pre_order,85 +89,Automatic Printer Premium,Cause hit candidate.,Park-Huffman,Fitness Equipment,472,USD,108,5364368037098,MediumSpringGreen,XS,backorder,61 +90,Eco Trimmer Touch,Film simply medical parent family first.,Christensen Inc,Skincare,566,USD,277,8479741829249,OrangeRed,10x10 cm,in_stock,39 +91,Compact Powerbank Radio Lite Digital Digital,Professional kitchen move hold pay.,Brown-Berg,Beauty & Personal Care,455,USD,8,7242294927431,LightSeaGreen,Extra Large,in_stock,27 +92,Automatic Vacuum Fan,Career my difficult may court policy.,Yates-Blankenship,Sports & Outdoors,264,USD,604,9657890857804,Salmon,12x18 in,in_stock,17 +93,Digital Iron Heater 360 Clean Edge,Cell grow instead top million item might ready.,"Frazier, Mitchell and Maxwell",Kids' Clothing,65,USD,683,4795373245672,Maroon,12x18 in,discontinued,60 +94,Silent Scanner Mouse Digital Edge,Key reality hair identify third situation blue dream.,Hale Inc,Grooming Tools,711,USD,178,3621597259700,RosyBrown,Large,in_stock,15 +95,Silent Freezer Projector Webcam,Say benefit civil woman work national involve north.,Serrano PLC,Toys & Games,717,USD,962,8309548019669,Orchid,XL,pre_order,98 +96,Clean Monitor Light Plus,Whom medical five face ok believe.,Ortiz LLC,Smartwatches,668,USD,46,7586501720683,SlateGray,Extra Large,backorder,84 +97,Rechargeable Freezer Powerbank Watch Wireless Rechargeable Prime,Significant six whole different.,Casey Ltd,Furniture,411,USD,373,0917441923631,IndianRed,Extra Large,backorder,55 +98,Rechargeable Mixer Drone Fan Sense Wireless,Instead age difference you.,"Mcknight, Parsons and Haley",Laptops & Computers,784,USD,901,8807432053671,Cornsilk,M,discontinued,82 +99,Wireless Brush,Do week wall control structure.,"Pennington, York and Roberson",Cameras & Accessories,330,USD,154,1245607220621,Linen,100x200 mm,out_of_stock,72 diff --git a/tests/test_sources.py b/tests/test_sources.py index 7b7c523a..7a27a42d 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -158,6 +158,7 @@ def test_auto_attach_detach_files_tools(client: LettaSDKClient): ("tests/data/data_structures.cpp", "BinarySearchTree", r"data_structures_[a-z0-9]+\.cpp"), ("tests/data/api_server.go", "UserService", r"api_server_[a-z0-9]+\.go"), ("tests/data/data_analysis.py", "StatisticalAnalyzer", r"data_analysis_[a-z0-9]+\.py"), + ("tests/data/test.csv", "Smart Fridge Plus", r"test_[a-z0-9]+\.csv"), ], ) def test_file_upload_creates_source_blocks_correctly(