feat: Add paginated memory queries (#825)

Co-authored-by: cpacker <packercharles@gmail.com>
This commit is contained in:
Sarah Wooders
2024-01-15 21:21:58 -08:00
committed by GitHub
parent a0a72a0faf
commit f47e800982
8 changed files with 131 additions and 26 deletions

View File

@@ -260,8 +260,7 @@ class SQLStorageConnector(StorageConnector):
all_filters = [getattr(self.db_model, key) == value for key, value in filter_conditions.items()]
return all_filters
def get_all_paginated(self, filters: Optional[Dict] = {}, page_size: Optional[int] = 1000) -> Iterator[List[Record]]:
offset = 0
def get_all_paginated(self, filters: Optional[Dict] = {}, page_size: Optional[int] = 1000, offset=0) -> Iterator[List[Record]]:
filters = self.get_filters(filters)
while True:
# Retrieve a chunk of records with the given page_size