output passage in scrape_docs

This commit is contained in:
Vivian Fang
2023-10-15 19:29:53 -07:00
parent 6f3a5d8864
commit 2de9820364

View File

@@ -36,6 +36,7 @@ def extract_text_from_sphinx_txt(file_path):
})
continue
curr_token_ct += line_token_ct
curr_passage.append(line)
if curr_token_ct > PASSAGE_TOKEN_LEN:
passages.append({
'title': title,