verint.com/rag-system
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updated chuck_size and overlap as they were too small

Browse Source
This commit is contained in:
Peter Morton
2025-06-02 18:41:08 -05:00
parent f0503e7aac
commit fde60f3341
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
rag_system/loaders/web_loader.py
View File

@@ -52,7 +52,7 @@ def load_web_crawl(url):
# docs_list = [item for sublist in docs for item in sublist] # docs_list = [item for sublist in docs for item in sublist]
# Initialize a text splitter with specified chunk size and overlap # Initialize a text splitter with specified chunk size and overlap
text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder( text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
chunk_size=250, chunk_overlap=0 chunk_size=1000, chunk_overlap=200
) )
# Split the documents into chunks # Split the documents into chunks
return text_splitter.split_documents(docs) return text_splitter.split_documents(docs)