verint.com/rag-system
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adding scrape_options

Browse Source
This commit is contained in:
Peter Morton 2025-05-27 17:54:11 -05:00
parent 8eba52df20
commit 696669aae4
1 changed files with 8 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
rag_system/loaders/web_loader.py
View File

@ -5,6 +5,7 @@ import logging
from langchain_community.document_loaders import WebBaseLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from rag_system.loaders.firecrawl import FireCrawlLoader
from firecrawl import ScrapeOptions
from dotenv import load_dotenv
@ -20,6 +21,13 @@ firecrawl_params = os.getenv("FIRECRAWL_PARAMS")
if firecrawl_params:
firecrawl_params = json.loads(firecrawl_params)
if firecrawl_params["scrape_options"]:
firecrawl_params["scrape_options"] = ScrapeOptions(
**firecrawl_params["scrape_options"]
)
logging.info(f"web_loader firecrawl_params: {firecrawl_params}")
logging.info(f"web_loader firecrawl_api_url: {firecrawl_api_url}")
logging.info(f"web_loader firecrawl_mode: {firecrawl_mode}")
logging.info(f"web_loader firecrawl_params: {firecrawl_params}")