40 lines
2.1 KiB
Plaintext
40 lines
2.1 KiB
Plaintext
# Azure Settings
|
|
AZURE_ENDPOINT=https://iva-open-ai.openai.azure.com/
|
|
AZURE_OPENAI_API_KEY=your-azure-openai-key
|
|
AZURE_OPENAI_API_VERSION=2024-08-01-preview
|
|
AZURE_DEPLOYMENT=text-embedding-3-small
|
|
|
|
VECTOR_STORE_ADDRESS=https://iva-demo-vector-service.search.windows.net
|
|
VECTOR_STORE_PASSWORD=your-store-password
|
|
|
|
INDEX_NAME=iva-vector-demo
|
|
RETRY_TOTAL=4
|
|
|
|
# Crawler settings
|
|
CRAWLER_LOADER=web # web or file
|
|
CRAWLER_COMPANY_NAME=Morton
|
|
CRAWLER_COMPANY_PDF_DIR=files
|
|
CRAWLER_COMPANY_URL=https://blog.mortons.site/
|
|
CRAWLER_COMPANY_FILE=data/verint-responsible-ethical-ai.pdf
|
|
FIRECRAWL_API_KEY=your-firecrawl-api-key
|
|
FIRECRAWL_API_URL=http://localhost:3002
|
|
FIRECRAWL_MODE=crawl
|
|
# Pass in the parameters as a JSON string
|
|
#
|
|
# include_paths (Optional[List[str]]): Patterns of URLs to include
|
|
# exclude_paths (Optional[List[str]]): Patterns of URLs to exclude
|
|
# max_depth (Optional[int]): Maximum crawl depth
|
|
# max_discovery_depth (Optional[int]): Maximum depth for finding new URLs
|
|
# limit (Optional[int]): Maximum pages to crawl
|
|
# allow_backward_links (Optional[bool]): Follow parent directory links
|
|
# allow_external_links (Optional[bool]): Follow external domain links
|
|
# ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
|
|
# scrape_options (Optional[ScrapeOptions]): Page scraping configuration (see https://docs.firecrawl.dev/advanced-scraping-guide#scrape-options)
|
|
# webhook (Optional[Union[str, WebhookConfig]]): Notification webhook settings
|
|
# deduplicate_similar_urls (Optional[bool]): Remove similar URLs
|
|
# ignore_query_parameters (Optional[bool]): Ignore URL parameters
|
|
# regex_on_full_url (Optional[bool]): Apply regex to full URLs
|
|
# delay (Optional[int]): Delay in seconds between scrapes
|
|
# poll_interval (Optional[int]): Seconds between status checks (default: 2)
|
|
# idempotency_key (Optional[str]): Unique key to prevent duplicate requests
|
|
FIRECRAWL_PARAMS='{ "limit": 100, "include_paths": ["/.*"], "ignore_sitemap": true, "poll_interval": 5, "scrape_options": { "headers": { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.4 Safari/605.1.15"}}}' |