forage/config.example.yaml at main · jaycdev/forage · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
# ============================================================
# Forage Configuration Template
# Auto-seeded to /etc/forage/config.yaml when mounting a config directory.
# Deleting config.yaml regenerates a fresh default copy on next restart.
# Note: Docker environment variables (e.g. FORAGE_SEARXNG_URL) override config.yaml settings.
# Apply changes: docker compose restart forage
# Full reference: docs/CONFIG.md
# ============================================================

# --- Server -------------------------------------------------
server:
  host: 0.0.0.0            # bind inside the container
  port: 3672               # HTTP port
  workers: 1               # uvicorn workers (1 avoids in-memory cache & SSE session split-brain)
  log_level: info          # debug | info | warning | error

# --- Cache --------------------------------------------------
cache:
  enabled: true            # master toggle (false disables all caching)
  max_entries: 500         # global LRU key cap
  search:
    enabled: true          # cache search results
    ttl: 300               # seconds (5 min)
  extract:
    enabled: true          # cache extracted web pages
    ttl: 60                # seconds (1 min); stores full content for instant dynamic max_chars slices

# --- Tools ---------------------------------------------------
tools:
  search_name: web_search  # tool call name exposed to LLMs
  extract_name: web_extract # tool call name exposed to LLMs
  youtube_name: youtube_search  # tool call name exposed to LLMs for YouTube search
  include_favicon: false   # if true, injects favicon URLs for search results & extracts

# --- Search (SearXNG) ----------------------------------------
search:
  searxng_url: http://searxng:8080   # SearXNG instance URL (see docs/SEARXNG.md)
  default_lang: en-US                # default search language
  default_engines: [google, bing, brave, duckduckgo, qwant] # engines queried when none are specified by model
  # available_engines: auto-discovered live from SearXNG GET /config.
  # (Not needed unless manually overriding live auto-discovery)
  # available_engines: [google, qwant, qwant news, brave, bing, startpage, duckduckgo, reddit, wikipedia, youtube, github, searxng, yahoo, wikidata]
  timeout: 15              # seconds per search
  default_limit: 10        # default results returned
  max_limit: 50            # maximum results allowed
  max_snippet_chars: 350   # max length per snippet
  max_total_snippet_chars: 3000 # total snippet budget per search
  citation_style: site_name # citation style: "site_name" ([Site Name](url)), "academic" ([[1]](url)), etc.

# --- Extract ------------------------------------------------
extract:
  timeout: 30              # seconds per URL
  max_content_chars: 100000 # character cap for extracted content
  only_main_content: true  # strip boilerplate nav/footers/ads
  engine: readability     # browser engine: "readability" (Readability.js + markdown) or "trafilatura"
  user_agent: "ForageBot/0.1 (+https://github.com/fupzlito/forage)"  # static fetch UA
  respect_robots: false    # respect robots.txt rules
  force_render: false      # hybrid mode (static first, fallback to browser on JS/challenges)
  wait_for: null           # default CSS selector or delay to wait before extracting
  min_content_chars: 200   # static text below this threshold triggers browser render
  raw_content_markdown: true # true: raw_content mirrors clean markdown; false: raw HTML
  prefer_markdown: true    # negotiate Accept: text/markdown with servers
  require_max_chars: false # require LLMs to pass character budget per URL
  allow_private_ips: false   # allow URLs resolving to private/internal IPs (SSRF guard bypass)
  reddit_mirror: null        # tier 2 redlib mirror (set to a self-hosted redlib instance to enable tier 2)

  # Domain-specific overrides (override patterns match host and subdomains)
  domain_overrides:
    x.com:
      force_render: true
      network_idle_timeout: 3   # streaming: never goes idle
      timeout: 45
    twitter.com:
      force_render: true
      network_idle_timeout: 3
      timeout: 45
    instagram.com:
      force_render: true
      timeout: 45
    linkedin.com:
      force_render: true
      timeout: 45
    tiktok.com:
      force_render: true
      challenge_timeout: 30
      timeout: 60
    youtube.com:
      force_render: true
      scroll: true              # comments mount on scroll
      network_idle_timeout: 10
      engine: readability
      timeout: 60
    youtu.be:
      force_render: true
      scroll: true
      network_idle_timeout: 10
      timeout: 60
    ".amazon.*":
      force_render: true        # product pages mount buybox dynamically
      engine: readability
      timeout: 60
    bestbuy.com:
      engine: readability
      timeout: 60
    bloomberg.com:
      engine: readability
      timeout: 60
    marketwatch.com:
      engine: readability
    yahoo.com:
      engine: readability
    openai.com:
      engine: readability
    playstation.com:
      engine: readability
    coursera.org:
      engine: readability
      timeout: 60
    udemy.com:
      engine: readability
    lyft.com:
      engine: readability
    docs.aws.amazon.com:
      engine: readability
    react.dev:
      engine: readability
    git-scm.com:
      engine: readability
    reddit.com:                 # 3-tier pipeline: .json API -> Redlib mirror -> browser
      engine: readability
      timeout: 30
      # Optional: pass cookies for private/age-gated Reddit feeds (or set FORAGE_REDDIT_SESSION and FORAGE_REDDIT_TOKEN_V2 env vars)
      # cookies:
      #   reddit_session: "..."
      #   token_v2: "..."
    fandom.com:
      force_render: true
      network_idle_timeout: 10
      timeout: 60
    dailymail.co.uk:
      force_render: true
      challenge_timeout: 30
      timeout: 60
    stackoverflow.com:
      force_render: true
      challenge_timeout: 30
      timeout: 45
    zoom.us:
      force_render: true
      timeout: 45
    twitch.tv:
      force_render: true
      network_idle_timeout: 3
      timeout: 45

# --- Browser pool (Playwright/Chromium/Scrapling, in-process) --
browser:
  engine: scrapling        # "scrapling" (default; strongest anti-bot), "playwright", "patchright", or "obscura" (CDP)
  min_idle: 1              # standby instances kept warm at boot
  max_instances: 5         # maximum parallel browser extraction concurrency
  idle_timeout: 60         # seconds before an idle browser instance terminates
  headless: true
  launch_timeout: 30       # seconds allowed for browser launch
  stealth: true            # mask automation signals (navigator.webdriver, UA)
  network_idle_timeout: 5  # cap for networkidle wait in seconds
  scroll_steps: 0          # scroll passes for lazy content (0 disables; override per-domain)
  challenge_timeout: 15    # max seconds to wait for Cloudflare/Turnstile auto-resolution
  solve_cloudflare: false  # scrapling: use built-in solver (adds ~5s/page) vs fast poll (default)
  fallback_solver: true    # on anti-bot failure, retry once with solver as last resort

# --- Auth (optional) ------------------------------------------
# When enabled, API keys are passed via FORAGE_API_KEYS environment variable
auth:
  enabled: false           # when true, requires Authorization: Bearer <key>

# --- YouTube Search (Native YouTube Data API) ------------------
youtube:
  api_key: null            # YouTube Data API v3 key (or set via FORAGE_YOUTUBE_API_KEY). When set, enables youtube_search.
  default_limit: 20        # default results per search
  max_limit: 50            # max allowed results per search

# --- Prompts & Citation Guidelines ----------------------------
# Prompts and citation rules are configured in prompts.yaml (auto-seeded in /etc/forage/prompts.yaml).
# Edit prompts.yaml directly to customize tool descriptions and system instructions.