# bcrumbs.net production robots.txt
# NOTE: User-agent groups are NOT additive. A bot that matches its own named
# group ignores "User-agent: *" entirely, so every group below repeats the
# privileged-route and tracking-parameter blocks. Do not remove them.

User-agent: *
Disallow: /api/
Disallow: /_section/
Disallow: /ahmad
Disallow: /ahmad.vcf
Disallow: /assets/image/cards/
Disallow: /*?*utm_
Disallow: /*?*fbclid=
Disallow: /*?*gclid=
Disallow: /*?*mc_cid=
Disallow: /*?*mc_eid=
Allow: /

# --- AI crawlers: welcomed deliberately (citations + referral traffic) ---

# OpenAI
User-agent: GPTBot
Disallow: /api/
Disallow: /_section/
Disallow: /ahmad
Disallow: /ahmad.vcf
Disallow: /assets/image/cards/
Disallow: /*?*utm_
Disallow: /*?*fbclid=
Disallow: /*?*gclid=
Disallow: /*?*mc_cid=
Disallow: /*?*mc_eid=
Allow: /

User-agent: OAI-SearchBot
Disallow: /api/
Disallow: /_section/
Disallow: /ahmad
Disallow: /ahmad.vcf
Disallow: /assets/image/cards/
Disallow: /*?*utm_
Disallow: /*?*fbclid=
Disallow: /*?*gclid=
Disallow: /*?*mc_cid=
Disallow: /*?*mc_eid=
Allow: /

User-agent: ChatGPT-User
Disallow: /api/
Disallow: /_section/
Disallow: /ahmad
Disallow: /ahmad.vcf
Disallow: /assets/image/cards/
Disallow: /*?*utm_
Disallow: /*?*fbclid=
Disallow: /*?*gclid=
Disallow: /*?*mc_cid=
Disallow: /*?*mc_eid=
Allow: /

# Anthropic
User-agent: ClaudeBot
Disallow: /api/
Disallow: /_section/
Disallow: /ahmad
Disallow: /ahmad.vcf
Disallow: /assets/image/cards/
Disallow: /*?*utm_
Disallow: /*?*fbclid=
Disallow: /*?*gclid=
Disallow: /*?*mc_cid=
Disallow: /*?*mc_eid=
Allow: /

User-agent: Claude-SearchBot
Disallow: /api/
Disallow: /_section/
Disallow: /ahmad
Disallow: /ahmad.vcf
Disallow: /assets/image/cards/
Disallow: /*?*utm_
Disallow: /*?*fbclid=
Disallow: /*?*gclid=
Disallow: /*?*mc_cid=
Disallow: /*?*mc_eid=
Allow: /

User-agent: Claude-User
Disallow: /api/
Disallow: /_section/
Disallow: /ahmad
Disallow: /ahmad.vcf
Disallow: /assets/image/cards/
Disallow: /*?*utm_
Disallow: /*?*fbclid=
Disallow: /*?*gclid=
Disallow: /*?*mc_cid=
Disallow: /*?*mc_eid=
Allow: /

# Perplexity
User-agent: PerplexityBot
Disallow: /api/
Disallow: /_section/
Disallow: /ahmad
Disallow: /ahmad.vcf
Disallow: /assets/image/cards/
Disallow: /*?*utm_
Disallow: /*?*fbclid=
Disallow: /*?*gclid=
Disallow: /*?*mc_cid=
Disallow: /*?*mc_eid=
Allow: /

User-agent: Perplexity-User
Disallow: /api/
Disallow: /_section/
Disallow: /ahmad
Disallow: /ahmad.vcf
Disallow: /assets/image/cards/
Disallow: /*?*utm_
Disallow: /*?*fbclid=
Disallow: /*?*gclid=
Disallow: /*?*mc_cid=
Disallow: /*?*mc_eid=
Allow: /

# Google Gemini / Vertex grounding only. This does NOT affect Google Search
# ranking or AI Overviews -- those follow Googlebot and the "*" group above.
User-agent: Google-Extended
Disallow: /api/
Disallow: /_section/
Disallow: /ahmad
Disallow: /ahmad.vcf
Disallow: /assets/image/cards/
Disallow: /*?*utm_
Disallow: /*?*fbclid=
Disallow: /*?*gclid=
Disallow: /*?*mc_cid=
Disallow: /*?*mc_eid=
Allow: /

# Common Crawl (feeds many downstream models)
User-agent: CCBot
Disallow: /api/
Disallow: /_section/
Disallow: /ahmad
Disallow: /ahmad.vcf
Disallow: /assets/image/cards/
Disallow: /*?*utm_
Disallow: /*?*fbclid=
Disallow: /*?*gclid=
Disallow: /*?*mc_cid=
Disallow: /*?*mc_eid=
Allow: /

# Apple Intelligence
User-agent: Applebot-Extended
Disallow: /api/
Disallow: /_section/
Disallow: /ahmad
Disallow: /ahmad.vcf
Disallow: /assets/image/cards/
Disallow: /*?*utm_
Disallow: /*?*fbclid=
Disallow: /*?*gclid=
Disallow: /*?*mc_cid=
Disallow: /*?*mc_eid=
Allow: /

Sitemap: https://bcrumbs.net/sitemap.xml