User-agent: *
Allow: /
Allow: /api/adi.json
Allow: /api/indicators.json
Allow: /api/indicators/
Allow: /api/downloads/
Allow: /api/research/
Allow: /api/guides/
Disallow: /queue/
Disallow: /command-center/
Disallow: /admin/
Disallow: /apply/
Disallow: /account/
Disallow: /auth/
Disallow: /login/
Disallow: /register/
Disallow: /forgot-password/
Disallow: /reset-password/
Disallow: /api/
Disallow: /cdn-cgi/
# /embed/ is intentionally noindex iframe-widget content (no H1, no meta description, low word count by design).
# Crawlers without their own UA stanza pick those pages up as audit-noise — Googlebot and Bingbot have
# explicit Allow: /embed/ in their own blocks below, so SERP coverage is unaffected.
Disallow: /embed/

# Google
User-agent: Googlebot
Allow: /
Allow: /embed/
Allow: /api/adi.json
Allow: /api/indicators.json
Allow: /api/indicators/
Allow: /api/downloads/
Allow: /api/research/
Allow: /api/guides/
Disallow: /queue/
Disallow: /command-center/
Disallow: /admin/
Disallow: /apply/
Disallow: /account/
Disallow: /auth/
Disallow: /login/
Disallow: /register/
Disallow: /forgot-password/
Disallow: /reset-password/
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: Google-Extended
Allow: /
Allow: /api/adi.json
Allow: /api/indicators.json
Allow: /api/indicators/
Allow: /api/downloads/
Allow: /api/research/
Allow: /api/guides/
Disallow: /queue/
Disallow: /command-center/
Disallow: /admin/
Disallow: /apply/
Disallow: /account/
Disallow: /auth/
Disallow: /login/
Disallow: /register/
Disallow: /forgot-password/
Disallow: /reset-password/
Disallow: /api/
Disallow: /cdn-cgi/

# Microsoft / Bing (powers Copilot citations)
User-agent: Bingbot
Allow: /
Allow: /embed/
Allow: /api/adi.json
Allow: /api/indicators.json
Allow: /api/indicators/
Allow: /api/downloads/
Allow: /api/research/
Allow: /api/guides/
Disallow: /queue/
Disallow: /command-center/
Disallow: /admin/
Disallow: /apply/
Disallow: /account/
Disallow: /auth/
Disallow: /login/
Disallow: /register/
Disallow: /forgot-password/
Disallow: /reset-password/
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: msnbot
Allow: /
Allow: /api/adi.json
Allow: /api/indicators.json
Allow: /api/indicators/
Allow: /api/downloads/
Allow: /api/research/
Allow: /api/guides/
Disallow: /queue/
Disallow: /command-center/
Disallow: /admin/
Disallow: /apply/
Disallow: /account/
Disallow: /auth/
Disallow: /login/
Disallow: /register/
Disallow: /forgot-password/
Disallow: /reset-password/
Disallow: /api/
Disallow: /cdn-cgi/

# OpenAI
User-agent: GPTBot
Allow: /
Allow: /glossary/
Allow: /api/adi.json
Allow: /api/indicators.json
Allow: /api/indicators/
Allow: /api/downloads/
Allow: /api/research/
Allow: /api/guides/
Disallow: /queue/
Disallow: /command-center/
Disallow: /admin/
Disallow: /apply/
Disallow: /account/
Disallow: /auth/
Disallow: /login/
Disallow: /register/
Disallow: /forgot-password/
Disallow: /reset-password/
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: OAI-SearchBot
Allow: /
Allow: /glossary/
Allow: /api/adi.json
Allow: /api/indicators.json
Allow: /api/indicators/
Allow: /api/downloads/
Allow: /api/research/
Allow: /api/guides/
Disallow: /queue/
Disallow: /command-center/
Disallow: /admin/
Disallow: /apply/
Disallow: /account/
Disallow: /auth/
Disallow: /login/
Disallow: /register/
Disallow: /forgot-password/
Disallow: /reset-password/
Disallow: /api/
Disallow: /cdn-cgi/

# Anthropic
User-agent: ClaudeBot
Allow: /
Allow: /glossary/
Allow: /api/adi.json
Allow: /api/indicators.json
Allow: /api/indicators/
Allow: /api/downloads/
Allow: /api/research/
Allow: /api/guides/
Disallow: /queue/
Disallow: /command-center/
Disallow: /admin/
Disallow: /apply/
Disallow: /account/
Disallow: /auth/
Disallow: /login/
Disallow: /register/
Disallow: /forgot-password/
Disallow: /reset-password/
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: anthropic-ai
Allow: /
Allow: /glossary/
Allow: /api/adi.json
Allow: /api/indicators.json
Allow: /api/indicators/
Allow: /api/downloads/
Allow: /api/research/
Allow: /api/guides/
Disallow: /queue/
Disallow: /command-center/
Disallow: /admin/
Disallow: /apply/
Disallow: /account/
Disallow: /auth/
Disallow: /login/
Disallow: /register/
Disallow: /forgot-password/
Disallow: /reset-password/
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: Claude-User
Allow: /
Allow: /glossary/
Allow: /api/adi.json
Allow: /api/indicators.json
Allow: /api/indicators/
Allow: /api/downloads/
Allow: /api/research/
Allow: /api/guides/
Disallow: /queue/
Disallow: /command-center/
Disallow: /admin/
Disallow: /apply/
Disallow: /account/
Disallow: /auth/
Disallow: /login/
Disallow: /register/
Disallow: /forgot-password/
Disallow: /reset-password/
Disallow: /api/
Disallow: /cdn-cgi/

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Allow: /glossary/
Allow: /api/adi.json
Allow: /api/indicators.json
Allow: /api/indicators/
Allow: /api/downloads/
Allow: /api/research/
Allow: /api/guides/
Disallow: /queue/
Disallow: /command-center/
Disallow: /admin/
Disallow: /apply/
Disallow: /account/
Disallow: /auth/
Disallow: /login/
Disallow: /register/
Disallow: /forgot-password/
Disallow: /reset-password/
Disallow: /api/
Disallow: /cdn-cgi/

# Cohere
User-agent: cohere-ai
Allow: /
Allow: /api/adi.json
Allow: /api/indicators.json
Allow: /api/indicators/
Allow: /api/downloads/
Allow: /api/research/
Allow: /api/guides/
Disallow: /queue/
Disallow: /command-center/
Disallow: /admin/
Disallow: /apply/
Disallow: /account/
Disallow: /auth/
Disallow: /login/
Disallow: /register/
Disallow: /forgot-password/
Disallow: /reset-password/
Disallow: /api/
Disallow: /cdn-cgi/

# ByteDance / TikTok
User-agent: Bytespider
Allow: /
Allow: /api/adi.json
Allow: /api/indicators.json
Allow: /api/indicators/
Allow: /api/downloads/
Allow: /api/research/
Allow: /api/guides/
Disallow: /queue/
Disallow: /command-center/
Disallow: /admin/
Disallow: /apply/
Disallow: /account/
Disallow: /auth/
Disallow: /login/
Disallow: /register/
Disallow: /forgot-password/
Disallow: /reset-password/
Disallow: /api/
Disallow: /cdn-cgi/

# Apple
User-agent: Applebot
Allow: /
Allow: /api/adi.json
Allow: /api/indicators.json
Allow: /api/indicators/
Allow: /api/downloads/
Allow: /api/research/
Allow: /api/guides/
Disallow: /queue/
Disallow: /command-center/
Disallow: /admin/
Disallow: /apply/
Disallow: /account/
Disallow: /auth/
Disallow: /login/
Disallow: /register/
Disallow: /forgot-password/
Disallow: /reset-password/
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: Applebot-Extended
Allow: /
Allow: /api/adi.json
Allow: /api/indicators.json
Allow: /api/indicators/
Allow: /api/downloads/
Allow: /api/research/
Allow: /api/guides/
Disallow: /queue/
Disallow: /command-center/
Disallow: /admin/
Disallow: /apply/
Disallow: /account/
Disallow: /auth/
Disallow: /login/
Disallow: /register/
Disallow: /forgot-password/
Disallow: /reset-password/
Disallow: /api/
Disallow: /cdn-cgi/

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /
Allow: /api/adi.json
Allow: /api/indicators.json
Allow: /api/indicators/
Allow: /api/downloads/
Allow: /api/research/
Allow: /api/guides/
Disallow: /queue/
Disallow: /command-center/
Disallow: /admin/
Disallow: /apply/
Disallow: /account/
Disallow: /auth/
Disallow: /login/
Disallow: /register/
Disallow: /forgot-password/
Disallow: /reset-password/
Disallow: /api/
Disallow: /cdn-cgi/

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /
Allow: /api/adi.json
Allow: /api/indicators.json
Allow: /api/indicators/
Allow: /api/downloads/
Allow: /api/research/
Allow: /api/guides/
Disallow: /queue/
Disallow: /command-center/
Disallow: /admin/
Disallow: /apply/
Disallow: /account/
Disallow: /auth/
Disallow: /login/
Disallow: /register/
Disallow: /forgot-password/
Disallow: /reset-password/
Disallow: /api/
Disallow: /cdn-cgi/

# Amazon / Alexa
User-agent: Amazonbot
Allow: /
Allow: /api/adi.json
Allow: /api/indicators.json
Allow: /api/indicators/
Allow: /api/downloads/
Allow: /api/research/
Allow: /api/guides/
Disallow: /queue/
Disallow: /command-center/
Disallow: /admin/
Disallow: /apply/
Disallow: /account/
Disallow: /auth/
Disallow: /login/
Disallow: /register/
Disallow: /forgot-password/
Disallow: /reset-password/
Disallow: /api/
Disallow: /cdn-cgi/

# Claude Search
User-agent: Claude-SearchBot
Allow: /
Allow: /api/adi.json
Allow: /api/indicators.json
Allow: /api/indicators/
Allow: /api/downloads/
Allow: /api/research/
Allow: /api/guides/
Disallow: /queue/
Disallow: /command-center/
Disallow: /admin/
Disallow: /apply/
Disallow: /account/
Disallow: /auth/
Disallow: /login/
Disallow: /register/
Disallow: /forgot-password/
Disallow: /reset-password/
Disallow: /api/
Disallow: /cdn-cgi/

# You.com
User-agent: YouBot
Allow: /
Allow: /api/adi.json
Allow: /api/indicators.json
Allow: /api/indicators/
Allow: /api/downloads/
Allow: /api/research/
Allow: /api/guides/
Disallow: /queue/
Disallow: /command-center/
Disallow: /admin/
Disallow: /apply/
Disallow: /account/
Disallow: /auth/
Disallow: /login/
Disallow: /register/
Disallow: /forgot-password/
Disallow: /reset-password/
Disallow: /api/
Disallow: /cdn-cgi/

# Common Crawl (training corpus for many LLMs)
User-agent: CCBot
Allow: /
Allow: /api/adi.json
Allow: /api/indicators.json
Allow: /api/indicators/
Allow: /api/downloads/
Allow: /api/research/
Allow: /api/guides/
Disallow: /queue/
Disallow: /command-center/
Disallow: /admin/
Disallow: /apply/
Disallow: /account/
Disallow: /auth/
Disallow: /login/
Disallow: /register/
Disallow: /forgot-password/
Disallow: /reset-password/
Disallow: /api/
Disallow: /cdn-cgi/

Sitemap: https://americandefault.org/sitemap-index.xml