# BrandCited — robots.txt
# Generated from src/lib/seo/ai-bots.ts. Edit there, not here.
# Policy: every reputable AI and search bot is allowed on every public
# marketing page. Disallowed paths are gated app surfaces (dashboard,
# admin, onboarding) and internal API routes. Lead-gen agent endpoints
# under /api/agent/ and /api/tools/ are explicitly re-allowed.
#
# ─── Agent discovery surfaces ───
# Beyond robots.txt, BrandCited publishes structured agent contracts:
#   https://www.brandcited.ai/llms.txt                        llmstxt.org spec directory
#   https://www.brandcited.ai/llms-full.txt                   long-form corpus
#   https://www.brandcited.ai/ai.txt                          training/retrieval policy
#   https://www.brandcited.ai/agent.json                      agent-skills index (root alias)
#   https://www.brandcited.ai/webmcp.json                     WebMCP manifest (2026-05-20 spec)
#   https://www.brandcited.ai/feed.json                       JSON Feed 1.1
#   https://www.brandcited.ai/feed.xml                        RSS 2.0
#   https://www.brandcited.ai/.well-known/agent-skills/index.json   agent-skills card
#   https://www.brandcited.ai/.well-known/mcp-server.json     MCP server card
#   https://www.brandcited.ai/.well-known/openapi.json        OpenAPI 3.1
#   https://www.brandcited.ai/.well-known/ai-plugin.json      OpenAI plugin compat
#   https://www.brandcited.ai/.well-known/c2pa.json           C2PA content credentials policy
#   https://www.brandcited.ai/.well-known/api-catalog         RFC 9727 service catalog
#   https://www.brandcited.ai/.well-known/security.txt        security contact
# Every public page also emits Vary: Accept + a multi-rel Link header
# pointing at the surfaces above, so agents that probe headers find
# them without parsing HTML.

User-agent: *
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# ─── AI Search Crawlers and Training Bots ───
# One block per bot for self-documentation: a reader can see at a glance
# which engines are explicitly allowed and which paths each is gated on.

# Google — Google Search primary crawler
User-agent: Googlebot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Google — Google Images
User-agent: Googlebot-Image
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Google — Google News
User-agent: Googlebot-News
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Google — Google Video / YouTube discovery
User-agent: Googlebot-Video
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Google — Gemini training + Google AI corpora (distinct from Search)
User-agent: Google-Extended
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Google — Google internal research / one-off crawls
User-agent: GoogleOther
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Google — NotebookLM source ingestion
User-agent: Google-NotebookLM
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Google — Browser-using AI agents (Project Mariner) — distinct from Googlebot and Google-Extended (announced May 2026)
User-agent: Google-Agent
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Google — Journalism research tool / Pinpoint document ingestion
User-agent: Google-Pinpoint
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Google — Chrome Web Store extension verification
User-agent: Google-CWS
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Google — Google Ads landing-page quality crawl
User-agent: AdsBot-Google
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Google — Google Shopping product feed validation
User-agent: Storebot-Google
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Microsoft — Bing Search + Copilot ground truth
User-agent: Bingbot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Microsoft — Bing snapshot previews
User-agent: BingPreview
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Microsoft — Legacy MSN crawler (still respected)
User-agent: msnbot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# OpenAI — Training data for GPT models
User-agent: GPTBot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# OpenAI — SearchGPT real-time indexing
User-agent: OAI-SearchBot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# OpenAI — Real-time fetch on behalf of a ChatGPT user
User-agent: ChatGPT-User
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Anthropic — Training data crawler
User-agent: ClaudeBot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Anthropic — Real-time search indexing
User-agent: Claude-SearchBot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Anthropic — Real-time fetch on behalf of a Claude user
User-agent: Claude-User
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Anthropic — Legacy UA (pre-rebrand) — still in some training pipelines
User-agent: anthropic-ai
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Perplexity — Indexing crawler
User-agent: PerplexityBot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Perplexity — Real-time fetch during user queries
User-agent: Perplexity-User
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Apple — Siri + Spotlight search
User-agent: Applebot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Apple — Apple Intelligence training (distinct from Applebot)
User-agent: Applebot-Extended
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# DuckDuckGo — DuckDuckGo search index
User-agent: DuckDuckBot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# DuckDuckGo — DuckAssist AI answers
User-agent: DuckAssistBot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Meta — Llama training + Meta AI retrieval
User-agent: Meta-ExternalAgent
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Meta — On-demand fetches for Meta AI
User-agent: Meta-ExternalFetcher
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Meta — Open Graph + sharing previews
User-agent: FacebookBot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Yandex — Yandex Search + Alice + YandexGPT
User-agent: YandexBot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Yandex — Yandex image index
User-agent: YandexImages
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# ByteDance — Doubao / Toutiao / TikTok AI training
User-agent: Bytespider
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Mistral — Real-time retrieval for Le Chat
User-agent: MistralAI-User
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Cohere — Cohere Command training
User-agent: cohere-ai
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Cohere — Newer Cohere training UA
User-agent: cohere-training-data-crawler
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# You.com — You.com search + AI modes
User-agent: YouBot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Amazon — Alexa, Amazon Q, Rufus
User-agent: Amazonbot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Brave — Brave Search + Leo AI
User-agent: Brave-Search-Bot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Kagi — Kagi premium search + assistant
User-agent: Kagibot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Diffbot — Entity / Knowledge Graph extraction (feeds many AI products)
User-agent: Diffbot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# TheHive — Image entity extraction
User-agent: ImagesiftBot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Common Crawl — Open dataset that feeds most open-source LLMs
User-agent: CCBot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Baidu — Baidu search + ERNIE AI training (China)
User-agent: Baiduspider
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Baidu — Baidu academic / specialized index
User-agent: Baidu-YJK
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Naver — Naver search (Korea)
User-agent: NaverBot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Naver — Naver image and content crawler (Korea)
User-agent: Yeti
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Sogou — Sogou search + AI (China)
User-agent: Sogou web spider
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Sogou — Sogou AI training corpus
User-agent: Sogou-AI
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Qwant — Qwant search (EU privacy-focused)
User-agent: Qwantify
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Yandex — Yandex secondary crawler / YandexGPT training
User-agent: YandexAdditional
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Huawei — Huawei Petal Search + Petal AI
User-agent: PetalBot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Yisou — Yisou search (China) — used by Quark AI
User-agent: YisouSpider
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Phind — Phind developer-focused AI search
User-agent: PhindBot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Komo — Komo AI search assistant
User-agent: KomoBot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Vectara — Vectara RAG infrastructure crawler
User-agent: VectaraCrawler
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Andi — Andi conversational search
User-agent: Andibot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Snowflake (ex-Neeva) — Snowflake / former Neeva search corpus
User-agent: NeevaBot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Friendly — Friendly AI content crawler
User-agent: FriendlyCrawler
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Awario — Awario brand-mention monitoring (feeds many GEO products)
User-agent: AwarioRssBot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# iAsk — iAsk.ai conversational answer engine
User-agent: iaskspider
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Ahrefs — Ahrefs index (used by Ahrefs Brand Radar GEO product)
User-agent: AhrefsBot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

# Semrush — Semrush index (used by Semrush AI Visibility)
User-agent: SemrushBot
Allow: /
Allow: /api/agent/
Allow: /api/tools/
Disallow: /dashboard
Disallow: /admin
Disallow: /onboarding
Disallow: /api/
Disallow: /design-preview
Disallow: /nav-preview
Disallow: /homepage-preview
Disallow: /dev/
Disallow: /auth/
Disallow: /accept-invite
Disallow: /share/
Disallow: /monitoring
Disallow: /ingest

Sitemap: https://www.brandcited.ai/sitemap.xml
Sitemap: https://www.brandcited.ai/sitemap-news.xml
Sitemap: https://www.brandcited.ai/sitemap-images.xml
Host: www.brandcited.ai