User-agent: *
Allow: /
Disallow: /dashboard
Disallow: /settings
Disallow: /api/
Disallow: /backoffice
Disallow: /verify-build
Disallow: /brand-memory
Disallow: /examples
Disallow: /_preview/

# AI training and data crawlers — ALLOWED (decision 2026-10-06, Tomer-approved).
# Reasoning: allowing these does not change Google ranking or ChatGPT/Claude live answers
# (those use the separate search and user agents listed below). Allowing them may help
# models learn the BlueVeta brand and support Gemini grounding.
# Each gets the same internal-page rules as every other agent here.
User-agent: GPTBot
Allow: /
Disallow: /dashboard
Disallow: /settings
Disallow: /api/
Disallow: /backoffice
Disallow: /verify-build
Disallow: /brand-memory
Disallow: /examples
Disallow: /_preview/

User-agent: ClaudeBot
Allow: /
Disallow: /dashboard
Disallow: /settings
Disallow: /api/
Disallow: /backoffice
Disallow: /verify-build
Disallow: /brand-memory
Disallow: /examples
Disallow: /_preview/

User-agent: Google-Extended
Allow: /
Disallow: /dashboard
Disallow: /settings
Disallow: /api/
Disallow: /backoffice
Disallow: /verify-build
Disallow: /brand-memory
Disallow: /examples
Disallow: /_preview/

User-agent: Amazonbot
Allow: /
Disallow: /dashboard
Disallow: /settings
Disallow: /api/
Disallow: /backoffice
Disallow: /verify-build
Disallow: /brand-memory
Disallow: /examples
Disallow: /_preview/

User-agent: Applebot-Extended
Allow: /
Disallow: /dashboard
Disallow: /settings
Disallow: /api/
Disallow: /backoffice
Disallow: /verify-build
Disallow: /brand-memory
Disallow: /examples
Disallow: /_preview/

# Scrapers and other crawlers — BLOCKED (kept as before, decision 2026-10-06).
# Listed here explicitly because the Cloudflare managed block that used to cover them is
# being turned off, and this file now carries the whole policy.
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: KimiBot
Disallow: /

User-agent: AwarioRssBot
Disallow: /

User-agent: AwarioSmartBot
Disallow: /

# Other training crawlers that the Cloudflare managed list blocked today, kept blocked.
User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: MistralAI-Training
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: GoogleOther
Disallow: /

User-agent: Google-CloudVertexBot
Disallow: /

# AI answer/search agents — explicitly allowed. These fetch a page live
# to answer a real user's question (e.g. "what tool can turn one product
# photo into a marketing campaign") — this is the path that actually
# drives BlueVeta being suggested by ChatGPT/Claude/Perplexity. Each gets
# its own block (robots.txt agent blocks don't inherit from User-agent: *)
# so the same internal-page rules apply here too.
User-agent: OAI-SearchBot
Allow: /
Disallow: /dashboard
Disallow: /settings
Disallow: /api/
Disallow: /backoffice
Disallow: /verify-build
Disallow: /brand-memory
Disallow: /examples
Disallow: /_preview/

User-agent: ChatGPT-User
Allow: /
Disallow: /dashboard
Disallow: /settings
Disallow: /api/
Disallow: /backoffice
Disallow: /verify-build
Disallow: /brand-memory
Disallow: /examples
Disallow: /_preview/

User-agent: Claude-SearchBot
Allow: /
Disallow: /dashboard
Disallow: /settings
Disallow: /api/
Disallow: /backoffice
Disallow: /verify-build
Disallow: /brand-memory
Disallow: /examples
Disallow: /_preview/

User-agent: Claude-User
Allow: /
Disallow: /dashboard
Disallow: /settings
Disallow: /api/
Disallow: /backoffice
Disallow: /verify-build
Disallow: /brand-memory
Disallow: /examples
Disallow: /_preview/

User-agent: PerplexityBot
Allow: /
Disallow: /dashboard
Disallow: /settings
Disallow: /api/
Disallow: /backoffice
Disallow: /verify-build
Disallow: /brand-memory
Disallow: /examples
Disallow: /_preview/

User-agent: Perplexity-User
Allow: /
Disallow: /dashboard
Disallow: /settings
Disallow: /api/
Disallow: /backoffice
Disallow: /verify-build
Disallow: /brand-memory
Disallow: /examples
Disallow: /_preview/

# AI/LLM crawlers: see https://blueveta.com/llms.txt for a structured
# summary of what BlueVeta is and which pages matter.

Sitemap: https://blueveta.com/sitemap.xml