# ══════════════════════════════════════════════════════════════
# SearchCans Global Robots.txt
# Optimized for search engines, AI models, and Astro+WP hybrid architecture
# Last updated: 2026-06-10 (V2.2 — narrowed sc/v1 allow, blocked blog/search/)
# ══════════════════════════════════════════════════════════════

# ── Default: Allow all crawlers ──
User-agent: *
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /login/
Disallow: /register/
Disallow: /blog/wp-admin/
Disallow: /blog/wp-login.php
Disallow: /blog/wp-includes/
Disallow: /blog/wp-content/plugins/
Disallow: /blog/wp-json/wp/
Disallow: /*?s=
Disallow: /*?p=
Disallow: /*&preview=true
# Allow only the specific BlogWidget endpoint (block the sc/ root listing)
Allow: /blog/wp-json/sc/v1/widgets
Disallow: /blog/wp-json/sc/
Disallow: /blog/search/
# Allow CSS/JS for correct rendering
Allow: /blog/wp-content/themes/
Allow: /*.css$
Allow: /*.js$
# Allow sitemaps
Allow: /sitemap*.xml
Allow: /blog/wp-sitemap*.xml
# Block duplicate indexing from tracking parameters
Disallow: /*?*utm_*
Disallow: /*?*fbclid*
Disallow: /*?*gclid*

# ══════════════════════════════════════════════════════════════
# Global Search Engines
# ══════════════════════════════════════════════════════════════

# Google
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /login/
Disallow: /register/
Disallow: /*?*utm_*
Disallow: /*?*fbclid*
Disallow: /*?*gclid*
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /
Crawl-delay: 1

User-agent: Googlebot-Video
Allow: /
Crawl-delay: 1

User-agent: Googlebot-News
Allow: /

User-agent: Storebot-Google
Allow: /

# Bing / Microsoft
User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /login/
Disallow: /register/
Crawl-delay: 1

User-agent: BingPreview
Allow: /

User-agent: MicrosoftPreview
Allow: /

User-agent: msnbot
Allow: /
Crawl-delay: 2

# Yahoo
User-agent: Slurp
Allow: /
Disallow: /api/
Crawl-delay: 2

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /
Disallow: /api/
Crawl-delay: 1

# Yandex (Russia)
User-agent: YandexBot
Allow: /
Disallow: /api/
Crawl-delay: 2

User-agent: YandexImages
Allow: /

# Baidu (China)
User-agent: Baiduspider
Allow: /
Disallow: /api/
Crawl-delay: 2

User-agent: Baiduspider-image
Allow: /

User-agent: Baiduspider-video
Allow: /

# Sogou (China)
User-agent: Sogou
Allow: /
Disallow: /api/
Crawl-delay: 2

User-agent: sogou spider
Allow: /

# 360 Search (China)
User-agent: 360Spider
Allow: /
Disallow: /api/
Crawl-delay: 2

User-agent: HaoSouSpider
Allow: /

# Bytedance / TikTok (China)
User-agent: Bytespider
Allow: /
Disallow: /api/
Crawl-delay: 2

# Naver (Korea)
User-agent: Yeti
Allow: /
Disallow: /api/
Crawl-delay: 2

# Coccoc (Vietnam)
User-agent: coccocbot
Allow: /
Disallow: /api/
Crawl-delay: 2

# Qwant (France)
User-agent: Qwantify
Allow: /
Disallow: /api/
Crawl-delay: 2

# Seznam (Czech)
User-agent: SeznamBot
Allow: /
Disallow: /api/
Crawl-delay: 2

# Brave Search
User-agent: BraveBot
Allow: /
Disallow: /api/
Crawl-delay: 2

# Mojeek (UK)
User-agent: MojeekBot
Allow: /
Disallow: /api/
Crawl-delay: 2

# PetalBot / Huawei (China)
User-agent: PetalBot
Allow: /
Disallow: /api/
Crawl-delay: 2

# Apple
User-agent: Applebot
Allow: /
Disallow: /api/

# ══════════════════════════════════════════════════════════════
# AI Search & Assistants (explicit allow for AI citation)
# ══════════════════════════════════════════════════════════════

# OpenAI (ChatGPT / SearchGPT)
User-agent: GPTBot
Allow: /
Disallow: /api/

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /
Disallow: /api/

User-agent: Claude-User
Allow: /
Disallow: /api/

User-agent: Claude-SearchBot
Allow: /
Disallow: /api/

User-agent: anthropic-ai
Allow: /
Disallow: /api/

# Google AI (Gemini)
User-agent: Google-Extended
Allow: /
Disallow: /api/

User-agent: GoogleOther
Allow: /
Disallow: /api/

User-agent: Google-CloudVertexBot
Allow: /
Disallow: /api/

User-agent: Gemini-Deep-Research
Allow: /
Disallow: /api/

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /
Disallow: /api/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /api/

User-agent: Perplexity-User
Allow: /
Disallow: /api/

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /api/

User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /api/

User-agent: Meta-WebIndexer
Allow: /
Disallow: /api/

User-agent: FacebookBot
Allow: /
Disallow: /api/

# Amazon / Alexa
User-agent: Amazonbot
Allow: /
Disallow: /api/

# Microsoft Copilot / Bing AI
User-agent: CopilotBot
Allow: /
Disallow: /api/

User-agent: bingpreview
Allow: /
Disallow: /api/

# DuckDuckGo AI
User-agent: DuckAssistBot
Allow: /
Disallow: /api/

# You.com
User-agent: YouBot
Allow: /
Disallow: /api/

# Cohere
User-agent: cohere-ai
Allow: /
Disallow: /api/

User-agent: cohere-training
Allow: /
Disallow: /api/

# DeepSeek (China)
User-agent: Deepseek
Allow: /
Disallow: /api/

User-agent: DeepSeekBot
Allow: /
Disallow: /api/

# Moonshot / Kimi (China)
User-agent: MoonshotBot
Allow: /
Disallow: /api/

# Zhipu / GLM (China)
User-agent: ChatGLMSpider
Allow: /
Disallow: /api/

# Baichuan (China)
User-agent: BaichuanBot
Allow: /
Disallow: /api/

# 01.AI / Yi (China)
User-agent: YiBot
Allow: /
Disallow: /api/

# Mistral (France)
User-agent: MistralBot
Allow: /
Disallow: /api/

# xAI (Grok)
User-agent: xAI-Grok
Allow: /
Disallow: /api/

# ══════════════════════════════════════════════════════════════
# Data / Research Crawlers
# ══════════════════════════════════════════════════════════════

# Common Crawl (open web dataset)
User-agent: CCBot
Allow: /
Disallow: /api/

# Diffbot (Knowledge Graph)
User-agent: Diffbot
Allow: /
Disallow: /api/

# AI2 (Allen Institute)
User-agent: AI2Bot
Allow: /
Disallow: /api/

User-agent: Ai2Bot-Dolma
Allow: /
Disallow: /api/

# Timpi (decentralized search)
User-agent: Timpibot
Allow: /
Disallow: /api/

# Webz.io
User-agent: webzio-extended
Allow: /
Disallow: /api/

# ══════════════════════════════════════════════════════════════
# Social Media Crawlers (link preview / sharing)
# ══════════════════════════════════════════════════════════════

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: facebookexternalhit
Allow: /

# ══════════════════════════════════════════════════════════════
# SEO & Analytics Tools
# ══════════════════════════════════════════════════════════════

User-agent: AhrefsBot
Allow: /
Crawl-delay: 2

User-agent: SemrushBot
Allow: /
Crawl-delay: 2

# ══════════════════════════════════════════════════════════════
# Block Malicious Bots
# ══════════════════════════════════════════════════════════════

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

# ══════════════════════════════════════════════════════════════
# Sitemaps
# ══════════════════════════════════════════════════════════════

Sitemap: https://www.searchcans.com/sitemap.xml
Sitemap: https://www.searchcans.com/blog/sitemap-posts.xml