# CrossKit robots.txt — crawler foundations for Google / Bing / AI search
# Host: https://www.crosskit.top

# --- Default: public site is crawlable ---
User-agent: *
Allow: /
Allow: /sitemap.xml
Allow: /blog/feed.xml
Allow: /static/
Allow: /google
Allow: /BingSiteAuth.xml
Allow: /indexnowkey.txt
Allow: /llms.txt
Allow: /llms-full.txt

Disallow: /dashboard
Disallow: /admin
Disallow: /api/
Disallow: /login
Disallow: /register
Disallow: /backend
Disallow: /static/admin

Disallow: /checkout
Disallow: /en/checkout
Disallow: /payment
Disallow: /en/payment
Disallow: /test
Disallow: /debug
Disallow: /offline
Disallow: /unsubscribe/

Disallow: /*?ref=
Disallow: /*&ref=
Disallow: /*?utm_
Disallow: /*&utm_
Disallow: /*?token=
Disallow: /*?session=

# --- Google ---
User-agent: Googlebot
Allow: /

User-agent: Google-Extended
Allow: /

# --- Bing (Webmaster + IndexNow) ---
User-agent: bingbot
Allow: /

# --- AI / answer engines (GEO) ---
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: YouBot
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Bytespider
Allow: /

# llms.txt is the AI catalog. Policy HTML: /docs/ai and /en/docs/ai
# Public knowledge is allowed. Do not invert this file to blanket-block AI crawlers.

Sitemap: https://www.crosskit.top/sitemap.xml