# LLM-Specific Crawling Policy for ncrechner
# This file specifies rules for large language model crawlers and AI training

# Default policy for all LLM crawlers
User-agent: *
Allow: /

# OpenAI GPTBot
User-agent: GPTBot
Allow: /
Disallow: /kontakt/
Disallow: /_next/

# Anthropic Claude
User-agent: Claude-Web
Allow: /
Disallow: /kontakt/
Disallow: /_next/

# Perplexity
User-agent: Perplexity
Allow: /
Disallow: /kontakt/
Disallow: /_next/

# Google Gemini / Bard
User-agent: Gemini
Allow: /
Disallow: /kontakt/
Disallow: /_next/

# Microsoft Copilot
User-agent: Copilot
Allow: /
Disallow: /kontakt/
Disallow: /_next/

# Apple Intelligence
User-agent: AppleBot-Extended
Allow: /
Disallow: /kontakt/
Disallow: /_next/

# Restrict commercial scrapers
User-agent: Amazonbot
Allow: /

# Pages that should NOT be indexed by LLMs for training
# Contact forms, temporary redirects, and internal pages are excluded

# Crawl delay recommendation (milliseconds between requests)
Crawl-Delay: 1

# Sitemap for LLM crawlers
Sitemap: /sitemap.xml