User-agent: *
Disallow: /bja/
Disallow: /plugins/
Disallow: /framework/
Disallow: /vendor/
Disallow: /r/aanbieding
Disallow: /static/

Sitemap: https://www.luier.nl/sitemap.xml

# AI crawlers rate-limiten
User-agent: ChatGPT-User
Crawl-delay: 2

User-agent: GPTBot
Crawl-delay: 2

User-agent: Google-Extended
Crawl-delay: 2

User-agent: ClaudeBot
Crawl-delay: 2

User-agent: anthropic-ai
Crawl-delay: 2

User-agent: PerplexityBot
Crawl-delay: 2

User-agent: Bytespider
Crawl-delay: 2

User-agent: CCBot
Crawl-delay: 2

User-agent: meta-externalagent
Crawl-delay: 2

User-agent: cohere-ai
Crawl-delay: 2

# SEO crawlers rate-limiten
User-agent: DotBot
Crawl-delay: 2

User-agent: AhrefsBot
Crawl-delay: 2

User-agent: SemrushBot
Crawl-delay: 2

User-agent: MJ12bot
Crawl-delay: 2

# Overige bots rate-limiten
User-agent: Googlebot
Crawl-delay: 2

User-agent: bingbot
Crawl-delay: 2

User-agent: Applebot
Crawl-delay: 2

User-agent: OAI-SearchBot
Crawl-delay: 2

User-agent: AwarioBot
Crawl-delay: 2

User-agent: Amazonbot
Crawl-delay: 2

User-agent: YandexBot
Crawl-delay: 2

User-agent: Baiduspider
Crawl-delay: 2