# Allow all major search engine bots
User-agent: *
Disallow: /check-jobs-table.js
Disallow: /reset-password.html

Sitemap: https://pxlworld.com/sitemap.xml

# Block known AI bots
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: dataforseo
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: FacebookBot
Disallow: /

# Optional: Block archivers / general scraping
User-agent: archive.org_bot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /