# ================================
# robots.txt for pxfuse mirror site
# ================================

# ==== AI / LLM Crawlers ====
User-agent: GPTBot
Disallow: /
Crawl-delay: 10

User-agent: bingbot
Disallow: /
Crawl-delay: 10

User-agent: ChatGPT-User
Disallow: /
Crawl-delay: 10

User-agent: ChatGPT-Collector
Disallow: /
Crawl-delay: 10

User-agent: ClaudeBot
Disallow: /
Crawl-delay: 10

User-agent: Claude-Web
Disallow: /
Crawl-delay: 10

User-agent: Google-Extended
Disallow: /
Crawl-delay: 10

User-agent: PerplexityBot
Disallow: /
Crawl-delay: 10

User-agent: FacebookBot
Disallow: /
Crawl-delay: 10

User-agent: Applebot-Extended
Disallow: /
Crawl-delay: 10

User-agent: CCBot
Disallow: /
Crawl-delay: 10

User-agent: Diffbot
Disallow: /
Crawl-delay: 10

User-agent: anthropic-ai
Disallow: /
Crawl-delay: 10

# ==== Search / SEO Crawlers ====
User-agent: SemrushBot
Disallow: /
Crawl-delay: 10

User-agent: AhrefsBot
Disallow: /
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /
Crawl-delay: 10

User-agent: dotbot
Disallow: /
Crawl-delay: 10

User-agent: YandexBot
Disallow: /
Crawl-delay: 10

User-agent: Baiduspider
Disallow: /
Crawl-delay: 10

User-agent: PetalBot
Disallow: /
Crawl-delay: 10

User-agent: archive.org_bot
Disallow: /
Crawl-delay: 10

# ==== E-commerce / Big Tech Bots ====
User-agent: Amazonbot
Disallow: /
Crawl-delay: 10


User-agent: Bytespider
Disallow: /
Crawl-delay: 10

User-agent: TikTokBot
Disallow: /
Crawl-delay: 10

User-agent: Applebot
Disallow: /
Crawl-delay: 10

User-agent: Googlebot
Disallow: /
Crawl-delay: 10