User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cgi-bin/
Crawl-delay: 30

# Sitemap location
Sitemap: https://worldstainless.org/sitemap_index.xml

# === Allow: OpenAI Bots (GPTBot, ChatGPT-User, OAI-SearchBot)
User-agent: GPTBot
Crawl-delay: 10
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cgi-bin/

User-agent: ChatGPT-User
Crawl-delay: 10
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cgi-bin/

User-agent: OAI-SearchBot
Crawl-delay: 10
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cgi-bin/

# === Allow: Anthropic Claude
User-agent: ClaudeBot
Crawl-delay: 10
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cgi-bin/

User-agent: anthropic-ai
Crawl-delay: 10
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cgi-bin/

# === Allow: Perplexity AI
User-agent: PerplexityBot
Crawl-delay: 10
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cgi-bin/

User-agent: Perplexity-User
Crawl-delay: 10
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cgi-bin/

# === Allow: Google LLM Crawling
User-agent: Google-Extended
Crawl-delay: 10
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cgi-bin/

# === Allow: Bing Copilot via Bingbot
User-agent: bingbot
Crawl-delay: 10
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cgi-bin/

# === Allow: ByteDance LLMs (TikTok, Toutiao)
User-agent: Bytespider
Crawl-delay: 10
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cgi-bin/

# === Allow: Cohere AI
User-agent: cohere-ai
Crawl-delay: 10
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cgi-bin/

# === Allow: Zhipu AI (ChatGLM)
User-agent: ChatGLM-Spider
Crawl-delay: 10
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cgi-bin/