User-agent: *

Disallow: /usc/v/
Disallow: /usc/d/
Disallow: /cfr/v/
Disallow: /cfr/d/
Disallow: /search

Allow: /

# AI training crawlers (scrape for model training, not search/assistant use)
User-agent: GPTBot
User-agent: CCBot
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: ClaudeBot
User-agent: Meta-ExternalAgent
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: Omgilibot
User-agent: FacebookBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Amazonbot
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: PanguBot
User-agent: TimpiBot
User-agent: VelenPublicWebCrawler
Disallow: /

Sitemap: https://uscodex.org/sitemap.xml