# robots.txt for justinforjustice.com
# Farahi Law Firm, APC
# Last updated: September 2026 — AI training crawlers opened (previously blocked)

# =============================================================
# DEFAULT RULES — All Crawlers
# =============================================================
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /README.txt
Disallow: /license.txt
Disallow: /wp-config.php
# Note: hard-blocking /?s= and /search/ is the safe default, but it also
# means Google can't recrawl an already-indexed spam search URL to see a
# noindex tag on it. If spam query URLs show up indexed in GSC, add
# noindex via Yoast/theme first, THEN keep this Disallow.
Disallow: /?s=
Disallow: /search/
Disallow: /*?replytocom=
Disallow: /*?utm_*
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /trackback/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Allow: /wp-includes/js/
Allow: /wp-includes/css/

# =============================================================
# AI SEARCH & CITATION CRAWLERS — ALLOWED
# These crawlers cite your content in AI search results
# and drive referral traffic. Critical for visibility.
# =============================================================

# OpenAI ChatGPT Search
User-agent: OAI-SearchBot
Allow: /

# OpenAI ChatGPT user-initiated browsing
User-agent: ChatGPT-User
Allow: /

# Perplexity AI search
User-agent: PerplexityBot
Allow: /

# Perplexity user-initiated browsing
User-agent: Perplexity-User
Allow: /

# Anthropic Claude search
User-agent: Claude-SearchBot
Allow: /

# Anthropic Claude user-initiated browsing
User-agent: Claude-User
Allow: /

# Google AI Overviews & Gemini
User-agent: Google-Extended
Allow: /

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /

# Amazon Alexa+ / Rufus AI shopping answers
User-agent: Amazonbot
Allow: /

# You.com AI search
User-agent: YouBot
Allow: /

# =============================================================
# SEO AUDIT TOOLS — ALLOWED
# Used for our own site audits, rank tracking, and competitor research.
# =============================================================

# Semrush organic/paid research crawler
User-agent: SemrushBot
Allow: /

# Semrush Site Audit tool
User-agent: SemrushBot-SA
Allow: /

# Semrush Backlink Audit tool
User-agent: SemrushBot-BA
Allow: /

# Ahrefs backlink/organic crawler
User-agent: AhrefsBot
Allow: /

# Ahrefs Site Audit tool
User-agent: AhrefsSiteAudit
Allow: /

# =============================================================
# AI TRAINING CRAWLERS — ALLOWED
# Opted in to AI model training access.
# =============================================================

# OpenAI training crawler
User-agent: GPTBot
Allow: /

# Common Crawl (used by many AI training datasets)
User-agent: CCBot
Allow: /

# Anthropic training crawler
User-agent: anthropic-ai
Allow: /

# Anthropic ClaudeBot (training)
User-agent: ClaudeBot
Allow: /

# ByteDance / TikTok training crawler
User-agent: Bytespider
Allow: /

# Meta / Facebook AI training
User-agent: Meta-ExternalAgent
Allow: /

# Cohere training crawler
User-agent: cohere-ai
Allow: /

# =============================================================
# LOW-VALUE / SPAM SCRAPERS — BLOCKED
# Bots that scrape content with no SEO or business benefit.
# =============================================================

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

# =============================================================
# SITEMAP
# =============================================================
Sitemap: https://justinforjustice.com/sitemap_index.xml