# Intercepta.ai robots.txt
# AI-Powered Marketing Compliance Platform
# Updated: May 2026

# ============================================
# DEFAULT RULES FOR ALL CRAWLERS
# ============================================

User-agent: *
Allow: /
Disallow: /ghost/
Disallow: /p/
Disallow: /tag/
Disallow: /author/

# ============================================
# CATEGORY 1: AI TRAINING CRAWLERS
# These fetch content to train LLMs.
# Allowing them means your content enters
# future model training data.
# ============================================

User-agent: GPTBot
Allow: /
Disallow: /ghost/
Disallow: /p/
Disallow: /tag/
Disallow: /author/

User-agent: ClaudeBot
Allow: /
Disallow: /ghost/
Disallow: /p/
Disallow: /tag/
Disallow: /author/

User-agent: Amazonbot
Allow: /
Disallow: /ghost/
Disallow: /p/
Disallow: /tag/
Disallow: /author/

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /ghost/
Disallow: /p/
Disallow: /tag/
Disallow: /author/

User-agent: CCBot
Allow: /
Disallow: /ghost/
Disallow: /p/
Disallow: /tag/
Disallow: /author/

User-agent: Bytespider
Allow: /
Disallow: /ghost/
Disallow: /p/
Disallow: /tag/
Disallow: /author/

# ============================================
# CATEGORY 2: AI SEARCH AND RETRIEVAL CRAWLERS
# These build the indexes that power AI search
# results. Blocking them removes you from AI
# generated answers.
# ============================================

User-agent: OAI-SearchBot
Allow: /
Disallow: /ghost/
Disallow: /p/
Disallow: /tag/
Disallow: /author/

User-agent: Claude-SearchBot
Allow: /
Disallow: /ghost/
Disallow: /p/
Disallow: /tag/
Disallow: /author/

User-agent: PerplexityBot
Allow: /
Disallow: /ghost/
Disallow: /p/
Disallow: /tag/
Disallow: /author/

# ============================================
# CATEGORY 3: USER-TRIGGERED FETCHERS
# These fetch your page in real time when a
# human asks an AI assistant a question.
# Note: Google-Agent ignores robots.txt but
# is included here for documentation.
# ============================================

User-agent: ChatGPT-User
Allow: /
Disallow: /ghost/
Disallow: /p/
Disallow: /tag/
Disallow: /author/

User-agent: Claude-User
Allow: /
Disallow: /ghost/
Disallow: /p/
Disallow: /tag/
Disallow: /author/

User-agent: Perplexity-User
Allow: /
Disallow: /ghost/
Disallow: /p/
Disallow: /tag/
Disallow: /author/

User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /ghost/
Disallow: /p/
Disallow: /tag/
Disallow: /author/

User-agent: Google-Agent
Allow: /
Disallow: /ghost/
Disallow: /p/
Disallow: /tag/
Disallow: /author/

# ============================================
# CATEGORY 4: OPT-OUT TOKENS
# These are not crawlers. They are directives
# that tell existing crawlers whether to use
# your content for AI training. Allow means
# you consent to AI training use.
# ============================================

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

# ============================================
# TRADITIONAL SEARCH AND DISCOVERY
# ============================================

User-agent: Applebot
Allow: /
Disallow: /ghost/
Disallow: /p/
Disallow: /tag/
Disallow: /author/

User-agent: YouBot
Allow: /
Disallow: /ghost/
Disallow: /p/
Disallow: /tag/
Disallow: /author/

# ============================================
# SITEMAP
# ============================================

Sitemap: https://intercepta.ai/sitemap.xml