# ---------------------------------------------------------
# ROBOTS.TXT
# Goal: support search and AI citation visibility while
# signalling opt-outs for selected model-training crawlers.
# ---------------------------------------------------------

# ---------------------------------------------------------
# GROUP 1: TRADITIONAL SEARCH CRAWLERS
# Required for Google Search, Bing Search, Apple search,
# AI Overviews, AI Mode, Copilot-style search and organic discovery.
# ---------------------------------------------------------

User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Video
User-agent: bingbot
User-agent: Applebot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /cdn-cgi/
Disallow: /*seraph_accel_gp=

# ---------------------------------------------------------
# GROUP 2: AI SEARCH, CITATION AND USER-REQUESTED FETCHERS
# Allowed because they support AI search visibility,
# answer-engine citation, user-directed retrieval and linking.
# ---------------------------------------------------------

User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-SearchBot
User-agent: Claude-User
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /cdn-cgi/
Disallow: /*seraph_accel_gp=

# ---------------------------------------------------------
# GROUP 3: MODEL TRAINING OPT-OUT
# Block crawlers or product tokens associated with model training
# or broader AI model development rather than search/citation.
# ---------------------------------------------------------

User-agent: GPTBot
User-agent: Google-Extended
User-agent: ClaudeBot
User-agent: Applebot-Extended
Disallow: /

# ---------------------------------------------------------
# GROUP 4: HIGH-VOLUME AI TRAINING / DATASET CRAWLERS
# Block unless there is a deliberate commercial decision to allow.
# Review Amazonbot and Meta crawlers separately if social/search
# ecosystem visibility becomes a priority.
# ---------------------------------------------------------

User-agent: CCBot
User-agent: Bytespider
User-agent: Amazonbot
User-agent: FacebookBot
User-agent: meta-externalagent
User-agent: ImagesiftBot
Disallow: /

# ---------------------------------------------------------
# GROUP 5: GLOBAL CATCH-ALL
# Default access for ordinary crawlers, SEO tools and unknown bots.
# ---------------------------------------------------------

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /cdn-cgi/
Disallow: /*seraph_accel_gp=

# ---------------------------------------------------------
# SITEMAPS
# ---------------------------------------------------------

Sitemap: https://www.sexualabusecompensationadvice.org.uk/sitemap_index.xml