# ======================================================
# robots.txt for https://calculadoraaliciaonline.net
# Optimized for SEO, performance, and AI visibility
# Last updated: 2025-10-29
# ======================================================

# ===============================
# 1) Default rule for all crawlers
# ===============================
User-agent: *
# Allow full crawl of all public pages
Disallow:

# ===============================
# 2) Block backend and low-value system paths
# ===============================
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-includes/
Disallow: /readme.html

# Allow necessary AJAX for front-end functionality
Allow: /wp-admin/admin-ajax.php

# ===============================
# 3) Block duplicate or crawl-waste URLs
# ===============================
Disallow: /*?s=             # Internal search results
Disallow: /*?replytocom     # Duplicate comment URLs
Disallow: /*?utm_           # Tracking parameters
Disallow: /*?fbclid=        # Facebook click IDs
Disallow: /?author=         # Optional: prevent author archives

# ===============================
# 4) Allow trusted search + AI crawlers
# ===============================
User-agent: Googlebot
Disallow:

User-agent: Bingbot
Disallow:

User-agent: Google-Extended
Disallow:                    # Allows AI Overviews / Bard data access

User-agent: ChatGPT-User
Disallow:                    # Allows ChatGPT Browse visibility

User-agent: PerplexityBot
Disallow:                    # Allows Perplexity AI citations

User-agent: ClaudeBot
Disallow:                    # Allows Anthropic (Claude) access

# ===============================
# 5) Manage aggressive / bandwidth-heavy SEO bots
# (Keep visibility, reduce server load)
# ===============================
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 15

User-agent: MJ12bot
Crawl-delay: 10

User-agent: DataForSeoBot
Crawl-delay: 15

User-agent: SEOkicks-Robot
Crawl-delay: 10

User-agent: DotBot
Crawl-delay: 10

User-agent: BLEXBot
Crawl-delay: 15

# Block low-value or resource-heavy crawlers completely
User-agent: archive.org_bot
Disallow: /

User-agent: SeekportBot
Disallow: /

# ===============================
# 6) Sitemap location
# ===============================
Sitemap: https://calculadoraaliciaonline.net/sitemap_index.xml