User-agent: *
Crawl-Delay: 60
Allow: /wp-admin/admin-ajax.php
Disallow: /download/*
Disallow: /*/download/*

Disallow: /cerrar-sesion/
Disallow: /en/log-out/
Disallow: /us/log-out/
Disallow: /fr/deconnexion/
Disallow: /restablecer-contrasena/
Disallow: /en/reset-password/
Disallow: /us/reset-password/
Disallow: /fr/reinitialiser-le-mot-de-passe/
Disallow: /newsletter/
Disallow: /en/newsletter/
Disallow: /us/newsletter/
Disallow: /fr/newsletter/
Disallow: /acceso-partners/
Disallow: /en/partner-access/
Disallow: /us/partner-access/
Disallow: /fr/acces-partenaires/
Disallow: /landing_base/

Disallow: /wp-admin/
Disallow: /login/
Disallow: /wp-includes/
Disallow: /trackback/
Disallow: /search
Disallow: */trackback/
Disallow: /index.php

# No rastrear copias de seguridad
Disallow: /*.git$
Disallow: /*.sql$
Disallow: /*.tgz$
Disallow: /*.gz$
Disallow: /*.tar$
Disallow: /*.svn$
Disallow: /*.bz2$
Disallow: /*.log$
Disallow: /wp-content/aiowps_backups/

# === CONFIGURACIÓN ESPECÍFICA DE BOTS ===

# Google

User-agent: Googlebot
Allow: /wp-content/uploads/
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# Googlebot para imágenes

User-agent: Googlebot-Image
Allow: /wp-content/uploads/

# Bing

User-agent: Bingbot
Crawl-delay: 1
Allow: /wp-content/uploads/

# === BOTS DE IA (PERMITIDOS) ===

# Claude (Anthropic)

User-agent: anthropic-ai
User-agent: Claude-Web
Crawl-delay: 2
Allow: /

# ChatGPT (OpenAI)

User-agent: GPTBot
User-agent: ChatGPT-User
Crawl-delay: 2
Allow: /

# Google Gemini / Bard

User-agent: Google-Extended
Crawl-delay: 2
Allow: /

# === HERRAMIENTAS SEO (PERMITIDAS) ===

# Semrush

User-agent: SemrushBot
Crawl-delay: 5
Allow: /

# Ahrefs

User-agent: AhrefsBot
Crawl-delay: 5
Allow: /

# Moz

User-agent: rogerbot
User-agent: dotbot
Crawl-delay: 5
Allow: /

# Majestic

User-agent: MJ12bot
Crawl-delay: 5
Allow: /

# === BOTS MALICIOSOS (BLOQUEADOS) ===

# Scrapers y bots agresivos

User-agent: HTTrack
User-agent: Offline Explorer
User-agent: Teleport
User-agent: WebZIP
User-agent: WebCopier
User-agent: WebReaper
User-agent: WebStripper
User-agent: EmailCollector
User-agent: EmailSiphon
User-agent: EmailWolf
User-agent: ExtractorPro
User-agent: CherryPicker
User-agent: WebBandit
User-agent: NICErsPRO
User-agent: WebmasterWorldForumBot
User-agent: Indy Library
User-agent: wget
User-agent: libwww-perl
User-agent: python-requests
User-agent: MegaIndex.ru
User-agent: BLEXBot
User-agent: AhrefsBot/5.0
Disallow: /

# Sitemap
Sitemap: https://urbiotica.com/sitemap_index.xml