# Robots.txt para blitznocode.com
# Última actualización: Agosto 2025

# ============================================
# REGLAS GENERALES - BOTS LEGÍTIMOS
# ============================================

# Permitir todos los bots legítimos por defecto
User-agent: *
Allow: /
Crawl-delay: 1

# ============================================
# BOTS DE BÚSQUEDA PRINCIPALES
# ============================================

# Google
User-agent: Googlebot
Allow: /
Crawl-delay: 0

# Bing
User-agent: Bingbot
Allow: /
Crawl-delay: 0

# ============================================
# BOTS DE LLM / AI TRAINING
# ============================================

# OpenAI (GPT)
User-agent: GPTBot
Allow: /

# OpenAI SearchBot
User-agent: ChatGPT-User
Allow: /

# Anthropic (Claude)
User-agent: anthropic-ai
Allow: /
User-agent: Claude-Web
Allow: /

# Google Bard/Gemini
User-agent: Google-Extended
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Common Crawl (usado por muchos LLMs)
User-agent: CCBot
Allow: /

# Internet Archive
User-agent: ia_archiver
Allow: /

# ============================================
# DIRECTORIOS Y ARCHIVOS A PROTEGER
# ============================================

User-agent: *
# Archivos de configuración y sistema
Disallow: /.env
Disallow: /.git/
Disallow: /.github/
Disallow: /node_modules/
Disallow: /.vscode/
Disallow: *.config.js
Disallow: /package*.json
Disallow: /.gitignore

# Áreas administrativas (si existen)
Disallow: /admin/
Disallow: /api/private/
Disallow: /dashboard/
Disallow: /login
Disallow: /register
Disallow: /reset-password

# Archivos temporales y de desarrollo
Disallow: /temp/
Disallow: /tmp/
Disallow: /test/
Disallow: /tests/
Disallow: /_next/
Disallow: /build/
Disallow: /dist/

# Páginas de búsqueda y filtros (evitar contenido duplicado)
Disallow: /search?
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?page=
Disallow: /*&

# Archivos sensibles por extensión
Disallow: /*.sql$
Disallow: /*.log$
Disallow: /*.bak$
Disallow: /*.zip$
Disallow: /*.tar$
Disallow: /*.gz$

# ============================================
# SITEMAPS
# ============================================

# Sitemap principal
Sitemap: https://blitznocode.com/sitemap.xml

# Si tienes sitemaps específicos (descomenta si los usas)
# Sitemap: https://blitznocode.com/sitemap-pages.xml
# Sitemap: https://blitznocode.com/sitemap-blog.xml
# Sitemap: https://blitznocode.com/sitemap-products.xml

# ============================================
# BOTS MALICIOSOS A BLOQUEAR
# ============================================

# Scrapers agresivos conocidos
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# Bots de spam
User-agent: Zealbot
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

# ============================================
# NOTAS IMPORTANTES
# ============================================

# 1. Este archivo permite el acceso a bots de IA/LLM principales
# 2. Protege directorios sensibles y archivos de configuración
# 3. Incluye crawl-delay para evitar sobrecarga del servidor
# 4. Bloquea bots maliciosos conocidos
# 5. Recuerda actualizar los sitemaps según tu estructura

Sitemap: https://www.blitznocode.com/sitemap.xml