# ============================================
# Naaora - robots.txt
# https://www.naaora.com
# ============================================

# Sitemap location
Sitemap: https://www.naaora.com/sitemap.xml

# LLM context files
# See https://llmstxt.org for specification
# llms.txt: https://www.naaora.com/llms.txt
# llms-full.txt: https://www.naaora.com/llms-full.txt

# ============================================
# SEARCH ENGINES - Allow all
# ============================================

User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# ============================================
# AI CRAWLERS - Allow all for LLM context
# ============================================

# OpenAI - ChatGPT, GPT-4, GPT-5
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Google AI - Gemini, Bard
User-agent: Google-Extended
Allow: /

# Anthropic - Claude
User-agent: anthropic-ai
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

# Meta AI - LLaMA, Meta AI
User-agent: FacebookBot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# Microsoft - Copilot, Bing AI
User-agent: Copilot
Allow: /

User-agent: bingbot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# Apple - Siri, Spotlight, Apple Intelligence
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Amazon - Alexa
User-agent: Amazonbot
Allow: /

# Common Crawl (training data for many AI models)
User-agent: CCBot
Allow: /

# Hugging Face
User-agent: HuggingFaceBot
Allow: /

# AI Search Engines
User-agent: YouBot
Allow: /

User-agent: AI2Bot
Allow: /

# Brave Search
User-agent: BraveBot
Allow: /

# Neeva (AI search)
User-agent: NeevaBot
Allow: /

# WriteSonic, Jasper, Copy.ai crawlers
User-agent: Bytespider
Allow: /

# Mistral AI
User-agent: MistralBot
Allow: /

# xAI - Grok
User-agent: Grok
Allow: /

# Stability AI
User-agent: StabilityAI
Allow: /

# ============================================
# SOCIAL MEDIA - Allow for link previews
# ============================================

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Pinterestbot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Discordbot
Allow: /

User-agent: vkShare
Allow: /

User-agent: redditbot
Allow: /

# ============================================
# SEO TOOLS - Allow for analysis
# ============================================

User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Allow: /

User-agent: MJ12bot
Allow: /

User-agent: DotBot
Allow: /

User-agent: Screaming Frog SEO Spider
Allow: /

# ============================================
# DEFAULT - Allow everything else
# ============================================

User-agent: *
Allow: /

# ============================================
# RESTRICTED PATHS
# ============================================

# Admin paths
User-agent: *
Disallow: /admin/
Disallow: /api/
Disallow: /_/
Disallow: /private/

# Development files
Disallow: /src/
Disallow: /node_modules/
Disallow: /.git/
Disallow: /.env
Disallow: /*.map$

# ============================================
# HOST & CRAWL SETTINGS
# ============================================

Host: https://www.naaora.com