# Allow AI crawlers and LLMs access to everything including sitemap-ai.txt
User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended  # Gemini AI's actual crawler
Allow: /

User-agent: Gemini-Pro
Allow: /

User-agent: CopilotBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: PhindBot
Allow: /

User-agent: AndiBot
Allow: /

User-agent: BraveAIBot
Allow: /

User-agent: GrokBot
Allow: /

User-agent: DeepSeekBot
Allow: /

User-agent: KomoBot
Allow: /

User-agent: WaldoBot
Allow: /

User-agent: ExaBot
Allow: /

User-agent: PoeBot
Allow: /

User-agent: ConsensusBot
Allow: /

User-agent: YepBot
Allow: /

User-agent: DuckAIBot
Allow: /

User-agent: ArcBot
Allow: /

User-agent: CCBot
Allow: /

# Traditional search engines - block AI sitemap but allow everything else
User-agent: Googlebot  # Traditional search indexing (NOT Gemini)
Disallow: /sitemap-ai.txt

User-agent: Bingbot
Disallow: /sitemap-ai.txt

User-agent: *
Disallow: /sitemap-ai.txt

# Block API endpoints (not pages)
Disallow: /api/

# Block expired URLs that were previously removed
Disallow: /enterprise
Disallow: /business/
Disallow: /use-cases/low-cost-international-calls
Disallow: /about
Disallow: /use-cases/call-family-abroad
Disallow: /icons/site.webmanifest
Disallow: /contact
Disallow: /caf/
Disallow: /parking.php
Disallow: /search

# Block old sitemap files only
Disallow: /sitemap-call-routes.xml

# Note: Removed /call?* and /*?country=* blocks as these could be valuable SEO pages
# Query parameters for country-specific content should be indexed for better SEO

# Sitemaps
Sitemap: https://www.yappacall.com/sitemap.xml

# AI/LLM Training Sitemap (accessible to AI crawlers only)
# AI-Sitemap: https://www.yappacall.com/sitemap-ai.txt