User-agent: *
Disallow: 
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /tag/
Disallow: */trackback/
Disallow: */feed/
Disallow: /rss
Disallow: */comments/
Allow: /wp-admin/admin-ajax.php

Sitemap: https://www.tatvic.com/sitemap.xml 

# --- ALLOW All AI Crawlers (Comprehensive List) ---

# OpenAI (ChatGPT, GPT-4, GPTBot)
User-agent: ChatGPT-User
Allow: /
User-agent: GPTBot
Allow: /

# Google AI (Gemini / Bard / Google-Extended)
User-agent: Google-Extended
Allow: /
User-agent: Googlebot
Allow: /

# Anthropic (Claude AI)
User-agent: ClaudeBot
Allow: /
User-agent: anthropic-ai
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Meta AI (Facebook / Instagram AI indexing)
User-agent: FacebookBot
Allow: /
User-agent: MetaBot
Allow: /

# Mistral AI
User-agent: MistralBot
Allow: /

# You.com AI
User-agent: YouBot
Allow: /

# Neeva / Snowflake AI
User-agent: NeevaBot
Allow: /

# Quora Poe Platform (aggregates multiple LLMs)
User-agent: QuoraBot
Allow: /

# DuckDuckGo AI
User-agent: DuckDuckBot
Allow: /

# Andi Search
User-agent: andi-scraper
Allow: /

# HuggingFace Transformers
User-agent: HFBot
Allow: /

# Kagi AI
User-agent: kagi-bot
Allow: /

# Phind (Developer AI assistant)
User-agent: PhindBot
Allow: /

# Yandex GPT-based systems
User-agent: Yandex
Allow: /

# Microsoft Bing / Copilot AI
User-agent: Bingbot
Allow: /
User-agent: BingPreview
Allow: /

User-agent: OAI-SearchBot
User-agent: ChatGPTBot
User-agent: AdsBot-Google
User-agent: AdsIdxBot
User-agent: Google-InspectionTool
User-agent: facebookexternalhit
User-agent: Twitterbot
User-agent: LinkedInBot
 

#Inner Page, Post and media
User-agent: *
Disallow: /wp-content/uploads/
Disallow: /wp-content/uploads/2022/03/NIKHIL-GUPTA-RESUME.pdf
Disallow: /job/*?export-pdf=
Disallow: /job/
Disallow: /ga4/
Disallow: /ga4/feed/
Disallow: /blog/category/
Disallow: /experiance-letter/