# Allow discovery and AI search indexing
User-agent: *
Allow: /
User-agent: OpenAI
Allow: /search
Allow: /index
User-agent: Anthropic
Allow: /search
Allow: /index
User-agent: Perplexity
Allow: /search
Allow: /index
User-agent: Google DeepMind
Allow: /search
Allow: /index
User-agent: Mistral
Allow: /search
Allow: /index
User-agent: OpenAI
Disallow: /dataset
User-agent: Anthropic
Disallow: /dataset
User-agent: Perplexity
Disallow: /dataset
User-agent: Google DeepMind
Disallow: /dataset
User-agent: Mistral
Disallow: /dataset