# robots.txt for https://thecatalystsgroup.com/ # Default rules for all crawlers User-agent: * Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php # Allow specific Other crawlers User-agent: Googlebot User-agent: Bingbot Allow: / # Allow specific AI crawlers User-agent: ChatGPT-User User-agent: OAI-SearchBot User-agent: GPTBot User-agent: ClaudeBot User-agent: Claude-Web User-agent: Claude-SearchBot User-agent: PerplexityBot User-agent: Gemini-Deep-Research User-agent: Google-Extended User-agent: SellbirdBot Allow: / Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php # Allow social preview crawlers (needed for Open Graph / link previews) User-agent: facebookexternalhit User-agent: Facebot User-agent: Twitterbot User-agent: LinkedInBot User-agent: WhatsApp User-agent: Slackbot Allow: / # Sitemap (change to match your SEO plugin if different) Sitemap: https://thecatalystsgroup.com/sitemap_index.xml