# robots.txt for getvisibleai.com # Last updated: July 2025 User-agent: * Allow: / # Public pages for indexing Allow: /faq Allow: /pricing Allow: /blog Allow: /blog/* Allow: /robots.txt Allow: /llms.txt Allow: /text.txt Allow: /sitemap.xml # Block private/auth-required pages from indexing Disallow: /dashboard Disallow: /admin Disallow: /auth Disallow: /verify-email Disallow: /reset-password Disallow: /content/* Disallow: /analytics Disallow: /intake Disallow: /api Disallow: /temp Disallow: /private Allow: /images # Block access to sensitive files and directories Disallow: /cgi-bin/ Disallow: /*.cgi$ Disallow: /*.pl$ Disallow: /*.php$ Disallow: /*.asp$ Disallow: /*.aspx$ Disallow: /*.inc$ Disallow: /*.sql$ Disallow: /*.bak$ Disallow: /*.zip$ Disallow: /*.tar$ Disallow: /*.gz$ Disallow: /.env Disallow: /config/ Disallow: /core/ Disallow: /includes/ Disallow: /scripts/ Disallow: /vendor/ Disallow: /node_modules/ Disallow: /composer.lock Disallow: /package.json Disallow: /package-lock.json Disallow: /yarn.lock # Allow image indexing Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.png$ Allow: /*.gif$ Allow: /*.webp$ Allow: /*.svg$ # Specific directives for major search engines and LLM bots User-agent: Googlebot Allow: / Allow: /faq Allow: /pricing Allow: /blog Allow: /blog/* Disallow: /dashboard Disallow: /admin Disallow: /auth Disallow: /private Disallow: /api Disallow: /temp User-agent: Bingbot Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: Slurp Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: DuckDuckBot Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: Baiduspider Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: Yandex Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: Sogou Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp # LLM and AI bots User-agent: GPTBot Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: OAI-SearchBot Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: ChatGPT-User Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: ChatGPT-User/2.0 Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: anthropic-ai Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: ClaudeBot Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: claude-web Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: PerplexityBot Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: Perplexity-User Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: Google-Extended Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: YouBot Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: AI2Bot Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: MistralAI-User Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: cohere-ai Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: CCBot Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: Amazonbot Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: Applebot Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: Applebot-Extended Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: FacebookBot Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: meta-externalagent Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: LinkedInBot Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: Bytespider Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp User-agent: DuckAssistBot Allow: / Disallow: /admin Disallow: /private Disallow: /api Disallow: /temp # Sitemap reference Sitemap: https://getvisibleai.com/sitemap.xml