# vcbacked.co robots.txt - Optimized for SEO and GEO (AI search engines)
User-agent: *
# Disallow rules come first so first-match parsers cannot read "Allow: /" as a blanket pass.
# Blocked: admin/API/internal routes, /jobs?<filters> (canonicalise to /jobs; the crawlable
# jobs surface is /jobs, /jobs/<role|series|remote>, /jobs/in/<city>), and duplicate query patterns.
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /jobs?*
Disallow: /auth/
Disallow: /debug
Disallow: /*?page=*&*
Disallow: /*?*&page=*
Disallow: /*?utm_*
Disallow: /*?ref=*
Disallow: /*?source=*

# Priority crawling - High value pages
Allow: /directory/
Allow: /directory/cities/
Allow: /directory/industries/
Allow: /directory/investors/
Allow: /directory/funding/
Allow: /directory/companies/
Allow: /recently-funded-companies
Allow: /startup-founder-contacts
Allow: /

# Allow AI search bots for citation and search indexing
User-agent: GPTBot
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /jobs?*
Allow: /

User-agent: OAI-SearchBot
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /jobs?*
Allow: /

User-agent: ChatGPT-User
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /jobs?*
Allow: /

User-agent: ClaudeBot
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /jobs?*
Allow: /

User-agent: Claude-SearchBot
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /jobs?*
Allow: /

User-agent: anthropic-ai
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /jobs?*
Allow: /

User-agent: Claude-Web
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /jobs?*
Allow: /

User-agent: PerplexityBot
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /jobs?*
Allow: /

User-agent: Perplexity-User
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /jobs?*
Allow: /

User-agent: Google-Extended
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /jobs?*
Allow: /

User-agent: Applebot
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /jobs?*
Allow: /

User-agent: Applebot-Extended
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /jobs?*
Allow: /

User-agent: DuckAssistBot
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /jobs?*
Allow: /

User-agent: meta-externalagent
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /jobs?*
Allow: /

# Meta search indexer (distinct token from meta-externalagent); it was crawling /jobs?filters
User-agent: meta-webindexer
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /jobs?*
Allow: /

# AI training/corpus crawlers — welcome (being in training data is long-term AEO;
# aligned with angelbacked's stance, Jul 2026)
User-agent: CCBot
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /jobs?*
Allow: /

User-agent: Amazonbot
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /jobs?*
Allow: /

# Bytespider stays blocked: ignores robots.txt, pure bulk scraper
User-agent: Bytespider
Disallow: /

# Sitemap locations
Sitemap: https://www.vcbacked.co/sitemap.xml
Sitemap: https://www.vcbacked.co/sitemap-static.xml
Sitemap: https://www.vcbacked.co/sitemap-cities.xml
Sitemap: https://www.vcbacked.co/sitemap-industries.xml
Sitemap: https://www.vcbacked.co/sitemap-investors.xml
Sitemap: https://www.vcbacked.co/sitemap-funding.xml
Sitemap: https://www.vcbacked.co/sitemap-companies-1.xml
Sitemap: https://www.vcbacked.co/sitemap-companies-2.xml
Sitemap: https://www.vcbacked.co/sitemap-companies-3.xml
Sitemap: https://www.vcbacked.co/sitemap-companies-4.xml
Sitemap: https://www.vcbacked.co/sitemap-companies-5.xml
Sitemap: https://www.vcbacked.co/sitemap-companies-6.xml
Sitemap: https://www.vcbacked.co/sitemap-companies-7.xml
Sitemap: https://www.vcbacked.co/sitemap-companies-8.xml
Sitemap: https://www.vcbacked.co/sitemap-city-industry.xml
Sitemap: https://www.vcbacked.co/sitemap-investor-industry.xml
Sitemap: https://www.vcbacked.co/sitemap-funding-city.xml
Sitemap: https://www.vcbacked.co/sitemap-funding-industry.xml
Sitemap: https://www.vcbacked.co/sitemap-yc.xml
Sitemap: https://www.vcbacked.co/sitemap-daily.xml