# robots.txt — marketing host (www.bridgetown.builders)
#
# SCOPE: This file governs the public marketing and documentation pages only.
# It does NOT apply to the app host (app.bridgetown.builders), API, or MCP
# server. Those hosts maintain their own access controls and should default
# to disallowing all crawlers until explicitly configured.
#
# POLICY SUMMARY:
#   - Standard search crawlers: allowed (drives organic traffic).
#   - AI search / retrieval crawlers: allowed (these cite sources and send
#     referral traffic; directly benefits product awareness).
#   - AI training-only crawlers: allowed (no product/legal decision to block;
#     revisit if policy changes). See the "training" section below.
#   - Private app paths are not served from this host, so no Disallow rules
#     for /api or /dashboard are needed here.

# ---
# Default: allow all unrecognised crawlers.
# Explicit sections below override for named agents.
# ---
User-agent: *
Allow: /

# ---
# Standard search engine crawlers
# ---

# Google web search index
User-agent: Googlebot
Allow: /

# Google AI features (Gemini, SGE, etc.)
# Allowed: public marketing pages are appropriate content for Google AI.
User-agent: Google-Extended
Allow: /

# Apple web crawler (Siri, Spotlight, Apple AI features)
User-agent: Applebot-Extended
Allow: /

# DuckDuckGo AI assistant (DuckAssist)
User-agent: DuckAssistBot
Allow: /

# ---
# AI search and retrieval crawlers
# These crawlers power AI assistants that cite sources and send referral
# traffic. Allowing them is directly beneficial to product awareness.
# ---

# OpenAI ChatGPT search (web browsing / retrieval)
User-agent: OAI-SearchBot
Allow: /

# OpenAI ChatGPT user-initiated browsing
User-agent: ChatGPT-User
Allow: /

# Anthropic Claude general web crawler
User-agent: ClaudeBot
Allow: /

# Anthropic Claude web search
User-agent: Claude-Web
Allow: /

# Anthropic general crawler identity
User-agent: anthropic-ai
Allow: /

# Perplexity AI search crawler
User-agent: PerplexityBot
Allow: /

# Perplexity user-initiated browsing
User-agent: Perplexity-User
Allow: /

# ---
# AI training crawlers
# These crawlers are used primarily to build training datasets with no direct
# referral traffic benefit. Current policy: allow (marketing content is public
# and non-proprietary). Revisit with product/legal if this changes.
# ---

# OpenAI training data crawler (GPT model training; not web search or ChatGPT browsing)
# Distinct from OAI-SearchBot (ChatGPT search) and ChatGPT-User (user-triggered browsing).
User-agent: GPTBot
Allow: /

# Amazon AI crawler (Alexa, Bedrock training data)
User-agent: Amazonbot
Allow: /

# Common Crawl (open dataset widely used for LLM pre-training)
User-agent: CCBot
Allow: /

# ByteDance / TikTok crawler
User-agent: Bytespider
Allow: /

# Meta AI crawler (Llama, Meta AI training data)
User-agent: Meta-ExternalAgent
Allow: /

Sitemap: https://www.bridgetown.builders/sitemap-index.xml