# robots.txt for https://proindies.com
# Pro Indies — Strategic websites & growth agency
# We WELCOME AI assistants, answer engines, and search crawlers.
# Goal: be findable and citable inside ChatGPT, Claude, Gemini,
# Perplexity, Copilot, Google AI Overviews, and traditional search.
# ----------------------------------------------------------------------

# ======================================================================
# AI ASSISTANTS & ANSWER ENGINES  (training, indexing, live retrieval)
# Explicitly allowed so our content can be read, indexed, and cited.
# ======================================================================

# --- OpenAI / ChatGPT ---
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# --- Anthropic / Claude ---
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

# --- Google (Gemini, AI Overviews, Vertex AI) ---
User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

# --- Microsoft / Copilot & Bing ---
User-agent: bingbot
Allow: /

# --- Perplexity ---
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# --- Apple Intelligence / Siri ---
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# --- Meta AI ---
User-agent: Meta-ExternalAgent
Allow: /

User-agent: meta-externalagent
Allow: /

# --- Amazon (Alexa / Nova) ---
User-agent: Amazonbot
Allow: /

# --- Other answer engines & AI search ---
User-agent: DuckAssistBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Diffbot
Allow: /

User-agent: Timpibot
Allow: /

# --- Common Crawl (dataset many LLMs are trained on) ---
User-agent: CCBot
Allow: /

# ======================================================================
# TRADITIONAL SEARCH ENGINES
# ======================================================================
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

# ======================================================================
# EVERYONE ELSE
# Open by default. Only internal/raw asset folders are de-prioritized.
# ======================================================================
User-agent: *
Allow: /
Disallow: /integrations/
Disallow: /meeting/

# Be polite, but don't throttle the AI/search bots above.
Crawl-delay: 1

# ======================================================================
# DISCOVERY
# ======================================================================
Sitemap: https://proindies.com/sitemap.xml

# llms.txt — a plain-language guide to this site for AI assistants:
# https://proindies.com/llms.txt

Host: proindies.com