# robots.txt for StarSymbols Site

# Allow all crawlers access to all content by default
User-agent: *
Allow: /

# Sitemap location
Sitemap: https://starsymbols.io/sitemap.xml

# --- Specific rules for AI crawlers ---
# This includes general AI bots and specific ones like Google's Vertex AI.
User-agent: GPTBot
User-agent: Claude-Web
User-agent: Anthropic-AI
User-agent: PerplexityBot
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: DuckAssistBot

# Guide AI crawlers to the structured context file
LLM-Content: https://starsymbols.io/llms.txt

# Allow AI crawlers access to all key sections of the site
Allow: /
Allow: /all-star-symbols/
Allow: /fancy-text-generator/
Allow: /blog/
Allow: /image-to-text/
Allow: /about/
Allow: /privacy/
Allow: /terms/
Allow: /copyright/

# Disallow access to admin or sensitive areas (if any)
Disallow: /admin/
Disallow: /private/
Disallow: /*.json$

# Crawl delay to be respectful of server resources
Crawl-delay: 1