# See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file

# Every crawler is welcome, AI crawlers included. Named explicitly below because some
# treat an absent rule as a reason to stay away, and because search, answer engines and
# coding agents are how people find these episodes now.
User-agent: *
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Allow: /

Disallow: /users/auth/*
Disallow: /tmp/

# Search and answer engines
User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

# Model training and agent fetching
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: CCBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: meta-externalagent
Allow: /

# Agent-readable catalogue: https://llmstxt.org
# /llms.txt        index of every published episode
# /llms-full.txt   full text of the catalogue in one document
#
# Any page is available as markdown by appending .md to its URL, or by sending
# Accept: text/markdown. Responses carry Content-Signal (https://contentsignals.org)
# stating the same permissions as the directive above.

Sitemap: https://superails-public.s3.eu-central-1.amazonaws.com/sitemaps/sitemap.xml.gz