# https://www.robotstxt.org/robotstxt.html
#
# Cloudflare prepended a conflicting managed robots block in August 2026.
# The served canonical https://www.tradewink.com/robots.txt had no such block
# when checked on 2026-10-01. Recheck the served file after edge policy changes:
# a prepended group can override the rules below for some crawlers.
# ─────────────────────────────────────────────────────────────────────────────

# ── Default policy: everything public is crawlable ───────────────────────────
# Content-Signal (Cloudflare/IETF content-signals draft) states our intent
# explicitly so it survives even if a managed block is re-enabled later.
# We WANT to be indexed, cited and grounded on — organic + AI referral is the
# acquisition channel. Flip ai-train to `no` only as a deliberate decision.
User-agent: *
Content-Signal: search=yes,ai-input=yes,ai-train=yes
Allow: /
Allow: /api/og
Disallow: /signals-dashboard
Disallow: /admin
Disallow: /learn/preview/
Disallow: /dashboard
Disallow: /app
Disallow: /sso-callback
Disallow: /auth/
Disallow: /api/
# Cloudflare Email Obfuscation rewrites every plain-text address on the page
# into <a href="/cdn-cgi/l/email-protection">, which returns 404 to anything
# that does not run the decoding JS. Crawlers then report a site-wide broken
# link. The decoder still works for real visitors; only the crawl is excluded.
Disallow: /cdn-cgi/

# ── AI crawlers, assistants and answer engines ───────────────────────────────
# One group, many user-agents (RFC 9309 §2.2.1 permits stacked User-agent
# lines). Keeping a single rule body removes the drift that used to occur when
# ten near-identical copies had to be edited in lockstep.
#
# Grouped by vendor:
#   OpenAI     — GPTBot (training), OAI-SearchBot (index), ChatGPT-User (fetch)
#   Anthropic  — ClaudeBot (crawl), Claude-User (fetch), Claude-SearchBot
#   Google     — Google-Extended (Gemini grounding / training)
#   Perplexity — PerplexityBot (index), Perplexity-User (fetch)
#   Apple      — Applebot (Siri/Spotlight), Applebot-Extended (Apple Intel.)
#   Meta       — FacebookBot, meta-externalagent, meta-externalfetcher
#   Others     — Amazonbot, Bytespider, CCBot (Common Crawl → many LLMs),
#                cohere-ai, DuckAssistBot, MistralAI-User, Diffbot, AI2Bot
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: FacebookBot
User-agent: meta-externalagent
User-agent: meta-externalfetcher
User-agent: Amazonbot
User-agent: Bytespider
User-agent: CCBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: DuckAssistBot
User-agent: MistralAI-User
User-agent: Diffbot
User-agent: AI2Bot
Content-Signal: search=yes,ai-input=yes,ai-train=yes
Allow: /
Allow: /api/og
Disallow: /signals-dashboard
Disallow: /admin
Disallow: /learn/preview/
Disallow: /dashboard
Disallow: /app
Disallow: /sso-callback
Disallow: /auth/
Disallow: /api/
Disallow: /cdn-cgi/

Sitemap: https://www.tradewink.com/sitemap-index.xml

# IndexNow — Instant URL submission to Bing, Yandex, Naver, Seznam
# Key file location: https://www.tradewink.com/a8f3c91e4b7d2e6051c9f0a4d6b8e3c1.txt

# AI-readable content guide
# See https://www.tradewink.com/llms.txt for machine-readable site description