# robots.txt for truthifi.com
# Last updated: 2026-09-25
# Contact: support@truthifi.com
#
# Policy:
#   - Search engines, AI search / assistant agents and AI training crawlers
#     may crawl the public site, but never app, API or utility surfaces.
#   - AI TRAINING IS WELCOME (decision D-1, 2026-09-25). Truthifi wants its
#     vocabulary and education content learned by AI models. Training
#     crawlers get the same public access as search, never the app.
#   - Named agents share ONE rule set with "*" (RFC 9309 grouped
#     user-agents), so the Disallow rules below apply to every bot that
#     is allowed in. Never give an allowed bot its own group without
#     copying these Disallow lines into it.
#   - Link-preview bots get their own group WITHOUT the tracking-parameter
#     blocks, so links shared with utm_/fbclid/gclid still unfurl.

# -----------------------------------------------
# All crawlers + named search, AI assistant and AI training agents
# -----------------------------------------------
User-agent: *
# Search engines (also feed Copilot, Siri/Spotlight, Alexa)
User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
User-agent: Amazonbot
# Anthropic (Claude)
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: anthropic-ai
# OpenAI search + user-initiated browsing (not training)
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User
# Other AI assistants / retrieval
User-agent: DuckAssistBot
User-agent: MistralAI-User
User-agent: Google-CloudVertexBot
# AI training crawlers + training control tokens — ALLOWED (decision D-1)
User-agent: GPTBot
User-agent: CCBot
User-agent: meta-externalagent
User-agent: Google-Extended
User-agent: Applebot-Extended
# Meta AI search citations + user-initiated fetches (not training)
User-agent: Meta-WebIndexer
User-agent: Meta-ExternalFetcher
Allow: /

# Block authenticated app and API surfaces
Disallow: /api/
Disallow: /app/
Disallow: /dashboard/
Disallow: /register
Disallow: /login
Disallow: /account/
Disallow: /settings/
Disallow: /onboarding/
Disallow: /internal/
Disallow: /admin/

# Block post-conversion / utility pages (no SEO value)
Disallow: /confirmation
Disallow: /thank-you
Disallow: /error
Disallow: /404

# Block staging / preview environments if served under this domain
Disallow: /staging/
Disallow: /preview/

# Block utility / asset paths that add no SEO value
Disallow: /cdn-cgi/
Disallow: /*.json$

# Block tracking-parameter variants (prevents duplicate page crawling)
Disallow: /*?*utm_
Disallow: /*?*ref=
Disallow: /*?*session
Disallow: /*?*fbclid=
Disallow: /*?*gclid=
Disallow: /*?*msclkid=
Disallow: /*?*ttclid=

# -----------------------------------------------
# Link-preview + ad-review bots (Meta, LinkedIn, X, Slack, WhatsApp,
# Telegram, Discord, Microsoft Teams, Reddit, Pinterest, Embedly) — same blocks as above EXCEPT the
# tracking-parameter rules (shared/ad links carry utm_, fbclid, gclid)
# -----------------------------------------------
User-agent: facebookexternalhit
User-agent: Meta-ExternalAds
User-agent: LinkedInBot
User-agent: Twitterbot
User-agent: Slackbot
User-agent: Slack-ImgProxy
User-agent: WhatsApp
User-agent: TelegramBot
User-agent: Discordbot
User-agent: SkypeUriPreview
User-agent: redditbot
User-agent: Pinterestbot
User-agent: Embedly
Allow: /
Disallow: /api/
Disallow: /app/
Disallow: /dashboard/
Disallow: /register
Disallow: /login
Disallow: /account/
Disallow: /settings/
Disallow: /onboarding/
Disallow: /internal/
Disallow: /admin/
Disallow: /confirmation
Disallow: /thank-you
Disallow: /error
Disallow: /404
Disallow: /staging/
Disallow: /preview/
Disallow: /cdn-cgi/
Disallow: /*.json$

# -----------------------------------------------
# Sitemaps
# -----------------------------------------------
Sitemap: https://truthifi.com/sitemap.xml