# Pipefort — https://pipefort.com
#
# Crawling and AI use policy. Pipefort is open to search engines, AI answer
# engines, and autonomous agents: everything public here is meant to be found,
# cited, and acted on. See /.well-known/api-catalog for the machine-readable
# API index and /sitemap.xml for canonical pages.
#
# Content Signals (https://contentsignals.org/) express how this content may be
# used AFTER it is fetched. Ours are permissive on every axis:
#   search    = appear in search results and link back here
#   ai-input  = ground AI answers (RAG, retrieval, summarization) in this content
#   ai-train  = use this content to train or fine-tune models

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
# Authenticated console (app.pipefort.com). Client-rendered and behind login —
# nothing to index, and crawling it only burns crawl budget.
Disallow: /dashboard
Disallow: /repositories
Disallow: /findings
Disallow: /triage
Disallow: /alerts
Disallow: /insights
Disallow: /secrets
Disallow: /inventory
Disallow: /remediation
Disallow: /rules
Disallow: /settings
Disallow: /landscape
Disallow: /connect
Disallow: /login
# Private/authenticated API surface. The public, documented endpoints below it
# are explicitly re-allowed (longest match wins, RFC 9309 §2.2.2).
Disallow: /api/
Allow: /api/health
Allow: /api/public/
Allow: /api/public-scan/
Allow: /api/public-stats
Allow: /api/badge/

# --- AI crawlers, named explicitly so the policy is unambiguous ---------------
# Each group repeats the directives above because RFC 9309 §2.2.1 gives a
# crawler exactly one group: the most specific match, with no inheritance
# from User-agent: *.

# OpenAI — training crawler
User-agent: GPTBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /dashboard
Disallow: /repositories
Disallow: /findings
Disallow: /triage
Disallow: /alerts
Disallow: /insights
Disallow: /secrets
Disallow: /inventory
Disallow: /remediation
Disallow: /rules
Disallow: /settings
Disallow: /landscape
Disallow: /connect
Disallow: /login

# OpenAI — search index / live user fetches
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /dashboard
Disallow: /repositories
Disallow: /findings
Disallow: /triage
Disallow: /alerts
Disallow: /insights
Disallow: /secrets
Disallow: /inventory
Disallow: /remediation
Disallow: /rules
Disallow: /settings
Disallow: /landscape
Disallow: /connect
Disallow: /login

# Anthropic
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /dashboard
Disallow: /repositories
Disallow: /findings
Disallow: /triage
Disallow: /alerts
Disallow: /insights
Disallow: /secrets
Disallow: /inventory
Disallow: /remediation
Disallow: /rules
Disallow: /settings
Disallow: /landscape
Disallow: /connect
Disallow: /login

# Google — Gemini / Vertex AI grounding (separate from Googlebot)
User-agent: Google-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /dashboard
Disallow: /repositories
Disallow: /findings
Disallow: /triage
Disallow: /alerts
Disallow: /insights
Disallow: /secrets
Disallow: /inventory
Disallow: /remediation
Disallow: /rules
Disallow: /settings
Disallow: /landscape
Disallow: /connect
Disallow: /login

# Perplexity, Apple, Meta, Common Crawl, Amazon, Cohere
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: CCBot
User-agent: Amazonbot
User-agent: cohere-ai
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /dashboard
Disallow: /repositories
Disallow: /findings
Disallow: /triage
Disallow: /alerts
Disallow: /insights
Disallow: /secrets
Disallow: /inventory
Disallow: /remediation
Disallow: /rules
Disallow: /settings
Disallow: /landscape
Disallow: /connect
Disallow: /login

Sitemap: https://pipefort.com/sitemap.xml