# Perfect World — crawler policy
#
# This is a civic-transparency platform. Public records are meant to be read,
# found and cited, so general search crawlers are welcome on the human surface
# and we steer them rather than block them.
#
# What is NOT welcome is bulk harvesting of the corpus by unattended agents.
# That surface is /api/*, it is governed by an API key and the machine terms at
# /terms-of-service/#machine-access, and it is disallowed here for everyone. A crawler walking the
# ~1,006 city markets drained the API connection pool and wedged the service on
# 2026-08-15; that is the behaviour this file asks for an end to.
#
# robots.txt is advisory. It is the polite notice. The enforcement lives in
# the API key tier and the rate limiter — see /terms-of-service/#machine-access.

# ---------------------------------------------------------------- everyone
User-agent: *
Allow: /
Crawl-delay: 10

# The machine surface. Use the API with a key, not the crawler.
Disallow: /api/

# Member, desk and admin surfaces — nothing here is public anyway.
Disallow: /account
Disallow: /admin
Disallow: /desk

# Bulk and generated artifacts: these are the corpus, not pages.
Disallow: /*/exports
Disallow: /*.jsonl.gz$
Disallow: /*.csv$
Disallow: /*format=csv
Disallow: /*format=json

# Expiring one-time download tokens. Crawling these burns a member's link.
Disallow: /bookstore/download/
Disallow: /bodega/download/

# -------------------------------------------------- AI training collectors
# Declined for training use. Not hostile to the companies — the corpus is a
# licensable asset (docs/ai-licensing-strategy.md) and licensing it is a
# conversation, not a scrape. Terms and contact: /terms-of-service/#machine-access

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Timpibot
Disallow: /

# ------------------------------------------------- AI SEARCH, deliberately
# allowed on the same terms as Google. These fetch to CITE and link, which is
# how a transparency project gets read. They are a different bot from the
# training collectors above, which is exactly why both lists exist.

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/

User-agent: Claude-SearchBot
Allow: /
Disallow: /api/

User-agent: Claude-User
Allow: /
Disallow: /api/

User-agent: PerplexityBot
Allow: /
Disallow: /api/

# Sitemap: none is published yet. The line goes here the day
# sitemap.xml exists — an absent sitemap is better than a 404 one.