# OfoxAI Robots.txt
# https://ofox.io
# Last updated: 2026-06-09

# ===========================================
# Default: Allow all crawlers
# ===========================================
User-agent: *
Allow: /
Disallow: /api/
Disallow: /_next/data/

# ===========================================
# AI Search & Inference Bots (ALLOW)
# These bots retrieve content at inference time
# for citations and real-time answers.
# ===========================================

# OpenAI SearchBot - AI search citations
User-agent: OAI-SearchBot
Allow: /

# ChatGPT User - Real-time browsing
User-agent: ChatGPT-User
Allow: /

# Google AI Overview / Gemini
User-agent: Google-Extended
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Anthropic Claude
User-agent: ClaudeBot
Allow: /
User-agent: Claude-Web
Allow: /

# Microsoft Copilot / Bing AI
User-agent: Bingbot
Allow: /

# ===========================================
# AI Training Bots (Selective Access)
# Allow AI-specific docs, restrict other pages.
# ===========================================

# OpenAI GPTBot - Training data
User-agent: GPTBot
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /openapi.json
Allow: /sitemap-ai.json
Allow: /.well-known/
Allow: /en/
Allow: /zh/
Disallow: /api/

# Common Crawl
User-agent: CCBot
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /

# ===========================================
# IETF AIPREF (draft-ietf-aipref-attach-04)
# Forward-compatible AI usage preferences.
# When finalized, these signal permitted uses.
# ===========================================
# Content-Usage: allow=search,inference,citation
# Content-Usage: disallow=training-without-attribution

# ===========================================
# Sitemaps & AI Discovery Resources
# ===========================================
Sitemap: https://ofox.io/sitemap.xml

# AI-optimized discovery endpoints:
# llms.txt:      https://ofox.io/llms.txt
# llms-full.txt: https://ofox.io/llms-full.txt
# AI Sitemap:    https://ofox.io/sitemap-ai.json
# OpenAPI:       https://ofox.io/openapi.json
# LLM Context:   https://ofox.io/.well-known/llm-context
# Changelog:     https://ofox.io/changelog.json