# Cairns Tour Advice & Booking Centre
# https://www.cairnstouradvice.com
#
# AI Data Layer (machine-readable):
#   /ai                       — AI-readable HTML landing
#   /ai/openapi.json          — OpenAPI 3.1 spec
#   /api/public/...           — no-auth public read-only API
#   /llms.txt                 — site purpose declaration

# Allow all standard search engine crawlers
User-agent: *
Allow: /
Allow: /ai
Allow: /api/public/
Disallow: /admin/
Disallow: /api/
Disallow: /dev/
Disallow: /confirm-payment/
Disallow: /provider-status/
Disallow: /affiliate/dashboard
Disallow: /api/__tour-data-export
Disallow: /api/__sitemap-data
Disallow: /__crawl-check

# Legacy WordPress URL patterns (old site — all return 410 Gone)
Disallow: /wp-
Disallow: /feed
Disallow: /category/
Disallow: /author/
Disallow: /search/
Disallow: /xmlrpc.php
Disallow: /index.php
Disallow: /payment-page-
Crawl-delay: 1

# Google (no crawl-delay - Google ignores it and uses Search Console settings)
User-agent: Googlebot
Allow: /
Allow: /ai
Allow: /api/public/
Disallow: /admin/
Disallow: /api/
Disallow: /dev/
Disallow: /wp-
Disallow: /feed
Disallow: /category/
Disallow: /author/
Disallow: /search/
Disallow: /xmlrpc.php
Disallow: /index.php
Disallow: /payment-page-

# Bing (important: ChatGPT Search uses Bing's index)
User-agent: Bingbot
Allow: /
Allow: /ai
Allow: /api/public/
Disallow: /admin/
Disallow: /api/
Disallow: /dev/
Disallow: /wp-
Disallow: /feed
Disallow: /category/
Disallow: /payment-page-
Crawl-delay: 1

# ChatGPT Search (OpenAI search results - NOT AI training)
User-agent: OAI-SearchBot
Allow: /
Allow: /ai
Allow: /api/public/
Disallow: /admin/
Disallow: /api/
Disallow: /dev/

# ChatGPT browsing agent
User-agent: ChatGPT-User
Allow: /
Allow: /ai
Allow: /api/public/
Disallow: /admin/
Disallow: /api/

# Perplexity AI search
User-agent: PerplexityBot
Allow: /
Allow: /ai
Allow: /api/public/
Disallow: /admin/
Disallow: /api/
Disallow: /dev/

# Google AI (Gemini search)
User-agent: Google-Extended
Allow: /
Allow: /ai
Allow: /api/public/
Disallow: /admin/
Disallow: /api/

# Apple AI (Siri, Apple Intelligence)
User-agent: Applebot
Allow: /
Allow: /ai
Allow: /api/public/
Disallow: /admin/
Disallow: /api/

User-agent: Applebot-Extended
Allow: /
Allow: /ai
Allow: /api/public/
Disallow: /admin/
Disallow: /api/

# Cohere AI search
User-agent: cohere-ai
Allow: /
Allow: /ai
Allow: /api/public/
Disallow: /admin/
Disallow: /api/

# Anthropic Claude crawler — explicitly allowed for the AI Data Layer
User-agent: ClaudeBot
Allow: /
Allow: /ai
Allow: /api/public/
Disallow: /admin/
Disallow: /api/

User-agent: anthropic-ai
Allow: /
Allow: /ai
Allow: /api/public/
Disallow: /admin/
Disallow: /api/

# OpenAI GPTBot — allow access to public AI Data Layer only
User-agent: GPTBot
Allow: /ai
Allow: /api/public/
Allow: /llms.txt
Disallow: /

# Common Crawl (used in many AI training corpora) — block by default
User-agent: CCBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: meta-externalagent
Allow: /
Allow: /ai
Allow: /api/public/
Disallow: /admin/
Disallow: /api/

# Sitemap
Sitemap: https://www.cairnstouradvice.com/sitemap.xml