# =========================================================
# 1. AI Search & Retrieval Bots (AI SEO)
# ALLOW these to ensure Villa Firenze appears in ChatGPT, Perplexity, 
# and other AI-driven search engines with citations.
# =========================================================
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

# =========================================================
# 2. AI Model Training & Data Scraping Bots 
# DISALLOW these to protect Villa Firenze's intellectual property 
# from being ingested into foundational models without attribution.
# (Change to 'Allow: /' only if you want to feed LLM training).
# =========================================================
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: Bytespider
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Omgilibot
Disallow: /

# =========================================================
# 3. AI Training Opt-Out Tokens
# These aren't crawlers, but protocol directives that tell 
# Google and Apple not to use your crawled data for AI training.
# Blocking them does NOT affect your standard search rankings.
# =========================================================
User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

# =========================================================
# 4. Standard Search Engines & All Other Bots
# =========================================================
User-agent: *
Allow: /

# Feeds
Disallow: /feed/
Disallow: /*/feed/
Disallow: /comments/feed/

# Crawled - not indexed cleanup
Disallow: /tag/
Disallow: /category/
Disallow: /web-story/
Disallow: /web-stories/
Disallow: /*?coblocks_pattern
Disallow: /*&coblocks_pattern
Disallow: /*coblocks_pattern_type=
Disallow: /*coblocks_pattern_category=
Disallow: /*post_type=coblocks_pattern

# =========================================================
# 5. Sitemaps
# =========================================================
Sitemap: https://villafirenzecr.com/sitemap_index.xml