# General rules for all bots 
User-agent: *
Disallow: /story-embed/*
Disallow: /search

# Allow Google (for search, news, image and video indexing)
User-agent: Googlebot
User-agent: Googlebot-News
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
Allow: /

# Allow Bingbot (for Bing Search indexing)
User-agent: Bingbot
Allow: /

# Allow DuckDuckGo (for DuckDuckGo search indexing)
User-agent: DuckDuckBot
Allow: /

# Allow Yahoo Slurp (for Yahoo Search and some news aggregators)
User-agent: Slurp
Allow: /

# Allow Applebot (for Siri & Safari suggestions)
User-agent: Applebot
Allow: /

# Allow Facebook bots (for link previews and sharing)
User-agent: facebookexternalhit
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
Allow: /

# Allow SmartNews bot 
User-agent: SmartNews-AdsBot
User-agent: SmartNewsBot
Allow: /

# Allow Twitterbot (for Twitter link previews)
User-agent: Twitterbot
Allow: /

# Block AI & aggressive scrapers
User-agent: Amazonbot
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: CCBot
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: Diffbot
User-agent: FriendlyCrawler
User-agent: GPTBot
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: ICC-Crawler
User-agent: ImagesiftBot
User-agent: PerplexityBot
User-agent: PetalBot
User-agent: Scrapy
User-agent: Timpibot
User-agent: VelenPublicWebCrawler
User-agent: Webzio-Extended
User-agent: YouBot
User-agent: anthropic-ai
User-agent: cohere-ai
User-agent: img2dataset
User-agent: omgili
User-agent: omgilibot
Disallow: /

# Sitemaps
Sitemap: https://vocalo.org/sitemap.xml