User-agent: gptbot
User-agent: GPTBot
User-agent: amazonbot
User-agent: Amazonbot
User-agent: AhrefsBot
User-agent: meta-externalagent
Disallow: /

User-agent: dotbot
User-agent: YandexBot
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: GrapeshotCrawler
User-agent: CriteoBot
User-agent: comscore
User-agent: proximic
User-agent: admantx
User-agent: integralads
User-agent: PetalBot
User-agent: TTD-Content
User-agent: bot
Disallow:
Allow: /
Crawl-delay: 10

User-agent: Googlebot
Disallow: 
Allow: /
Crawl-delay:

# updated 2025-04-24