# -----------------------------------------
# ALLOW MAJOR SEARCH & AD CRAWLERS
# -----------------------------------------

# Google Search
User-agent: Googlebot
Disallow:

# Google Mobile
User-agent: Googlebot-Mobile
Disallow:

# Google Images
User-agent: Googlebot-Image
Disallow:

# Google News
User-agent: Googlebot-News
Disallow:

# Google Video
User-agent: Googlebot-Video
Disallow:

# Google Ads Crawlers
User-agent: AdsBot-Google
Disallow:

User-agent: AdsBot-Google-Mobile
Disallow:

# Google Inspection Tool (used by Search Console)
User-agent: Google-InspectionTool
Disallow:

# GoogleOther (generic crawlers)
User-agent: GoogleOther
Disallow:

# Google’s new unified crawler (covers future Google crawlers)
User-agent: GoogleOther
Disallow:

# Google Merchant Center Crawler
User-agent: Mediapartners-Google
Disallow:

# Bing
User-agent: Bingbot
Disallow:

# DuckDuckGo
User-agent: DuckDuckBot
Disallow:

# Yahoo
User-agent: Slurp
Disallow:

# Apple
User-agent: Applebot
Disallow:

# Yandex *optional* – if the site targets US only, consider blocking instead
User-agent: Yandex
Disallow:

# -----------------------------------------
# BLOCK BAD/UNWANTED BOTS & SCRAPERS
# (Non-exhaustive, but covers major offenders)
# -----------------------------------------

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: Seznambot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Mail.RU_Bot
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: Baiduspider
Disallow: /