User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile-Apps
Allow: /

User-agent: Baiduspider*
User-agent: Sogou*
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
User-agent: Amazonbot
User-agent: BLEXBot
User-agent: CCBot
User-agent: Cliqzbot
User-agent: dotbot
User-agent: ia_archiver
User-agent: MBCrawler
User-agent: MJ12bot
User-agent: PetalBot
User-agent: Photon
User-agent: Scrapy
User-agent: SemrushBot
User-agent: SeznamBot
User-agent: Linguee
User-agent: Similarweb
User-agent: SnapchatAds
User-agent: facebookexternalhit
User-agent: meta-externalagent
User-agent: crawler
User-agent: Barkrowler
User-agent: BLEXBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: *
Allow: /*.html

Sitemap: /post-sitemap.xml