User-agent: *
Disallow: /site/snap
Disallow: /site/gethardata
Disallow: /site/pdf

User-agent: Dotbot
Disallow:/
User-agent: BLEXBot
Disallow:/
User-agent: Barkrowler/0.9
Disallow:/
User-agent: Semrush
Disallow:/

User-agent: SemrushBot
Disallow:/