User-agent: Baiduspider
User-agent: Yandex
User-agent: Clickagy Intelligence Bot v2
User-agent: DotBot
User-agent: Sogou web spider
User-agent: Updownerbot
User-agent: SurveyBot
User-agent: webmeupbot
User-agent: sogou spider
User-agent: seoscanners.net
User-agent: PetalBot
Disallow: /

User-agent: AhrefsBot
Crawl-Delay: 2

User-Agent: bingbot
Crawl-delay: 1
Disallow: /salaries/all/
Disallow: /pensions/all/
Disallow: /salaries/search/
Disallow: /pensions/search/

User-Agent: Verity/1.1 (https://gumgum.com/verity; verity-support@gumgum.com)
Allow: /

User-agent: *
Disallow: /salaries/all/
Disallow: /pensions/all/
Disallow: /salaries/search/
Disallow: /pensions/search/

Sitemap: https://transcal.s3.amazonaws.com/public/sitemaps/sitemap-index.xml