#
# Robots.txt file https://www.agriholland.nl
#
Sitemap: https://www.agriholland.nl/sitemap.xml
User-agent: Sphider-agriholland
Disallow: /images/
User-agent: Googlebot
Disallow: /images/
User-agent: Gulliver
Disallow: /images/
User-agent: htdig
Disallow: /images/
User-agent: InfoSeek
Disallow: /images/
User-agent: Jeeves
Disallow: /images/
User-Agent: Jooblebot
Disallow: /images/
User-agent: LookSmart
Disallow: /images/
User-agent: INGRID 
Disallow: /images/
User-agent: LycosNL
Disallow:/images/
User-agent: Vagabondo
Disallow:/images/
User-agent: Mozilla/5.0 (Slurp/cat; slurp@inktomi.com; http://www.inktomi.com/slurp.html)
Disallow: /images/
User-agent: Nederland.zoek 
Disallow: /images/
User-agent: Scooter 
Disallow: /images/
User-agent: Scooter/1.0 
Disallow: /images/
User-agent: Slurp 
Disallow: /images/
User-agent: WebFerret
Disallow: /images/
User-agent: Snooper
Disallow: /images/
User-agent: MSNBOT
Disallow: /images/
User-agent: Surfbot 
Disallow: /images/
User-agent: woelmuis.nl
Disallow: /images/
User-agent: Yahoo! Slurp 
Disallow: /images/
User-agent: Mediapartners-Google
Disallow: /images/
# deze staat er op speciaal verzoek in:
User-agent: WizeNozeSpider
Disallow: /images/
# Helemaal blokkeren
User-agent: AhrefsBot
Disallow: /

# Block MJ12bot as it is just noise
User-agent: MJ12bot
Disallow: /

User-agent: Baiduspider
Disallow: /

# Block CazoodleBot as it does not present correct accept content headers
User-agent: CazoodleBot
Disallow: /

# Block dotbot as it cannot parse base urls properly
User-agent: dotbot/1.0
Disallow: /
User-agent: ezooms
Disallow: /

# Block Gigabot
User-agent: Gigabot
Disallow: /

User-agent: Mediatoolkitbot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /
User-agent: SeznamBot
Disallow: /

User-agent: sogou spider
Disallow: /

User-agent: Sogou web spider
Disallow: /

# Block trendkite-akashic-crawler
User-agent: trendkite-akashic-crawler
Disallow: /

User-agent: *
Disallow: /