# For all robots
User-agent: *

# Block access to specific groups of pages
Disallow: /cart
Disallow: /checkout
Disallow: /my-account
Disallow: /my-company
Disallow: /section
Disallow: /Section/
Disallow: /search/suggest
Disallow: /shop/search
Disallow: /search
Disallow: /shop/cart/addToCart*

# Allow search crawlers to discover the sitemap
#Sitemap: /sitemap.xml

# Block CazoodleBot as it does not present correct accept content headers
User-agent: CazoodleBot
Disallow: /

# Block dotbot as it cannot parse base urls properly
User-agent: dotbot/1.0
Disallow: /

# Block Gigabot
User-agent: Gigabot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: ltx71
Disallow: /

User-agent: 360Spider
Disallow: /

User-agent: sogou spider
Disallow: /

User-agent: Seznam screenshot-generator 2.1
Disallow: /

User-agent: YisouSpider
Disallow: /

User-agent: IstellaBot
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: semvisuBot 2.0
Disallow: /

User-agent: MJ12bot
Crawl-delay: 30

User-agent: archive.org_bot
Crawl-delay: 10

User-agent: filterdb.iss.net/crawler
Crawl-delay: 10

User-agent: meanpathbot
Crawl-delay: 10

User-agent: msnbot
Crawl-delay: 10

User-agent: aihitdata
Crawl-delay: 30

User-agent: BLEXBot
Crawl-delay: 10

User-agent: Slurp
Crawl-delay: 20

User-agent: bingbot
Crawl-delay: 10

User-agent: Wespe.de
Disallow: /

User-agent: Superarama v.01
Disallow: /

User-agent: AdsBot-Google
Disallow: /checkout/