# For all robots
User-agent: *

# Allow access to all pages on site
Allow: /

# Block specific pages
Disallow: /cart$
Disallow: /checkout
Disallow: /signin
Disallow: /login
Disallow: /register
Disallow: /*/review
Disallow: /search
Disallow: /my-account
Disallow: /*/c/*?q=*  #faceted navigation on category pages
Disallow: /advanced-search/
Disallow: /?s=*
Disallow: /tag/
Disallow: /*/requestSample
Disallow: /*/email
Disallow: /*?sort=
Disallow: /c/autoship*
Disallow: /c/*?q=* #faceted navigation autoship
Disallow: /*/c/*?&q=* #faceted navigation on all categories pages
Disallow: /build-a-berm
Disallow: /events
Disallow: /*?searchTerm=*
Disallow: /action/recommendations/?id=*
Disallow: /captureEmail?
Disallow: /cart/
Disallow: /cartReminder?
Disallow: /cmscockpit/
Disallow: /hmc/hybris?
Allow: /medias/Product-*
Allow: /medias/Content-*
Allow: /medias/Category-*
Allow: /medias/CustomerProduct-*
Allow: /medias/Homepage-*
Disallow: /medias/
Disallow: /p/pv/*
Disallow: /recentVisitedProducts?
Disallow: /region/citiesForZipCode?
Disallow: /region/regionsForCountry?
Disallow: /tabs/categoryTab?
Disallow: /pig/CapturePageView?
Disallow: /pig/CatalogRequestControllerCmd?
Disallow: /pig/CatalogRequestPPCView?
Disallow: /pig/CatgGroupView?
Disallow: /pig/CityPageView?
Disallow: /pig/contentpageview?
Disallow: /pig/ContentPageView?
Disallow: /pig/EmailProductPageView?
Disallow: /webapp/wcs/stores/servlet/
Disallow: /expertadvice/author/
Disallow: /expertadvice/advanced-search/
Disallow: /expertadvice/have-a-question/
Disallow: /expertadvice/join-the-conversation/
Disallow: /expertadvice/success/
Disallow: /expertadvice/tag/
Disallow: /expertadvice/category/
Disallow: /lab-area/c/5249
Disallow: /warehouse/c/5250
Disallow: /packaging/c/5251
Disallow: /operations/c/5252
Disallow: /chemical-plant/c/5253
Disallow: /exxonmobil/c/154
Disallow: /refineries/c/5254
Disallow: /j_spring_security_check
Disallow: /emailPopup?*

# Allow search crawlers to discover the sitemap
Sitemap: https://www.newpig.com/sitemap.xml

# Block CazoodleBot as it does not present correct accept content headers
User-agent: CazoodleBot
Disallow: /

# Block MJ12bot as it is just noise
User-agent: MJ12bot
Disallow: /

# Block dotbot as it cannot parse base urls properly
User-agent: dotbot/1.0
Disallow: /

# Block Gigabot
User-agent: Gigabot
Disallow: /

# Block Amazonbot
User-agent: Amazonbot
Disallow: /