User-agent: *
Allow: /
Sitemap: https://www.marshallwhite.com.au/sitemap-index.xml
Host: https://www.marshallwhite.com.au
User-agent: Googlebot
# Allow files critical for rendering
Allow: *.js
Allow: *.css
# Allow AJAX - Do Not Remove -
#Prevent private admin areas from being crawled
Disallow: *admin.datocms.com*
# Prevent duplicate /feed/ pages from being crawled
Disallow: /*/feed/
# Prevent Trackback Neg SEO
Disallow: /trackback/
#These rules apply to everyone else.
User-agent:*
# A list of misbehaving crawlers.
User-agent: OAI-SearchBot
Disallow: /
User-agent: GPTBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
User-agent: DOC
Disallow: /
User-agent: Download Ninja
Disallow: /
User-agent: Zao
Disallow: /
# Some bots are known to be trouble, particularly those designed to copy entire sites.
User-agent: Fetch
Disallow: /
User-agent: HTTrack
Disallow: /
User-agent: larbin
Disallow: /
User-agent: libwww.
Disallow: /
User-agent: linko
Disallow: /
User-agent: Microsoft.URL.Control
Disallow: /