Sitemap: https://www.bentasker.co.uk/sitemap.xml

User-agent: Amazonbot
User-agent: Applebot-Extended
User-agent: anthropic-ai
User-agent: Bytespider
User-agent: Google-Extended
User-agent: GPTBot
User-agent: CCBot
User-agent: PerplexityBot
User-agent: ChatGPT-User
User-agent: ImagesiftBot
User-agent: img2dataset
User-agent: ClaudeBot
Disallow: /


User-agent: *
Disallow: /paid/


# Excluded specifically because of it's self serving interpretation
# of robots.txt
#
# "Because it is not a crawler, WellKnownBot does not follow generic User-Agent: * crawling rules in robots.txt files."
#
# https://well-known.dev/about/
#
User-agent: WellKnownBot
Disallow: /