# Block all crawlers for
User-agent: GPTBot
Disallow: /

User-agent: *
Disallow: /*?
Disallow: /dataValidator
Disallow: /wildcard
Disallow: /api
Disallow: /lists
Disallow: /dashboard
#disallowing all the /define pages so that others can be indexed properly - 11th Feb
Disallow: /define/*
Disallow: /define/*-*
Disallow: /define/*.*
Disallow: /word-finder/*-*
Disallow: /word-finder/*.*
Disallow: /rhyming-words/*-*
Disallow: /rhyming-words/*.*
Disallow: /adjectives/*.*
Disallow: /adjectives/*-*
Disallow: /syllables/*.*
Disallow: /syllables/*-*
Disallow: /thesaurus/*.*
Disallow: /thesaurus/*-*

Allow: /word-finder/sitemap.xml
Allow: /rhyming-words/sitemap.xml
Allow: /adjectives/sitemap.xml
Allow: /syllables/sitemap.xml
Allow: /thesaurus/sitemap.xml

#block pages causing redirect errors and soft 404s
Disallow: /r/obot
Disallow: /define/tightfitting
Disallow: /define/abls
Disallow: /define/asperated
Disallow: /define/unexplorative
Disallow: /define/adducer
Disallow: /define/expired
Disallow: /define/orthopaedist
Disallow: /define/artillerymen
Disallow: /define/against
Disallow: /define/antbear
Disallow: /define/24/7
Disallow: /define/closeup
Disallow: /define/until
Disallow: /define/backdoor
Disallow: /define/babysit
Disallow: /define/abduced
Disallow: /define/acridest
Disallow: /define/adesite
Disallow: /define/admixt
Disallow: /define/affrap
Disallow: /define/ahchoo
Disallow: /define/airworthiest
Disallow: /define/anvilled
Disallow: /define/attritting
Disallow: /define/auguster
Disallow: /define/zibit
Disallow: /rhyming-words/interspecies