# ---------------------------------------- # # robots.txt for http://www.merton.gov.uk/ # Created by David Cook 29 November 2004 # # ---------------------------------------- User-agent: ia_archiver Disallow: User-agent: ia-archiver Disallow: User-agent: Browsershots Allow: / User-agent: London Portal Allow: / User-agent: Ultraseek Allow: / User-agent: ultraseek Allow: / User-agent: googlebot Allow: / User-agent: Mediapartners-Google Allow: / User-agent: msnbot Allow: / User-agent: slurp Allow: / User-agent: scooter Allow: / User-agent: lycos Allow: / User-agent: teoma Allow: / User-agent: robozilla Allow: / User-agent: mantraagent Allow: / User-agent: architech Allow: / User-Agent: W3C-checklink Allow: / User-agent: * Disallow: /