##ACAP version=1.0 # robots.txt for http://www.thisislondon.co.uk # version 1.1 User-agent: * Crawl-delay: 3 Disallow: /fame/list.html Disallow: /fame/detail.html Disallow: /nydailynews/ # robots.txt for http://www.thisislondon.co.uk # version 1.1 ACAP-crawler: * # User-agent: * Sitemap: http://www.thisislondon.co.uk/sitemap.xml Sitemap: http://www.thisislondon.co.uk/news-sitemap.xml