# don't visit some of the dirs & files # see syntax details at http://www.robotstxt.org/robotstxt.html User-Agent: * Disallow: /php_uploads/ Disallow: /auth/ Disallow: /ads/ Disallow: /fonts/ Disallow: /kcaptcha/ Disallow: /swf/ Disallow: /common/ Disallow: /ru/common/ Disallow: /texts/ Disallow: /ru/texts/ Disallow: /openid/ # Slows down the activity of all the bots by seconds indicated User-Agent: * Crawl-Delay: 5