# $Id: robots.txt,v 1.7.2.3 2014/11/18 20:24:38 drumm Exp $
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/wc/robots.html
#
# For syntax checking, see:
# https://www.google.com/webmasters/tools/robots-testing-tool

User-agent: *
Crawl-delay: 20
# Directories / Paths
# Note: x dir is to prevent spidering/following affs
Disallow: /admin/
Disallow: /calibre/
Disallow: /chat/
Disallow: /comment/edit/
Disallow: /comment/reply/
Disallow: /contact/
Disallow: /dbbackup/
Disallow: /devtools/
Disallow: /doe/
Disallow: /files/
Disallow: /forums/everything-else/phrozen-crew/private-phrozen-crew/
Disallow: /forums/private-contributing-editors/
Disallow: /forums/private-moderation/
Disallow: /forums/private-pandoras-box/
Disallow: /forums/private-premium-member-area/
Disallow: /forums/private-the-core/
Disallow: /ignore_user/add/
Disallow: /IM/
Disallow: /imce/
Disallow: /includes/
Disallow: /logout/
Disallow: /misc/
Disallow: /modules/
Disallow: /myspace/
Disallow: /node/add/
Disallow: /og/subscribe/
Disallow: /privatemsg/
Disallow: /profiles/
Disallow: /reference/
Disallow: /scripts/
# Disallow: /search/
Disallow: /sites/
Disallow: /sxm-gen/
Disallow: /test/
Disallow: /themes/
Disallow: /user/login/
Disallow: /user/password/
Disallow: /user/register/
Disallow: /userpoints/
Disallow: /x/
# Enable Following 2 lines to have User Profiles crawled
# Disallow: /user/
# Disallow: /users/
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=contact/
Disallow: /?q=logout/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
# Files
Disallow: /2de0f5212b76.html
Disallow: /?q=warning.html
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /image2.gif
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /LiveSearchSiteAuth.xml
Disallow: /MAINTAINERS.txt
Disallow: /moderators-long.inc
Disallow: /php.ini
Disallow: /pinterest-1ed04.html
Disallow: /premie-list.htm
Disallow: /sites/default/settings.php
Disallow: /temp-blacklist.html
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /warning.html
Disallow: /xmlrpc.php
# Bots and Scrapers
User-agent: Aboundexbot
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: AlphaSeoBot
Disallow: /
User-agent: AlphaSeoBot-SA
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: BOT/0.1 (BOT for JCE)
Disallow: /
User-agent: coccoc
Disallow: /
User-agent: DAUM
Disallow: /
User-agent: discoverybot
Disallow: /
User-agent: dotbot
Disallow: /
User-agent: Ezooms
Disallow: /
User-agent: MegaIndex.ru
Disallow: /
User-agent: MegaIndex.ru/2.0
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: Riddler
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: SEOkicks-Robot
Disallow: /
User-agent: SeznamBot
Disallow: /
User-agent: SMTBot
Disallow: /
User-agent: Sogou spider
Disallow: /
User-agent: Sogou Pic Spider
Disallow: /
User-agent: Sogou Pic Spider/3.0
Disallow: /
User-agent: Sogou Web spider
Disallow: /
User-agent: Spinn3r
Disallow: /
User-agent: TurnitinBot
Disallow: /
User-agent: Yandex
Disallow: /
User-agent: Y!J-ASR/0.1 crawler
Disallow: /
User-agent: magpie-crawler
Disallow: /
User-agent: netEstate NE Crawler (+http://www.website-datenbank.de/)
Disallow: /
User-agent: netEstate NE Crawler
Disallow: /