# This file is used by automatic WWW Robots, Web Crawlers, and
# other automated web mapper programs.  It defines which areas
# are not to be explored.
# See http://web.nexor.co.uk:mak/doc/robots/norobots.html
# and http://searchenginewatch.com/webmasters/spiderchart.html
# for more information.
#
#
#To prevent email harvesters, employed by spammers, AlanNET Internet Services 
#(http://www.alannet.net) has decided to dissallow all domain crawling robots
#from traversing our domain, except those robots which do so for benevolent 
#or benign purposes.  To have your robot be put on our list of allowable robots,
#please contact us at:privacy@alannet.net.
#
#
# Robots.txt
# 941023:2335.MSM

User-agent: *   # for all robots
Disallow: :     # This will prevent all robots from traversing our domain.

#Then we can list those which are allowed to crawl our domain.

#  Planet Direct knows where to go
User-agent: Mozilla:2.0 (Normandy 1.0b3a) Planet Direct          # Planet Direct IRS crawler
Disallow: /admin/               #only main sections of alannet.net are allowed.
Disallow: /mail/					  # Clients sites must have their own robots.txt				   
Disallow: /servlets/
Disallow: /plug-ins/
Disallow: /cgi-bin/
Disallow: /chat/
Disallow: /affiliates/ 
Disallow: /billing/ 
Disallow: /CL/  
Disallow: /users/
Disallow: /members/
Disallow: /ubb/  
Disallow: /ubb-non/
Disallow: /download/
Disallow: /employees/
Disallow: /forums/
Disallow: /store/
Disallow: /images/
Disallow: /logs/
Disallow: /marketing/
Disallow: /new/
Disallow: /new.data/
Disallow: /tests/ 
Disallow: /mac-perl/ 
Disallow: /Tools/
Disallow: /index.html
Disallow: /weather/
Disallow: /site/
Disallow: /lasso/
Disallow: /documentation/
Disallow: /error.html
Disallow: /noaccess.html
Disallow: /pi_admin.*

# Lycos knows where to go.
User-agent: Lycos_Spider_(T-Rex):3.0
Disallow: /admin/               #only main sections of alannet.net are allowed.
Disallow: /mail/					  # Clients sites must have their own robots.txt				   
Disallow: /servlets/
Disallow: /plug-ins/
Disallow: /cgi-bin/
Disallow: /chat/
Disallow: /affiliates/ 
Disallow: /billing/ 
Disallow: /CL/  
Disallow: /users/
Disallow: /members/
Disallow: /ubb/  
Disallow: /ubb-non/
Disallow: /download/
Disallow: /employees/
Disallow: /forums/
Disallow: /store/
Disallow: /images/
Disallow: /logs/
Disallow: /marketing/
Disallow: /new/
Disallow: /new.data/
Disallow: /tests/ 
Disallow: /mac-perl/ 
Disallow: /Tools/
Disallow: /index.html
Disallow: /weather/
Disallow: /site/
Disallow: /lasso/
Disallow: /documentation/
Disallow: /error.html
Disallow: /noaccess.html
Disallow: /pi_admin.*

# InfoSeek knows where to go.
User-agent: InfoSeek Sidewinder/0.9
Disallow: /admin/               #only main sections of alannet.net are allowed.
Disallow: /mail/					  # Clients sites must have their own robots.txt				   
Disallow: /servlets/
Disallow: /plug-ins/
Disallow: /cgi-bin/
Disallow: /chat/
Disallow: /affiliates/ 
Disallow: /billing/ 
Disallow: /CL/  
Disallow: /users/
Disallow: /members/
Disallow: /ubb/  
Disallow: /ubb-non/
Disallow: /download/
Disallow: /employees/
Disallow: /forums/
Disallow: /store/
Disallow: /images/
Disallow: /logs/
Disallow: /marketing/
Disallow: /new/
Disallow: /new.data/
Disallow: /tests/ 
Disallow: /mac-perl/ 
Disallow: /Tools/
Disallow: /index.html
Disallow: /weather/
Disallow: /site/
Disallow: /lasso/
Disallow: /documentation/
Disallow: /error.html
Disallow: /noaccess.html
Disallow: /pi_admin.*

# WebCrawler knows where to go.
User-agent: WebCrawler/3.0_Robot libwww/5.0a
Disallow: /admin/               #only main sections of alannet.net are allowed.
Disallow: /mail/					  # Clients sites must have their own robots.txt				   
Disallow: /servlets/
Disallow: /plug-ins/
Disallow: /cgi-bin/
Disallow: /chat/
Disallow: /affiliates/ 
Disallow: /billing/ 
Disallow: /CL/  
Disallow: /users/
Disallow: /members/
Disallow: /ubb/  
Disallow: /ubb-non/
Disallow: /download/
Disallow: /employees/
Disallow: /forums/
Disallow: /store/
Disallow: /images/
Disallow: /logs/
Disallow: /marketing/
Disallow: /new/
Disallow: /new.data/
Disallow: /tests/ 
Disallow: /mac-perl/ 
Disallow: /Tools/
Disallow: /index.html
Disallow: /weather/
Disallow: /site/
Disallow: /lasso/
Disallow: /documentation/
Disallow: /error.html
Disallow: /noaccess.html
Disallow: /pi_admin.*

# WebCrawler knows where to go.
User-agent: WebCrawler-AddURL/2.0
Disallow: /admin/               #only main sections of alannet.net are allowed.
Disallow: /mail/					  # Clients sites must have their own robots.txt				   
Disallow: /servlets/
Disallow: /plug-ins/
Disallow: /cgi-bin/
Disallow: /chat/
Disallow: /affiliates/ 
Disallow: /billing/ 
Disallow: /CL/  
Disallow: /users/
Disallow: /members/
Disallow: /ubb/  
Disallow: /ubb-non/
Disallow: /download/
Disallow: /employees/
Disallow: /forums/
Disallow: /store/
Disallow: /images/
Disallow: /logs/
Disallow: /marketing/
Disallow: /new/
Disallow: /new.data/
Disallow: /tests/ 
Disallow: /mac-perl/ 
Disallow: /Tools/
Disallow: /index.html
Disallow: /weather/
Disallow: /site/
Disallow: /lasso/
Disallow: /documentation/
Disallow: /error.html
Disallow: /noaccess.html
Disallow: /pi_admin.*

# WebCrawler knows where to go.
User-agent: WebCrawler-Status/1.0
Disallow: /admin/               #only main sections of alannet.net are allowed.
Disallow: /mail/					  # Clients sites must have their own robots.txt				   
Disallow: /servlets/
Disallow: /plug-ins/
Disallow: /cgi-bin/
Disallow: /chat/
Disallow: /affiliates/ 
Disallow: /billing/ 
Disallow: /CL/  
Disallow: /users/
Disallow: /members/
Disallow: /ubb/  
Disallow: /ubb-non/
Disallow: /download/
Disallow: /employees/
Disallow: /forums/
Disallow: /store/
Disallow: /images/
Disallow: /logs/
Disallow: /marketing/
Disallow: /new/
Disallow: /new.data/
Disallow: /tests/ 
Disallow: /mac-perl/ 
Disallow: /Tools/
Disallow: /index.html
Disallow: /weather/
Disallow: /site/
Disallow: /lasso/
Disallow: /documentation/
Disallow: /error.html
Disallow: /noaccess.html
Disallow: /pi_admin.*

# Excite (formerly known as Architext) knows where to go.
User-agent: ArchitextSpider
Disallow: /admin/               #only main sections of alannet.net are allowed.
Disallow: /mail/					  # Clients sites must have their own robots.txt				   
Disallow: /servlets/
Disallow: /plug-ins/
Disallow: /cgi-bin/
Disallow: /chat/
Disallow: /affiliates/ 
Disallow: /billing/ 
Disallow: /CL/  
Disallow: /users/
Disallow: /members/
Disallow: /ubb/  
Disallow: /ubb-non/
Disallow: /download/
Disallow: /employees/
Disallow: /forums/
Disallow: /store/
Disallow: /images/
Disallow: /logs/
Disallow: /marketing/
Disallow: /new/
Disallow: /new.data/
Disallow: /tests/ 
Disallow: /mac-perl/ 
Disallow: /Tools/
Disallow: /index.html
Disallow: /weather/
Disallow: /site/
Disallow: /lasso/
Disallow: /documentation/
Disallow: /error.html
Disallow: /noaccess.html
Disallow: /pi_admin.*

# AltaVista (from Digital)knows where to go.
User-agent: Scooter/1.0
Disallow: /admin/               #only main sections of alannet.net are allowed.
Disallow: /mail/					  # Clients sites must have their own robots.txt				   
Disallow: /servlets/
Disallow: /plug-ins/
Disallow: /cgi-bin/
Disallow: /chat/
Disallow: /affiliates/ 
Disallow: /billing/ 
Disallow: /CL/  
Disallow: /users/
Disallow: /members/
Disallow: /ubb/  
Disallow: /ubb-non/
Disallow: /download/
Disallow: /employees/
Disallow: /forums/
Disallow: /store/
Disallow: /images/
Disallow: /logs/
Disallow: /marketing/
Disallow: /new/
Disallow: /new.data/
Disallow: /tests/ 
Disallow: /mac-perl/ 
Disallow: /Tools/
Disallow: /index.html
Disallow: /weather/
Disallow: /site/
Disallow: /lasso/
Disallow: /documentation/
Disallow: /error.html
Disallow: /noaccess.html
Disallow: /pi_admin.*

# Northern Light knows where to go.
User-agent: Gulliver/1.2
Disallow: /admin/               #only main sections of alannet.net are allowed.
Disallow: /mail/					  # Clients sites must have their own robots.txt				   
Disallow: /servlets/
Disallow: /plug-ins/
Disallow: /cgi-bin/
Disallow: /chat/
Disallow: /affiliates/ 
Disallow: /billing/ 
Disallow: /CL/  
Disallow: /users/
Disallow: /members/
Disallow: /ubb/  
Disallow: /ubb-non/
Disallow: /download/
Disallow: /employees/
Disallow: /forums/
Disallow: /store/
Disallow: /images/
Disallow: /logs/
Disallow: /marketing/
Disallow: /new/
Disallow: /new.data/
Disallow: /tests/ 
Disallow: /mac-perl/ 
Disallow: /Tools/
Disallow: /index.html
Disallow: /weather/
Disallow: /site/
Disallow: /lasso/
Disallow: /documentation/
Disallow: /error.html
Disallow: /noaccess.html
Disallow: /pi_admin.*

# MOMSpider knows where to go.
User-agent: MOMSpider/1.00
Disallow: /admin/               #only main sections of alannet.net are allowed.
Disallow: /mail/					  # Clients sites must have their own robots.txt				   
Disallow: /servlets/
Disallow: /plug-ins/
Disallow: /cgi-bin/
Disallow: /chat/
Disallow: /affiliates/ 
Disallow: /billing/ 
Disallow: /CL/  
Disallow: /users/
Disallow: /members/
Disallow: /ubb/  
Disallow: /ubb-non/
Disallow: /download/
Disallow: /employees/
Disallow: /forums/
Disallow: /store/
Disallow: /images/
Disallow: /logs/
Disallow: /marketing/
Disallow: /new/
Disallow: /new.data/
Disallow: /tests/ 
Disallow: /mac-perl/ 
Disallow: /Tools/
Disallow: /index.html
Disallow: /weather/
Disallow: /site/
Disallow: /lasso/
Disallow: /documentation/
Disallow: /error.html
Disallow: /noaccess.html
Disallow: /pi_admin.*

# SEAQuest! (Southeast Asia Search Engine) http://SEAQuest.kusza.edu.my knows where to go.
User-agent: ANKabut/1.5
Disallow: /admin/               #only main sections of alannet.net are allowed.
Disallow: /mail/					  # Clients sites must have their own robots.txt				   
Disallow: /servlets/
Disallow: /plug-ins/
Disallow: /cgi-bin/
Disallow: /chat/
Disallow: /affiliates/ 
Disallow: /billing/ 
Disallow: /CL/  
Disallow: /users/
Disallow: /members/
Disallow: /ubb/  
Disallow: /ubb-non/
Disallow: /download/
Disallow: /employees/
Disallow: /forums/
Disallow: /store/
Disallow: /images/
Disallow: /logs/
Disallow: /marketing/
Disallow: /new/
Disallow: /new.data/
Disallow: /tests/ 
Disallow: /mac-perl/ 
Disallow: /Tools/
Disallow: /index.html
Disallow: /weather/
Disallow: /site/
Disallow: /lasso/
Disallow: /documentation/
Disallow: /error.html
Disallow: /noaccess.html
Disallow: /pi_admin.*

# FreeFind knows where to go.
User-agent: FreeFind
Disallow: /admin/               #only main sections of alannet.net are allowed.
Disallow: /mail/					  # Clients sites must have their own robots.txt				   
Disallow: /servlets/
Disallow: /plug-ins/
Disallow: /cgi-bin/
Disallow: /chat/
Disallow: /affiliates/ 
Disallow: /billing/ 
Disallow: /CL/  
Disallow: /users/
Disallow: /members/
Disallow: /ubb/  
Disallow: /ubb-non/
Disallow: /download/
Disallow: /employees/
Disallow: /forums/
Disallow: /store/
Disallow: /images/
Disallow: /logs/
Disallow: /marketing/
Disallow: /new/
Disallow: /new.data/
Disallow: /tests/ 
Disallow: /mac-perl/ 
Disallow: /Tools/
Disallow: /index.html
Disallow: /weather/
Disallow: /site/
Disallow: /lasso/
Disallow: /documentation/
Disallow: /error.html
Disallow: /noaccess.html
Disallow: /pi_admin.*

# NetMechanic knows where to go.
User-agent: NetMechanic
Disallow: /admin/               #only main sections of alannet.net are allowed.
Disallow: /mail/					  # Clients sites must have their own robots.txt				   
Disallow: /servlets/
Disallow: /plug-ins/
Disallow: /cgi-bin/
Disallow: /chat/
Disallow: /affiliates/ 
Disallow: /billing/ 
Disallow: /CL/  
Disallow: /users/
Disallow: /members/
Disallow: /ubb/  
Disallow: /ubb-non/
Disallow: /download/
Disallow: /employees/
Disallow: /forums/
Disallow: /store/
Disallow: /images/
Disallow: /logs/
Disallow: /marketing/
Disallow: /new/
Disallow: /new.data/
Disallow: /tests/ 
Disallow: /mac-perl/ 
Disallow: /Tools/
Disallow: /index.html
Disallow: /weather/
Disallow: /site/
Disallow: /lasso/
Disallow: /documentation/
Disallow: /error.html
Disallow: /noaccess.html
Disallow: /pi_admin.*

# LinkAlarm knows where to go.
User-agent: LinkAlarm
Disallow: /admin/               #only main sections of alannet.net are allowed.
Disallow: /mail/					  # Clients sites must have their own robots.txt				   
Disallow: /servlets/
Disallow: /plug-ins/
Disallow: /cgi-bin/
Disallow: /chat/
Disallow: /affiliates/ 
Disallow: /billing/ 
Disallow: /CL/  
Disallow: /users/
Disallow: /members/
Disallow: /ubb/  
Disallow: /ubb-non/
Disallow: /download/
Disallow: /employees/
Disallow: /forums/
Disallow: /store/
Disallow: /images/
Disallow: /logs/
Disallow: /marketing/
Disallow: /new/
Disallow: /new.data/
Disallow: /tests/ 
Disallow: /mac-perl/ 
Disallow: /Tools/
Disallow: /index.html
Disallow: /weather/
Disallow: /site/
Disallow: /lasso/
Disallow: /documentation/
Disallow: /error.html
Disallow: /noaccess.html
Disallow: /pi_admin.*

# N2H2-BirdDog knows where to go.
User-agent: N2H2-BirdDog
Disallow: /admin/               #only main sections of alannet.net are allowed.
Disallow: /mail/					  # Clients sites must have their own robots.txt				   
Disallow: /servlets/
Disallow: /plug-ins/
Disallow: /cgi-bin/
Disallow: /chat/
Disallow: /affiliates/ 
Disallow: /billing/ 
Disallow: /CL/  
Disallow: /users/
Disallow: /members/
Disallow: /ubb/  
Disallow: /ubb-non/
Disallow: /download/
Disallow: /employees/
Disallow: /forums/
Disallow: /store/
Disallow: /images/
Disallow: /logs/
Disallow: /marketing/
Disallow: /new/
Disallow: /new.data/
Disallow: /tests/ 
Disallow: /mac-perl/ 
Disallow: /Tools/
Disallow: /index.html
Disallow: /weather/
Disallow: /site/
Disallow: /lasso/
Disallow: /documentation/
Disallow: /error.html
Disallow: /noaccess.html
Disallow: /pi_admin.*

# htdig knows where to go.
User-agent: htdig/3.1.0b1
Disallow: /admin/               #only main sections of alannet.net are allowed.
Disallow: /mail/					  # Clients sites must have their own robots.txt				   
Disallow: /servlets/
Disallow: /plug-ins/
Disallow: /cgi-bin/
Disallow: /chat/
Disallow: /affiliates/ 
Disallow: /billing/ 
Disallow: /CL/  
Disallow: /users/
Disallow: /members/
Disallow: /ubb/  
Disallow: /ubb-non/
Disallow: /download/
Disallow: /employees/
Disallow: /forums/
Disallow: /store/
Disallow: /images/
Disallow: /logs/
Disallow: /marketing/
Disallow: /new/
Disallow: /new.data/
Disallow: /tests/ 
Disallow: /mac-perl/ 
Disallow: /Tools/
Disallow: /index.html
Disallow: /weather/
Disallow: /site/
Disallow: /lasso/
Disallow: /documentation/
Disallow: /error.html
Disallow: /noaccess.html
Disallow: /pi_admin.*

# PicoSearch knows where to go.
User-agent: PicoSearch/1.0
Disallow: /admin/               #only main sections of alannet.net are allowed.
Disallow: /mail/					  # Clients sites must have their own robots.txt				   
Disallow: /servlets/
Disallow: /plug-ins/
Disallow: /cgi-bin/
Disallow: /chat/
Disallow: /affiliates/ 
Disallow: /billing/ 
Disallow: /CL/  
Disallow: /users/
Disallow: /members/
Disallow: /ubb/  
Disallow: /ubb-non/
Disallow: /download/
Disallow: /employees/
Disallow: /forums/
Disallow: /store/
Disallow: /images/
Disallow: /logs/
Disallow: /marketing/
Disallow: /new/
Disallow: /new.data/
Disallow: /tests/ 
Disallow: /mac-perl/ 
Disallow: /Tools/
Disallow: /index.html
Disallow: /weather/
Disallow: /site/
Disallow: /lasso/
Disallow: /documentation/
Disallow: /error.html
Disallow: /noaccess.html
Disallow: /pi_admin.*

# FashionSpider <http://www.media.mit.edu:80/> knows where to go.
User-agent: FashionSpider
Disallow: /admin/               #only main sections of alannet.net are allowed.
Disallow: /mail/					  # Clients sites must have their own robots.txt				   
Disallow: /servlets/
Disallow: /plug-ins/
Disallow: /cgi-bin/
Disallow: /chat/
Disallow: /affiliates/ 
Disallow: /billing/ 
Disallow: /CL/  
Disallow: /users/
Disallow: /members/
Disallow: /ubb/  
Disallow: /ubb-non/
Disallow: /download/
Disallow: /employees/
Disallow: /forums/
Disallow: /store/
Disallow: /images/
Disallow: /logs/
Disallow: /marketing/
Disallow: /new/
Disallow: /new.data/
Disallow: /tests/ 
Disallow: /mac-perl/ 
Disallow: /Tools/
Disallow: /index.html
Disallow: /weather/
Disallow: /site/
Disallow: /lasso/
Disallow: /documentation/
Disallow: /error.html
Disallow: /noaccess.html
Disallow: /pi_admin.*