sitemap: /cbooks-sitemap.xml

#====================
# ALL BOTS
#====================
User-agent: *
Disallow: /aspnet_client/
Disallow: /cgi-bin/
Disallow: /ftproot/
Disallow: /logtmp/
Disallow: /Merchant2/
Disallow: /MivaBackup/
Disallow: /stats/
Disallow: /_private/

#====================
# Bad Bot BOT LIST
#====================

User-agent: ecollector				# Bad Bot (e-collector)
User-agent: GenCrawler				# Bad Bot (GenDoor)
User-agent: Internet Cruiser Robot	# Bad Bot (Internet Cruiser)
User-agent: jcrawler				# Bad Bot (VietGATE)
User-agent: Zeus					# Bad Bot (Zeus)
User-agent: TurnitinBot				# Bad Bot (TurnitinBOT)
User-agent: T-H-U-N-D-E-R-S-T-O-N-E	# Bad Bot (Thunderstone)
User-agent: LinkWalker				# Bad Bot (Seven Twentyfour)
User-agent: polybot					# Bad Bot (PolyBOT)
User-agent: LinkAlarm				# Bad Bot (LinkAlarm)
User-agent: Googlebot-Image			# Bad Bot (Google Image) - crawls pages for google's image index
Disallow: /

#=====================
# Nice Bot BOT LIST
#=====================
User-agent: whowhere				#  Nice Bot (WhoWhere?)
User-agent: ZyBorg					#  Nice Bot (WiseNut)
User-agent: Scooter					#  Nice Bot (Alta Vista)
User-agent: ArchitextSpider			#  Nice Bot (Excite)
User-agent: Googlebot				#  Nice Bot (Google)
User-agent: Googlebot-Mobile		#  Nice Bot (Google) - crawls pages for google's mobile index
User-agent: Mediapartners-Google	#  Nice Bot (Google) - crawls pages to determine AdSense content.
User-agent: Adsbot-Google			#  Nice Bot (Google) - crawls pages to measure AdWords landing page quality.
User-agent: topiclink				#  Nice Bot (TopicLink)
User-agent: ia_archiver				#  Nice Bot (Wayback Machine)
User-agent: DaviesBot				#  Nice Bot (WholeWeb)
User-agent: winona					#  Nice Bot (whatUseek)
User-agent: ru-robot				#  Nice Bot (RUTGERS)
User-agent: ScoutAbout				#  Nice Bot (ScoutAbout)
User-agent: Speedy_Spider			#  Nice Bot (Speedy)
User-agent: Lycos_Spider_(T-Rex)	#  Nice Bot (Lycos)
User-agent: NetMechanic				#  Nice Bot (NetMechanic)
User-agent: Gulliver				#  Nice Bot (Northern Light)
User-agent: fido					#  Nice Bot (PlanetSearch)
User-agent: Slurp					#  Nice Bot (Inktomi)
User-agent: Enterprise_Search		#  Nice Bot (Innerprise)
User-agent: legs					#  Nice Bot (Legs)
User-agent: grabber					#  Nice Bot (Direct Hit Grabber)
User-agent: FAST-WebCrawler			#  Nice Bot (FAST/AllTheWeb)
User-agent: Gigabot					#  Nice Bot (Gigabot)
Disallow: /aspnet_client/
Disallow: /cgi-bin/
Disallow: /ftproot/
Disallow: /logtmp/
Disallow: /Merchant2/
Disallow: /MivaBackup/
Disallow: /stats/
Disallow: /_private/