# robots.txt
# Revision 1.0
# Match all robots
User-agent: *

# Explicitly deny most URIs
Disallow: /pdbstatic/
Disallow: /pdb/img/
Disallow: /pdb/cgi/
Disallow: /pdb/search/
Disallow: /pdb/skins/
Disallow: /pdb/files/
Disallow: /pdb/download/
Disallow: /pdb/results/
Disallow: /pdb/ligand/
Disallow: /pdb/Viewers/
Disallow: /pdb/geometry/
Disallow: /pdb/common/
Disallow: /pdb/navbarsearch.do
Disallow: /pdb/struts/
Disallow: /pdb/queries/
Disallow: /pdb/search.do
Disallow: /pdb/statistics/
Disallow: /pdb/explore/

# Explicitly allow greenlighted URIs
Allow: /pdb/education_discussion/
Allow: /pdb/general_information/
Allow: /pdb/static.do
# Note we allow /pdb/explore.do but not /pdb/explore/
Allow: /pdb/explore/explore.do
Allow: /pdb/explore/derivedData.do
Allow: /pdb/explore/literature.do
Allow: /pdb/explore/biologyAndChemistry.do
Allow: /pdb/explore/materialsAndMethods.do

Allow: /pdb/explore.do
Allow: /pdb/home/
Allow: /pdb/common/
Allow: /pdb/rss/
Allow: /pdb/tutorials/
Allow: /robohelp_f/
Allow: /robohelp/

# Advanced robots.txt features
# Allow one page request every 10 minutes
Request-rate: 1/10
# Limit crawl time to one request per 10 seconds
Crawl-delay: 10
# Limit robot visit times to 4AM-10AM GMT (8PM-2AM PST)
Visit-time: 0400-1000

sitemap: http://www.rcsb.org/pdb/sitemap/index.jsp