# Require all robots to follow
User-agent: *

Crawl-delay: 60

# Uncomment after deploying site
# change-site-name-here with to domain
Sitemap: http://www.urbanlibraries.org/sitemap.xml

# Comment/remove after deploying
#Disallow: /

# Uncomment/enable the following after deploying
# Block all pdf files from getting indexed
Disallow: /*.pdf

# Block include files while still allowing images
Disallow: /filebin/*
Allow: /filebin/images/
Allow: /filebin/flash/
Allow: /filebin/css/

# Block facets
Disallow: /*?filters=

# Block mightysite files
Disallow: /mightysite
Disallow: /mightysite2

# Block facets
Disallow: /*?filters=