# Warren County's Robots File - "Please do not throck the grass."
# e-mail: "webmaster" at the warren municipal center (co.warren.ny.us)

Sitemap: http://www.co.warren.ny.us/site_index.xml

# Stanford's Web Archiving bot
User-agent: WebVac
Disallow: /ems/secure/
Disallow: /gis/dpw/
Visit-time: 0000-1200

# Archive.org
User-agent: ia_archiver
Disallow: /ems/secure/
Disallow: /gis/dpw/
Visit-time: 0000-1200

# W3 Link Checker
User-agent: W3C-checklink
Disallow:

# Gooooooogle
User-agent: Googlebot
Allow: *.xml$
Disallow: /ems/secure/
Disallow: /gis/dpw/
Disallow: /temp/
# Disallow: /gov/proceedings/ # Okay, Google; DON'T THROCK THE GRASS, DAMNIT. 
Crawl-delay: 15
Visit-time: 0000-1200

# Bad Spider - creates multiple 404 errors
User-Agent: Charlotte
Disallow: /

# General Rules for All Spiders
User-agent: *
Allow: *.xml$
Disallow: /ems/secure/
Disallow: /gis/dpw/
Disallow: /healthservices/default_files/
Disallow: /scripts/
Disallow: /temp/