#MITDEV ROBOTS FILE

User-agent: * # match all bots. 
Crawl-delay: 60 # per http://en.wikipedia.org/wiki/Robots.txt#Nonstandard_extensions, sets number of seconds to wait between requests to 5 seconds. may not work
Request-rate: 1/60

Allow: /kb/home
Allow: /kb/home.do
Allow: /kb/search
Allow: /kb/search.do
Allow: /kb/kb_view
Allow: /kb/kb_view.do
Allow: /kb/create
Allow: /kb/create.do
Allow: /kb/help
Allow: /kb/help.do
Allow: /KnowledgeSiteMap-Public
Allow: /KnowledgeSiteMap-Public.do
Allow: /sys_attachment
Allow: /sys_attachment.do

Allow: /*.png
Allow: /*.pngx
Allow: /*.jpg
Allow: /*.jpgx
Allow: /*.jpeg
Allow: /*.jpegx
Allow: /*.gif
Allow: /*.gifx

Disallow: /

sitemap: https://mitdev.service-now.com/KnowledgeSiteMap-Public.do