# robots.txt file # # purpose to World Wide Web Robots trawling your web site (or parts thereof) # # For more info consult: # http://info.webcrawler.com/mak/projects/robots/norobots.html #User-Agent: * # Applies to all robots #Disallow: / # do not index anything User-agent: ia_archiver Allow: /