#User-agent: seoengbot #Disallow: / #20120828 #User-agent: spbot #Disallow: / #20120828 #User-agent: WBSearchBot #Disallow: / #20120828 #User-agent: MLBot #Disallow: / #20120828 #User-agent: Exabot #Disallow: / #20120826 #User-agent: ezooms #Disallow: / #20120826 #User-agent: sistrix #Disallow: / #20120826 #User-agent: TurnitinBot #Disallow: / #20110830 #User-agent: Bingbot #Crawl-delay: 10 #20110615 User-agent: Slurp Crawl-delay: 10 #20110615 #User-agent: YRSpider #Disallow: / #20110503 #User-agent: GosoSpider #Disallow: / #20110411 #User-agent: Baiduspider #Disallow: / #20091108 User-agent: dotbot Crawl-delay: 30 #20091202 User-agent: MJ12bot Crawl-delay: 30 #20091013 User-agent: ia_archiver Disallow: / #20091021 User-agent: archive.org_bot Disallow: / #20120829 #User-agent: Sosospider #Disallow: / #User-agent: Yeti #Disallow: / #User-agent: YodaoBot #Disallow: / #User-agent: Yandex #Disallow: / # User-agent: * Disallow: /search Disallow: /audit #20140929 Crawl-delay: 60 #20110618