User-agent: * Disallow: /config Disallow: /controllers Disallow: /elements Disallow: /helpers Disallow: /jobs Disallow: /languages Disallow: /libraries Disallow: /mail Disallow: /models Disallow: /single_pages Disallow: /tools Disallow: /cmcqldmobile Disallow: /blocks/*.php$ Disallow: /blocks/*.xml$ Disallow: /concrete/*.php$ Disallow: /concrete/*.xml$ Disallow: /packages/*.php$ Disallow: /packages/*.xml$ Disallow: /updates/*.php$ Disallow: /updates/*.xml$ # Exclude Files From All Robots: User-Agent: businessseek Disallow: / User-Agent: Blekkobot Disallow: / User-Agent: SafeDNSBot Disallow: / User-Agent: ScoutJet Disallow: / User-Agent: BUbiNG Disallow: / User-Agent: Daumoa Disallow: / User-Agent: MojeekBot Disallow: / User-Agent: IstellaBot Disallow: / User-Agent: seebot Disallow: / User-Agent: spbot Disallow: / User-Agent: Exabot Disallow: / User-Agent: SiteExplorer Disallow: / User-Agent: Wegtam Crawler Disallow: / User-Agent: SEOENGWorldBot Disallow: / User-Agent: Nutch Disallow: / User-Agent: Nutch-1.5 Disallow: / User-Agent: netEstate NE Crawler Disallow: / User-Agent: oBot Disallow: / User-Agent: Genieo Disallow: / User-Agent: Facebot/1.0 Disallow: / User-Agent: thunderstone Disallow: / User-Agent: Nutch-1.7 Disallow: / User-Agent: GrapeshotCrawler Disallow: / User-Agent: SEOkicks-Robot Disallow: / User-agent: netEstate NE Crawler (+http://www.website-datenbank.de/) Disallow: / User-agent: WBSearchBot Disallow: / User-agent: Baiduspider Disallow: / User-agent: Yeti Disallow: / User-agent: Wada.vn Disallow: / User-agent: panscient.com Disallow: / User-Agent: W3C-checklink Disallow: User-agent: super-goo Disallow: / User-agent: NCBot Disallow: / User-agent: meanpathbot Disallow: / User-agent: SEOstats Disallow: / User-agent: acoon.de Disallow: / User-agent: wscheck.com Disallow: / User-agent: checks.panopta.com Disallow: / User-agent: PagesInventory Disallow: / User-agent: Aboundex Disallow: / User-agent: Sosospider Disallow:/ User-agent: SeznamBot Disallow:/ User-agent: healthbot Disallow: / # blocking people search engines User-agent: 123People Disallow: / User-agent: findestars Disallow: / User-agent: MyOnID Disallow: / User-agent: peekyou Disallow: / User-agent: aihit Disallow: / User-agent: Pipl Disallow: / User-agent: rapleaf Disallow: / User-agent: snitch Disallow: / User-agent: spock Disallow: / User-agent: tweepz Disallow: / User-agent: wink Disallow: / User-agent: Yasni Disallow: / User-agent: Yandex Disallow: / User-agent: yoname Disallow: / User-agent: YoudaoBot Disallow: / User-agent: yourtraces Disallow: / User-agent: zoominfo Disallow: / User-agent: MJ12bot Disallow: / User-agent: HuaweiSymantecSpider Disallow: / User-agent: PagePeeker Disallow: / User-agent: Pagespeed/1.1 Fetcher Disallow: / User-agent: psbot Disallow: / User-agent: GarlikCrawler Disallow: / User-agent: betaBot Disallow: / User-agent: hn.kd.ny.adsl Disallow: / User-agent: NextGenSearchBot Disallow: / # End robots.txt file