# # robots.txt for http://www.wikipedia.org/ and friends # # Please note: There are a lot of pages on this site, and there are # some misbehaved spiders out there that go _way_ too fast. If you're # irresponsible, your access to the site may be blocked. # # advertising-related bots: User-agent: Mediapartners-Google* Disallow: / # Wikipedia work bots: User-agent: IsraBot Disallow: User-agent: Orthogaffe Disallow: # Crawlers that are kind enough to obey, but which we'd rather not have # unless they're feeding search engines. User-agent: UbiCrawler Disallow: / User-agent: DOC Disallow: / User-agent: Zao Disallow: / # Some bots are known to be trouble, particularly those designed to copy # entire sites. Please obey robots.txt. User-agent: sitecheck.internetseer.com Disallow: / User-agent: Zealbot Disallow: / User-agent: MSIECrawler Disallow: / User-agent: SiteSnagger Disallow: / User-agent: WebStripper Disallow: / User-agent: WebCopier Disallow: / User-agent: Fetch Disallow: / User-agent: Offline Explorer Disallow: / User-agent: Teleport Disallow: / User-agent: TeleportPro Disallow: / User-agent: WebZIP Disallow: / User-agent: linko Disallow: / User-agent: HTTrack Disallow: / User-agent: Microsoft.URL.Control Disallow: / User-agent: Xenu Disallow: / User-agent: larbin Disallow: / User-agent: libwww Disallow: / User-agent: ZyBORG Disallow: / User-agent: Download Ninja Disallow: / # # Sorry, wget in its recursive mode is a frequent problem. # Please read the man page and use it properly; there is a # --wait option you can use to set the delay between hits, # for instance. # User-agent: wget Disallow: / # # The 'grub' distributed client has been *very* poorly behaved. # User-agent: grub-client Disallow: / # # Doesn't follow robots.txt anyway, but... # User-agent: k2spider Disallow: / # # Hits many times per second, not acceptable # http://www.nameprotect.com/botinfo.html User-agent: NPBot Disallow: / # A capture bot, downloads gazillions of pages with no public benefit # http://www.webreaper.net/ User-agent: WebReaper Disallow: / # # Friendly, low-speed bots are welcome viewing article pages, but not # dynamically-generated pages please. # # Inktomi's "Slurp" can read a minimum delay between hits; if your # bot supports such a thing using the 'Crawl-delay' or another # instruction, please let us know. # User-agent: * Disallow: /tvwiki/ Disallow: /trap/ Disallow: /en/Especial:Search Disallow: /en/Especial%3ASearch Disallow: /en/Special:Randompage Disallow: /en/Special%3ARandompage Disallow: /en/Special:Search Disallow: /en/Special%3ASearch Disallow: /en/Spesial:Search Disallow: /en/Spesial%3ASearch Disallow: /en/Spezial:Search Disallow: /en/Spezial%3ASearch Disallow: /en/Specjalna:Search Disallow: /en/Specjalna%3ASearch Disallow: /en/Speciaal:Search Disallow: /en/Speciaal%3ASearch Disallow: /en/Speciaal:Randompage Disallow: /en/Speciaal%3ARandompage Disallow: /en/Speciel:Search Disallow: /en/Speciel%3ASearch Disallow: /en/Speciale:Search Disallow: /en/Speciale%3ASearch Disallow: /en/Istimewa:Search Disallow: /en/Istimewa%3ASearch Disallow: /en/Toiminnot:Search Disallow: /en/Toiminnot%3ASearch #ar Disallow: /en/%D8%AE%D8%A7%D8%B5:Search Disallow: /en/%D8%AE%D8%A7%D8%B5%3ASearch #he Disallow: /en/%D7%9E%D7%99%D7%95%D7%97%D7%93:Search Disallow: /en/%D7%9E%D7%99%D7%95%D7%97%D7%93%3ASearch #hu Disallow: /en/Speci%C3%A1lis:Search Disallow: /en/Speci%C3%A1lis%3ASearch #ja Disallow: /en/%E7%89%B9%E5%88%A5:Search Disallow: /en/%E7%89%B9%E5%88%A5%3ASearch #ru Disallow: /en/%D0%A1%D0%BF%D0%B5%D1%86%D0%B8%D0%B0%D0%BB%D1%8C%D0%BD%D1%8B%D0%B5:Search Disallow: /en/%D0%A1%D0%BF%D0%B5%D1%86%D0%B8%D0%B0%D0%BB%D1%8C%D0%BD%D1%8B%D0%B5%3ASearch Disallow: /es/Especial:Search Disallow: /es/Especial%3ASearch Disallow: /es/Special:Randompage Disallow: /es/Special%3ARandompage Disallow: /es/Special:Search Disallow: /es/Special%3ASearch Disallow: /es/Spesial:Search Disallow: /es/Spesial%3ASearch Disallow: /es/Spezial:Search Disallow: /es/Spezial%3ASearch Disallow: /es/Specjalna:Search Disallow: /es/Specjalna%3ASearch Disallow: /es/Speciaal:Search Disallow: /es/Speciaal%3ASearch Disallow: /es/Speciaal:Randompage Disallow: /es/Speciaal%3ARandompage Disallow: /es/Speciel:Search Disallow: /es/Speciel%3ASearch Disallow: /es/Speciale:Search Disallow: /es/Speciale%3ASearch Disallow: /es/Istimewa:Search Disallow: /es/Istimewa%3ASearch Disallow: /es/Toiminnot:Search Disallow: /es/Toiminnot%3ASearch #ar Disallow: /es/%D8%AE%D8%A7%D8%B5:Search Disallow: /es/%D8%AE%D8%A7%D8%B5%3ASearch #he Disallow: /es/%D7%9E%D7%99%D7%95%D7%97%D7%93:Search Disallow: /es/%D7%9E%D7%99%D7%95%D7%97%D7%93%3ASearch #hu Disallow: /es/Speci%C3%A1lis:Search Disallow: /es/Speci%C3%A1lis%3ASearch #ja Disallow: /es/%E7%89%B9%E5%88%A5:Search Disallow: /es/%E7%89%B9%E5%88%A5%3ASearch #ru Disallow: /es/%D0%A1%D0%BF%D0%B5%D1%86%D0%B8%D0%B0%D0%BB%D1%8C%D0%BD%D1%8B%D0%B5:Search Disallow: /es/%D0%A1%D0%BF%D0%B5%D1%86%D0%B8%D0%B0%D0%BB%D1%8C%D0%BD%D1%8B%D0%B5%3ASearch Disallow: /ca/Especial:Search Disallow: /ca/Especial%3ASearch Disallow: /ca/Special:Randompage Disallow: /ca/Special%3ARandompage Disallow: /ca/Special:Search Disallow: /ca/Special%3ASearch Disallow: /ca/Spesial:Search Disallow: /ca/Spesial%3ASearch Disallow: /ca/Spezial:Search Disallow: /ca/Spezial%3ASearch Disallow: /ca/Specjalna:Search Disallow: /ca/Specjalna%3ASearch Disallow: /ca/Speciaal:Search Disallow: /ca/Speciaal%3ASearch Disallow: /ca/Speciaal:Randompage Disallow: /ca/Speciaal%3ARandompage Disallow: /ca/Speciel:Search Disallow: /ca/Speciel%3ASearch Disallow: /ca/Speciale:Search Disallow: /ca/Speciale%3ASearch Disallow: /ca/Istimewa:Search Disallow: /ca/Istimewa%3ASearch Disallow: /ca/Toiminnot:Search Disallow: /ca/Toiminnot%3ASearch #ar Disallow: /ca/%D8%AE%D8%A7%D8%B5:Search Disallow: /ca/%D8%AE%D8%A7%D8%B5%3ASearch #he Disallow: /ca/%D7%9E%D7%99%D7%95%D7%97%D7%93:Search Disallow: /ca/%D7%9E%D7%99%D7%95%D7%97%D7%93%3ASearch #hu Disallow: /ca/Speci%C3%A1lis:Search Disallow: /ca/Speci%C3%A1lis%3ASearch #ja Disallow: /ca/%E7%89%B9%E5%88%A5:Search Disallow: /ca/%E7%89%B9%E5%88%A5%3ASearch #ru Disallow: /ca/%D0%A1%D0%BF%D0%B5%D1%86%D0%B8%D0%B0%D0%BB%D1%8C%D0%BD%D1%8B%D0%B5:Search Disallow: /ca/%D0%A1%D0%BF%D0%B5%D1%86%D0%B8%D0%B0%D0%BB%D1%8C%D0%BD%D1%8B%D0%B5%3ASearch # *at least* 1 second please. preferably more :D Crawl-delay: 1