2020-06-30 18:03:06 +02:00
|
|
|
User-Agent: *
|
|
|
|
Disallow: /upload
|
2021-08-09 14:31:19 +02:00
|
|
|
Disallow: /search/find/*community="covid-19*
|
2021-10-11 15:08:35 +02:00
|
|
|
|
|
|
|
#Search engine bots limitation
|
|
|
|
User-Agent:Yandex #Yandex
|
|
|
|
Request-rate: 1/10 # maximum rate is one page every 10 seconds
|
|
|
|
Crawl-delay: 10 # 10 seconds between page requests
|
|
|
|
|
|
|
|
User-agent: Baiduspider #Baidu
|
|
|
|
Request-rate: 1/10
|
|
|
|
Crawl-delay: 10
|
|
|
|
|
|
|
|
User-agent: PinterestbotCrawl #Pinterest
|
|
|
|
Request-rate: 1/10
|
|
|
|
Crawl-delay: 10
|
|
|
|
|
|
|
|
User-agent: Slurp #Yahoo
|
|
|
|
Request-rate: 1/10
|
|
|
|
Crawl-delay: 10
|
|
|
|
|
|
|
|
User-agent: DuckDuckBot #DuckDuckGo
|
|
|
|
Request-rate: 1/10
|
|
|
|
Crawl-delay: 10
|
|
|
|
|
|
|
|
User-agent: Exabot #Exelead
|
|
|
|
Request-rate: 1/10
|
|
|
|
Crawl-delay: 10
|
|
|
|
|
|
|
|
User-agent: facebot #Facebook
|
|
|
|
Request-rate: 1/10
|
|
|
|
Crawl-delay: 10
|
|
|
|
|
|
|
|
User-agent: ia_archiver # Alexa
|
|
|
|
Request-rate: 1/10
|
|
|
|
Crawl-delay: 10
|
|
|
|
|
|
|
|
|
|
|
|
# Blocking SEO-Related services Bot
|
|
|
|
User-agent: SiteAuditBot
|
|
|
|
User-agent: SemrushBot-BA
|
|
|
|
User-agent: SemrushBot-SI
|
|
|
|
User-agent: SemrushBot-BM
|
|
|
|
User-agent: rogerbot #Moz
|
|
|
|
User-agent: MJ12bot #Majestic
|
|
|
|
User-agent: dotbot #Moz
|
|
|
|
User-agent: AhrefsBot #Ahref
|
|
|
|
Disallow:/
|
|
|
|
|
|
|
|
# Blocking Generic Bots
|
|
|
|
User-agent: asterias
|
|
|
|
User-agent: BackDoorBot/1.0
|
|
|
|
User-agent: Black Hole
|
|
|
|
User-agent: BlowFish/1.0
|
|
|
|
User-agent: BotALot
|
|
|
|
User-agent: BuiltBotTough
|
|
|
|
User-agent: Bullseye/1.0
|
|
|
|
User-agent: BunnySlippers
|
|
|
|
User-agent: Cegbfeieh
|
|
|
|
User-agent: CheeseBot
|
|
|
|
User-agent: CherryPicker
|
|
|
|
User-agent: CherryPickerElite/1.0
|
|
|
|
User-agent: CherryPickerSE/1.0
|
|
|
|
User-agent: CopyRightCheck
|
|
|
|
User-agent: cosmos
|
|
|
|
User-agent: Crescent
|
|
|
|
User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
|
|
|
|
User-agent: DittoSpyder
|
|
|
|
User-agent: EmailCollector
|
|
|
|
User-agent: EmailSiphon
|
|
|
|
User-agent: EmailWolf
|
|
|
|
User-agent: EroCrawler
|
|
|
|
User-agent: ExtractorPro
|
|
|
|
User-agent: Foobot
|
|
|
|
User-agent: Harvest/1.5
|
|
|
|
User-agent: hloader
|
|
|
|
User-agent: httplib
|
|
|
|
User-agent: humanlinks
|
|
|
|
User-agent: InfoNaviRobot
|
|
|
|
User-agent: JennyBot
|
|
|
|
User-agent: Kenjin Spider
|
|
|
|
User-agent: Keyword Density/0.9
|
|
|
|
User-agent: LexiBot
|
|
|
|
User-agent: libWeb/clsHTTP
|
|
|
|
User-agent: line-poker/1.0
|
|
|
|
User-agent: LinkextractorPro
|
|
|
|
User-agent: LinkScan/8.1a Unix
|
|
|
|
User-agent: LinkWalker
|
|
|
|
User-agent: LNSpiderguy
|
|
|
|
User-agent: lwp-trivial
|
|
|
|
User-agent: lwp-trivial/1.34
|
|
|
|
User-agent: Mata Hari
|
|
|
|
User-agent: Microsoft URL Control - 5.01.4511
|
|
|
|
User-agent: Microsoft URL Control - 6.00.8169
|
|
|
|
User-agent: MIIxpc
|
|
|
|
User-agent: MIIxpc/4.2
|
|
|
|
User-agent: Mister PiX
|
|
|
|
User-agent: moget
|
|
|
|
User-agent: moget/2.1
|
|
|
|
User-agent: mozilla/4
|
|
|
|
User-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95)
|
|
|
|
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 95)
|
|
|
|
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 98)
|
|
|
|
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows NT)
|
|
|
|
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows XP)
|
|
|
|
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 2000)
|
|
|
|
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows ME)
|
|
|
|
User-agent: mozilla/5
|
|
|
|
User-agent: NetAnts
|
|
|
|
User-agent: NICErsPRO
|
|
|
|
User-agent: Offline Explorer
|
|
|
|
User-agent: Openfind
|
|
|
|
User-agent: Openfind data gathere
|
|
|
|
User-agent: ProPowerBot/2.14
|
|
|
|
User-agent: ProWebWalker
|
|
|
|
User-agent: QueryN Metasearch
|
|
|
|
User-agent: RepoMonkey
|
|
|
|
User-agent: RepoMonkey Bait & Tackle/v1.01
|
|
|
|
User-agent: RMA
|
|
|
|
User-agent: SiteSnagger
|
|
|
|
User-agent: SpankBot
|
|
|
|
User-agent: spanner
|
|
|
|
User-agent: suzuran
|
|
|
|
User-agent: Szukacz/1.4
|
|
|
|
User-agent: Teleport
|
|
|
|
User-agent: TeleportPro
|
|
|
|
User-agent: Telesoft
|
|
|
|
User-agent: The Intraformant
|
|
|
|
User-agent: TheNomad
|
|
|
|
User-agent: TightTwatBot
|
|
|
|
User-agent: Titan
|
|
|
|
User-agent: toCrawl/UrlDispatcher
|
|
|
|
User-agent: True_Robot
|
|
|
|
User-agent: True_Robot/1.0
|
|
|
|
User-agent: turingos
|
|
|
|
User-agent: URLy Warning
|
|
|
|
User-agent: VCI
|
|
|
|
User-agent: VCI WebViewer VCI WebViewer Win32
|
|
|
|
User-agent: Web Image Collector
|
|
|
|
User-agent: WebAuto
|
|
|
|
User-agent: WebBandit
|
|
|
|
User-agent: WebBandit/3.50
|
|
|
|
User-agent: WebCopier
|
|
|
|
User-agent: WebEnhancer
|
|
|
|
User-agent: WebmasterWorldForumBot
|
|
|
|
User-agent: WebSauger
|
|
|
|
User-agent: Website Quester
|
|
|
|
User-agent: Webster Pro
|
|
|
|
User-agent: WebStripper
|
|
|
|
User-agent: WebZip
|
|
|
|
User-agent: WebZip/4.0
|
|
|
|
User-agent: Wget
|
|
|
|
User-agent: Wget/1.5.3
|
|
|
|
User-agent: Wget/1.6
|
|
|
|
User-agent: WWW-Collector-E
|
|
|
|
User-agent: Xenu's
|
|
|
|
User-agent: Xenu's Link Sleuth 1.1c
|
|
|
|
User-agent: Zeus
|
|
|
|
User-agent: Zeus 32297 Webster Pro V2.9 Win32
|
|
|
|
User-agent: Riddler
|
|
|
|
Disallow: /
|