argirok
3 years ago
parent
813e4a8057
commit
c7993b1cc9
@ -1,3 +1,161 @@
|
||||
User-Agent: *
|
||||
Disallow: /upload
|
||||
Disallow: /search/find/*community="covid-19*
|
||||
|
||||
#Search engine bots limitation
|
||||
User-Agent:Yandex #Yandex
|
||||
Request-rate: 1/10 # maximum rate is one page every 10 seconds
|
||||
Crawl-delay: 10 # 10 seconds between page requests
|
||||
|
||||
User-agent: Baiduspider #Baidu
|
||||
Request-rate: 1/10
|
||||
Crawl-delay: 10
|
||||
|
||||
User-agent: PinterestbotCrawl #Pinterest
|
||||
Request-rate: 1/10
|
||||
Crawl-delay: 10
|
||||
|
||||
User-agent: Slurp #Yahoo
|
||||
Request-rate: 1/10
|
||||
Crawl-delay: 10
|
||||
|
||||
User-agent: DuckDuckBot #DuckDuckGo
|
||||
Request-rate: 1/10
|
||||
Crawl-delay: 10
|
||||
|
||||
User-agent: Exabot #Exelead
|
||||
Request-rate: 1/10
|
||||
Crawl-delay: 10
|
||||
|
||||
User-agent: facebot #Facebook
|
||||
Request-rate: 1/10
|
||||
Crawl-delay: 10
|
||||
|
||||
User-agent: ia_archiver # Alexa
|
||||
Request-rate: 1/10
|
||||
Crawl-delay: 10
|
||||
|
||||
|
||||
# Blocking SEO-Related services Bot
|
||||
User-agent: SiteAuditBot
|
||||
User-agent: SemrushBot-BA
|
||||
User-agent: SemrushBot-SI
|
||||
User-agent: SemrushBot-BM
|
||||
User-agent: rogerbot #Moz
|
||||
User-agent: MJ12bot #Majestic
|
||||
User-agent: dotbot #Moz
|
||||
User-agent: AhrefsBot #Ahref
|
||||
Disallow:/
|
||||
|
||||
# Blocking Generic Bots
|
||||
User-agent: asterias
|
||||
User-agent: BackDoorBot/1.0
|
||||
User-agent: Black Hole
|
||||
User-agent: BlowFish/1.0
|
||||
User-agent: BotALot
|
||||
User-agent: BuiltBotTough
|
||||
User-agent: Bullseye/1.0
|
||||
User-agent: BunnySlippers
|
||||
User-agent: Cegbfeieh
|
||||
User-agent: CheeseBot
|
||||
User-agent: CherryPicker
|
||||
User-agent: CherryPickerElite/1.0
|
||||
User-agent: CherryPickerSE/1.0
|
||||
User-agent: CopyRightCheck
|
||||
User-agent: cosmos
|
||||
User-agent: Crescent
|
||||
User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
|
||||
User-agent: DittoSpyder
|
||||
User-agent: EmailCollector
|
||||
User-agent: EmailSiphon
|
||||
User-agent: EmailWolf
|
||||
User-agent: EroCrawler
|
||||
User-agent: ExtractorPro
|
||||
User-agent: Foobot
|
||||
User-agent: Harvest/1.5
|
||||
User-agent: hloader
|
||||
User-agent: httplib
|
||||
User-agent: humanlinks
|
||||
User-agent: InfoNaviRobot
|
||||
User-agent: JennyBot
|
||||
User-agent: Kenjin Spider
|
||||
User-agent: Keyword Density/0.9
|
||||
User-agent: LexiBot
|
||||
User-agent: libWeb/clsHTTP
|
||||
User-agent: line-poker/1.0
|
||||
User-agent: LinkextractorPro
|
||||
User-agent: LinkScan/8.1a Unix
|
||||
User-agent: LinkWalker
|
||||
User-agent: LNSpiderguy
|
||||
User-agent: lwp-trivial
|
||||
User-agent: lwp-trivial/1.34
|
||||
User-agent: Mata Hari
|
||||
User-agent: Microsoft URL Control - 5.01.4511
|
||||
User-agent: Microsoft URL Control - 6.00.8169
|
||||
User-agent: MIIxpc
|
||||
User-agent: MIIxpc/4.2
|
||||
User-agent: Mister PiX
|
||||
User-agent: moget
|
||||
User-agent: moget/2.1
|
||||
User-agent: mozilla/4
|
||||
User-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95)
|
||||
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 95)
|
||||
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 98)
|
||||
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows NT)
|
||||
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows XP)
|
||||
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 2000)
|
||||
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows ME)
|
||||
User-agent: mozilla/5
|
||||
User-agent: NetAnts
|
||||
User-agent: NICErsPRO
|
||||
User-agent: Offline Explorer
|
||||
User-agent: Openfind
|
||||
User-agent: Openfind data gathere
|
||||
User-agent: ProPowerBot/2.14
|
||||
User-agent: ProWebWalker
|
||||
User-agent: QueryN Metasearch
|
||||
User-agent: RepoMonkey
|
||||
User-agent: RepoMonkey Bait & Tackle/v1.01
|
||||
User-agent: RMA
|
||||
User-agent: SiteSnagger
|
||||
User-agent: SpankBot
|
||||
User-agent: spanner
|
||||
User-agent: suzuran
|
||||
User-agent: Szukacz/1.4
|
||||
User-agent: Teleport
|
||||
User-agent: TeleportPro
|
||||
User-agent: Telesoft
|
||||
User-agent: The Intraformant
|
||||
User-agent: TheNomad
|
||||
User-agent: TightTwatBot
|
||||
User-agent: Titan
|
||||
User-agent: toCrawl/UrlDispatcher
|
||||
User-agent: True_Robot
|
||||
User-agent: True_Robot/1.0
|
||||
User-agent: turingos
|
||||
User-agent: URLy Warning
|
||||
User-agent: VCI
|
||||
User-agent: VCI WebViewer VCI WebViewer Win32
|
||||
User-agent: Web Image Collector
|
||||
User-agent: WebAuto
|
||||
User-agent: WebBandit
|
||||
User-agent: WebBandit/3.50
|
||||
User-agent: WebCopier
|
||||
User-agent: WebEnhancer
|
||||
User-agent: WebmasterWorldForumBot
|
||||
User-agent: WebSauger
|
||||
User-agent: Website Quester
|
||||
User-agent: Webster Pro
|
||||
User-agent: WebStripper
|
||||
User-agent: WebZip
|
||||
User-agent: WebZip/4.0
|
||||
User-agent: Wget
|
||||
User-agent: Wget/1.5.3
|
||||
User-agent: Wget/1.6
|
||||
User-agent: WWW-Collector-E
|
||||
User-agent: Xenu's
|
||||
User-agent: Xenu's Link Sleuth 1.1c
|
||||
User-agent: Zeus
|
||||
User-agent: Zeus 32297 Webster Pro V2.9 Win32
|
||||
User-agent: Riddler
|
||||
Disallow: /
|
||||
|
||||
Loading…
Reference in New Issue