From c7993b1cc980e35bbf83e5876d15cbee8d3c6b5d Mon Sep 17 00:00:00 2001
From: argirok <argirokokogiannaki@gmail.com>
Date: Mon, 11 Oct 2021 16:08:35 +0300
Subject: [PATCH] update robots.txt

---
 explore/src/prod/robots.txt | 158 ++++++++++++++++++++++++++++++++++++
 1 file changed, 158 insertions(+)

diff --git a/explore/src/prod/robots.txt b/explore/src/prod/robots.txt
index e87bc075..01e59663 100644
--- a/explore/src/prod/robots.txt
+++ b/explore/src/prod/robots.txt
@@ -1,3 +1,161 @@
 User-Agent: *
 Disallow: /upload
 Disallow: /search/find/*community="covid-19*
+
+#Search engine bots limitation
+User-Agent:Yandex				#Yandex
+Request-rate: 1/10				# maximum rate is one page every 10 seconds
+Crawl-delay: 10					# 10 seconds between page requests
+
+User-agent: Baiduspider			#Baidu
+Request-rate: 1/10
+Crawl-delay: 10
+
+User-agent: PinterestbotCrawl	#Pinterest
+Request-rate: 1/10
+Crawl-delay: 10
+
+User-agent: Slurp				#Yahoo
+Request-rate: 1/10
+Crawl-delay: 10
+
+User-agent: DuckDuckBot			#DuckDuckGo
+Request-rate: 1/10
+Crawl-delay: 10
+
+User-agent: Exabot				#Exelead
+Request-rate: 1/10
+Crawl-delay: 10
+
+User-agent: facebot				#Facebook
+Request-rate: 1/10
+Crawl-delay: 10
+
+User-agent: ia_archiver			# Alexa
+Request-rate: 1/10
+Crawl-delay: 10
+
+
+# Blocking SEO-Related services Bot
+User-agent: SiteAuditBot
+User-agent: SemrushBot-BA
+User-agent: SemrushBot-SI
+User-agent: SemrushBot-BM
+User-agent: rogerbot		#Moz
+User-agent: MJ12bot			#Majestic
+User-agent: dotbot			#Moz
+User-agent: AhrefsBot		#Ahref
+Disallow:/
+
+# Blocking Generic Bots
+User-agent: asterias
+User-agent: BackDoorBot/1.0
+User-agent: Black Hole
+User-agent: BlowFish/1.0
+User-agent: BotALot
+User-agent: BuiltBotTough
+User-agent: Bullseye/1.0
+User-agent: BunnySlippers
+User-agent: Cegbfeieh
+User-agent: CheeseBot
+User-agent: CherryPicker
+User-agent: CherryPickerElite/1.0
+User-agent: CherryPickerSE/1.0
+User-agent: CopyRightCheck
+User-agent: cosmos
+User-agent: Crescent
+User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
+User-agent: DittoSpyder
+User-agent: EmailCollector
+User-agent: EmailSiphon
+User-agent: EmailWolf
+User-agent: EroCrawler
+User-agent: ExtractorPro
+User-agent: Foobot
+User-agent: Harvest/1.5
+User-agent: hloader
+User-agent: httplib
+User-agent: humanlinks
+User-agent: InfoNaviRobot
+User-agent: JennyBot
+User-agent: Kenjin Spider
+User-agent: Keyword Density/0.9
+User-agent: LexiBot
+User-agent: libWeb/clsHTTP
+User-agent: line-poker/1.0
+User-agent: LinkextractorPro
+User-agent: LinkScan/8.1a Unix
+User-agent: LinkWalker
+User-agent: LNSpiderguy
+User-agent: lwp-trivial
+User-agent: lwp-trivial/1.34
+User-agent: Mata Hari
+User-agent: Microsoft URL Control - 5.01.4511
+User-agent: Microsoft URL Control - 6.00.8169
+User-agent: MIIxpc
+User-agent: MIIxpc/4.2
+User-agent: Mister PiX
+User-agent: moget
+User-agent: moget/2.1
+User-agent: mozilla/4
+User-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95)
+User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 95)
+User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 98)
+User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows NT)
+User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows XP)
+User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 2000)
+User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows ME)
+User-agent: mozilla/5
+User-agent: NetAnts
+User-agent: NICErsPRO
+User-agent: Offline Explorer
+User-agent: Openfind
+User-agent: Openfind data gathere
+User-agent: ProPowerBot/2.14
+User-agent: ProWebWalker
+User-agent: QueryN Metasearch
+User-agent: RepoMonkey
+User-agent: RepoMonkey Bait & Tackle/v1.01
+User-agent: RMA
+User-agent: SiteSnagger
+User-agent: SpankBot
+User-agent: spanner
+User-agent: suzuran
+User-agent: Szukacz/1.4
+User-agent: Teleport
+User-agent: TeleportPro
+User-agent: Telesoft
+User-agent: The Intraformant
+User-agent: TheNomad
+User-agent: TightTwatBot
+User-agent: Titan
+User-agent: toCrawl/UrlDispatcher
+User-agent: True_Robot
+User-agent: True_Robot/1.0
+User-agent: turingos
+User-agent: URLy Warning
+User-agent: VCI
+User-agent: VCI WebViewer VCI WebViewer Win32
+User-agent: Web Image Collector
+User-agent: WebAuto
+User-agent: WebBandit
+User-agent: WebBandit/3.50
+User-agent: WebCopier
+User-agent: WebEnhancer
+User-agent: WebmasterWorldForumBot
+User-agent: WebSauger
+User-agent: Website Quester
+User-agent: Webster Pro
+User-agent: WebStripper
+User-agent: WebZip
+User-agent: WebZip/4.0
+User-agent: Wget
+User-agent: Wget/1.5.3
+User-agent: Wget/1.6
+User-agent: WWW-Collector-E
+User-agent: Xenu's
+User-agent: Xenu's Link Sleuth 1.1c
+User-agent: Zeus
+User-agent: Zeus 32297 Webster Pro V2.9 Win32
+User-agent: Riddler
+Disallow: /