From 6dbdecafb43c1b3e6dd307925d4004a01d1c300e Mon Sep 17 00:00:00 2001
From: "konstantina.galouni"
 <konstantina.galouni@d315682c-612b-4755-9ff5-7f18f6832af3>
Date: Mon, 12 Jul 2021 14:09:55 +0000
Subject: [PATCH] [Trunk | Explore Services]: extractUrlsFromSearch.ts: Fixes
 in script | Services on production, not on beta.

git-svn-id: https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/uoa-services-portal/trunk@61367 d315682c-612b-4755-9ff5-7f18f6832af3
---
 services/sitemaps/extractUrlsFromSearch.ts | 26 +++++++++++-----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/services/sitemaps/extractUrlsFromSearch.ts b/services/sitemaps/extractUrlsFromSearch.ts
index cb00a80d..00ddb20b 100644
--- a/services/sitemaps/extractUrlsFromSearch.ts
+++ b/services/sitemaps/extractUrlsFromSearch.ts
@@ -56,7 +56,9 @@ function get(resultsPerUrl) {
 
             const url = resultsUrlPrefix + "&fq=" + key + " " + fieldIdsMap[key].equalityOperator + " \"" + encodeURIComponent(value.id) + "\"" + "&type=results&page=0&size=" + resultsPerUrl;
 
-            promiseArray.push(new Promise((resolve, reject) => {
+            await new Promise(resolve => setTimeout(resolve, 500));
+            promiseArray.push(
+              new Promise((resolve, reject) => {
               request.get(url, function (err: any, response: any) {
                 if (!response && err) {
                   reject(err);
@@ -67,8 +69,8 @@ function get(resultsPerUrl) {
               })
             }).catch(error => {
               console.error("Error getting results ", error);
-              fs.appendFileSync("./"+errorFileName, "no response  "+url);
-              fs.appendFileSync("./"+errorFileName, err);
+              fs.appendFileSync("./"+errorFileName, "no response  "+url+"  ");
+              fs.appendFileSync("./"+errorFileName, error);
               fs.appendFileSync("./"+errorFileName, "\n");
             }));
           }
@@ -129,7 +131,7 @@ function parseAllUrls(response: any, allUrls: any) {
     let pid: any = Identifier.getResultPIDFromIdentifiers(resultPreview.identifiers);
     let url;
     if(pid && pid.id) {
-      url = getUrlByType(resultPreview.resultType, pid, pid.id);
+      url = getUrlByType(resultPreview.resultType, pid, encodeURIComponent(pid.id));
     } else {
       url = getUrlByType(resultPreview.resultType, null, resultPreview.id);
     }
@@ -137,10 +139,8 @@ function parseAllUrls(response: any, allUrls: any) {
       alreadyin++;
     } else {
       allUrls.add(url);
-      let urlPre = "<url>\n" +
-        "    <loc>";
-      let urlSuf = "</loc>\n" +
-        "    </url>";
+      let urlPre = "\n<url><loc>";
+      let urlSuf = "</loc></url>";
       fs.appendFileSync("./"+fileName, urlPre + url + urlSuf);
       notin++;
     }
@@ -225,7 +225,7 @@ function buildSiteMap(resultsPerUrl) {
   console.log("Buiding sitemap in file: "+fileName+"\n");
 
   let sitemap = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
-    "<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd\">";
+    "<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">";
 
   fs.writeFile("./"+fileName, sitemap,  function(err) {
     if(err) {
@@ -248,9 +248,9 @@ let notin= 0;
 let communitiesPromise;
 let publicCommunities = [];
 
-const refineUrl = "https://beta.services.openaire.eu/search/v2/api/resources2/?format=json&refine=true&fields=resultbestaccessright&fields=relfunder&fields=instancetypename&fields=resultlanguagename&fields=community&fields=resulthostingdatasource&fields=country&type=results&page=0&size=0";
-const resultsUrlPrefix = "https://beta.services.openaire.eu/search/v2/api/resources2/?format=json";
-const landingPrefix = "https://beta.explore.openaire.eu/search/";
-const contextUrl = "https://beta.services.openaire.eu/openaire/contexts/";
+const refineUrl = "https://services.openaire.eu/search/v2/api/resources2/?format=json&refine=true&fields=resultbestaccessright&fields=relfunder&fields=instancetypename&fields=resultlanguagename&fields=community&fields=resulthostingdatasource&fields=country&type=results&page=0&size=0";
+const resultsUrlPrefix = "https://services.openaire.eu/search/v2/api/resources2/?format=json";
+const landingPrefix = "https://explore.openaire.eu/search/";
+const contextUrl = "https://services.openaire.eu/openaire/contexts/";
 
 buildSiteMap(100);