diff --git a/services/sitemaps/extractUrlsFromSearch.ts b/services/sitemaps/extractUrlsFromSearch.ts index cb00a80d..00ddb20b 100644 --- a/services/sitemaps/extractUrlsFromSearch.ts +++ b/services/sitemaps/extractUrlsFromSearch.ts @@ -56,7 +56,9 @@ function get(resultsPerUrl) { const url = resultsUrlPrefix + "&fq=" + key + " " + fieldIdsMap[key].equalityOperator + " \"" + encodeURIComponent(value.id) + "\"" + "&type=results&page=0&size=" + resultsPerUrl; - promiseArray.push(new Promise((resolve, reject) => { + await new Promise(resolve => setTimeout(resolve, 500)); + promiseArray.push( + new Promise((resolve, reject) => { request.get(url, function (err: any, response: any) { if (!response && err) { reject(err); @@ -67,8 +69,8 @@ function get(resultsPerUrl) { }) }).catch(error => { console.error("Error getting results ", error); - fs.appendFileSync("./"+errorFileName, "no response "+url); - fs.appendFileSync("./"+errorFileName, err); + fs.appendFileSync("./"+errorFileName, "no response "+url+" "); + fs.appendFileSync("./"+errorFileName, error); fs.appendFileSync("./"+errorFileName, "\n"); })); } @@ -129,7 +131,7 @@ function parseAllUrls(response: any, allUrls: any) { let pid: any = Identifier.getResultPIDFromIdentifiers(resultPreview.identifiers); let url; if(pid && pid.id) { - url = getUrlByType(resultPreview.resultType, pid, pid.id); + url = getUrlByType(resultPreview.resultType, pid, encodeURIComponent(pid.id)); } else { url = getUrlByType(resultPreview.resultType, null, resultPreview.id); } @@ -137,10 +139,8 @@ function parseAllUrls(response: any, allUrls: any) { alreadyin++; } else { allUrls.add(url); - let urlPre = "\n" + - " "; - let urlSuf = "\n" + - " "; + let urlPre = "\n"; + let urlSuf = ""; fs.appendFileSync("./"+fileName, urlPre + url + urlSuf); notin++; } @@ -225,7 +225,7 @@ function buildSiteMap(resultsPerUrl) { console.log("Buiding sitemap in file: "+fileName+"\n"); let sitemap = "\n" + - ""; + ""; fs.writeFile("./"+fileName, sitemap, function(err) { if(err) { @@ -248,9 +248,9 @@ let notin= 0; let communitiesPromise; let publicCommunities = []; -const refineUrl = "https://beta.services.openaire.eu/search/v2/api/resources2/?format=json&refine=true&fields=resultbestaccessright&fields=relfunder&fields=instancetypename&fields=resultlanguagename&fields=community&fields=resulthostingdatasource&fields=country&type=results&page=0&size=0"; -const resultsUrlPrefix = "https://beta.services.openaire.eu/search/v2/api/resources2/?format=json"; -const landingPrefix = "https://beta.explore.openaire.eu/search/"; -const contextUrl = "https://beta.services.openaire.eu/openaire/contexts/"; +const refineUrl = "https://services.openaire.eu/search/v2/api/resources2/?format=json&refine=true&fields=resultbestaccessright&fields=relfunder&fields=instancetypename&fields=resultlanguagename&fields=community&fields=resulthostingdatasource&fields=country&type=results&page=0&size=0"; +const resultsUrlPrefix = "https://services.openaire.eu/search/v2/api/resources2/?format=json"; +const landingPrefix = "https://explore.openaire.eu/search/"; +const contextUrl = "https://services.openaire.eu/openaire/contexts/"; buildSiteMap(100);