diff --git a/services/sitemaps/extractUrlsFromSearch.ts b/services/sitemaps/extractUrlsFromSearch.ts
index cb00a80d..00ddb20b 100644
--- a/services/sitemaps/extractUrlsFromSearch.ts
+++ b/services/sitemaps/extractUrlsFromSearch.ts
@@ -56,7 +56,9 @@ function get(resultsPerUrl) {
const url = resultsUrlPrefix + "&fq=" + key + " " + fieldIdsMap[key].equalityOperator + " \"" + encodeURIComponent(value.id) + "\"" + "&type=results&page=0&size=" + resultsPerUrl;
- promiseArray.push(new Promise((resolve, reject) => {
+ await new Promise(resolve => setTimeout(resolve, 500));
+ promiseArray.push(
+ new Promise((resolve, reject) => {
request.get(url, function (err: any, response: any) {
if (!response && err) {
reject(err);
@@ -67,8 +69,8 @@ function get(resultsPerUrl) {
})
}).catch(error => {
console.error("Error getting results ", error);
- fs.appendFileSync("./"+errorFileName, "no response "+url);
- fs.appendFileSync("./"+errorFileName, err);
+ fs.appendFileSync("./"+errorFileName, "no response "+url+" ");
+ fs.appendFileSync("./"+errorFileName, error);
fs.appendFileSync("./"+errorFileName, "\n");
}));
}
@@ -129,7 +131,7 @@ function parseAllUrls(response: any, allUrls: any) {
let pid: any = Identifier.getResultPIDFromIdentifiers(resultPreview.identifiers);
let url;
if(pid && pid.id) {
- url = getUrlByType(resultPreview.resultType, pid, pid.id);
+ url = getUrlByType(resultPreview.resultType, pid, encodeURIComponent(pid.id));
} else {
url = getUrlByType(resultPreview.resultType, null, resultPreview.id);
}
@@ -137,10 +139,8 @@ function parseAllUrls(response: any, allUrls: any) {
alreadyin++;
} else {
allUrls.add(url);
- let urlPre = "\n" +
- " ";
- let urlSuf = "\n" +
- " ";
+ let urlPre = "\n";
+ let urlSuf = "";
fs.appendFileSync("./"+fileName, urlPre + url + urlSuf);
notin++;
}
@@ -225,7 +225,7 @@ function buildSiteMap(resultsPerUrl) {
console.log("Buiding sitemap in file: "+fileName+"\n");
let sitemap = "\n" +
- "";
+ "";
fs.writeFile("./"+fileName, sitemap, function(err) {
if(err) {
@@ -248,9 +248,9 @@ let notin= 0;
let communitiesPromise;
let publicCommunities = [];
-const refineUrl = "https://beta.services.openaire.eu/search/v2/api/resources2/?format=json&refine=true&fields=resultbestaccessright&fields=relfunder&fields=instancetypename&fields=resultlanguagename&fields=community&fields=resulthostingdatasource&fields=country&type=results&page=0&size=0";
-const resultsUrlPrefix = "https://beta.services.openaire.eu/search/v2/api/resources2/?format=json";
-const landingPrefix = "https://beta.explore.openaire.eu/search/";
-const contextUrl = "https://beta.services.openaire.eu/openaire/contexts/";
+const refineUrl = "https://services.openaire.eu/search/v2/api/resources2/?format=json&refine=true&fields=resultbestaccessright&fields=relfunder&fields=instancetypename&fields=resultlanguagename&fields=community&fields=resulthostingdatasource&fields=country&type=results&page=0&size=0";
+const resultsUrlPrefix = "https://services.openaire.eu/search/v2/api/resources2/?format=json";
+const landingPrefix = "https://explore.openaire.eu/search/";
+const contextUrl = "https://services.openaire.eu/openaire/contexts/";
buildSiteMap(100);