MaDgIK
/
explore-services
13
0
Fork 0
Code Issues Pull Requests Projects Releases 17 Wiki Activity

[Trunk | Explore Services]: extractUrlsFromSearch.ts: Fixes in script | Services on production, not on beta. 

git-svn-id: https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/uoa-services-portal/trunk@61367 d315682c-612b-4755-9ff5-7f18f6832af3
This commit is contained in:
konstantina.galouni 2021-07-12 14:09:55 +00:00
parent c2c81e7f77
commit 6dbdecafb4
1 changed files with 13 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
services/sitemaps/extractUrlsFromSearch.ts
View File

@ -56,7 +56,9 @@ function get(resultsPerUrl) {
const url = resultsUrlPrefix + "&fq=" + key + " " + fieldIdsMap[key].equalityOperator + " \"" + encodeURIComponent(value.id) + "\"" + "&type=results&page=0&size=" + resultsPerUrl;
promiseArray.push(new Promise((resolve, reject) => {
await new Promise(resolve => setTimeout(resolve, 500));
promiseArray.push(
new Promise((resolve, reject) => {
request.get(url, function (err: any, response: any) {
if (!response && err) {
reject(err);
@ -67,8 +69,8 @@ function get(resultsPerUrl) {
})
}).catch(error => {
console.error("Error getting results ", error);
fs.appendFileSync("./"+errorFileName, "no response "+url);
fs.appendFileSync("./"+errorFileName, err);
fs.appendFileSync("./"+errorFileName, "no response "+url+" ");
fs.appendFileSync("./"+errorFileName, error);
fs.appendFileSync("./"+errorFileName, "\n");
}));
}
@ -129,7 +131,7 @@ function parseAllUrls(response: any, allUrls: any) {
let pid: any = Identifier.getResultPIDFromIdentifiers(resultPreview.identifiers);
let url;
if(pid && pid.id) {
url = getUrlByType(resultPreview.resultType, pid, pid.id);
url = getUrlByType(resultPreview.resultType, pid, encodeURIComponent(pid.id));
} else {
url = getUrlByType(resultPreview.resultType, null, resultPreview.id);
}
@ -137,10 +139,8 @@ function parseAllUrls(response: any, allUrls: any) {
alreadyin++;
} else {
allUrls.add(url);
let urlPre = "<url>\n" +
" <loc>";
let urlSuf = "</loc>\n" +
" </url>";
let urlPre = "\n<url><loc>";
let urlSuf = "</loc></url>";
fs.appendFileSync("./"+fileName, urlPre + url + urlSuf);
notin++;
}
@ -225,7 +225,7 @@ function buildSiteMap(resultsPerUrl) {
console.log("Buiding sitemap in file: "+fileName+"\n");
let sitemap = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
"<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd\">";
"<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">";
fs.writeFile("./"+fileName, sitemap, function(err) {
if(err) {
@ -248,9 +248,9 @@ let notin= 0;
let communitiesPromise;
let publicCommunities = [];
const refineUrl = "https://beta.services.openaire.eu/search/v2/api/resources2/?format=json&refine=true&fields=resultbestaccessright&fields=relfunder&fields=instancetypename&fields=resultlanguagename&fields=community&fields=resulthostingdatasource&fields=country&type=results&page=0&size=0";
const resultsUrlPrefix = "https://beta.services.openaire.eu/search/v2/api/resources2/?format=json";
const landingPrefix = "https://beta.explore.openaire.eu/search/";
const contextUrl = "https://beta.services.openaire.eu/openaire/contexts/";
const refineUrl = "https://services.openaire.eu/search/v2/api/resources2/?format=json&refine=true&fields=resultbestaccessright&fields=relfunder&fields=instancetypename&fields=resultlanguagename&fields=community&fields=resulthostingdatasource&fields=country&type=results&page=0&size=0";
const resultsUrlPrefix = "https://services.openaire.eu/search/v2/api/resources2/?format=json";
const landingPrefix = "https://explore.openaire.eu/search/";
const contextUrl = "https://services.openaire.eu/openaire/contexts/";
buildSiteMap(100);