MaDgIK
/
explore-services
13
0
Fork 0
Code Issues Pull Requests Projects Releases 17 Wiki Activity

[Trunk | Explore Services]: extractUrlsFromSearch.ts: Fixes in script | Services on production, not on beta. 

git-svn-id: https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/uoa-services-portal/trunk@61367 d315682c-612b-4755-9ff5-7f18f6832af3
This commit is contained in:
konstantina.galouni 2021-07-12 14:09:55 +00:00
parent c2c81e7f77
commit 6dbdecafb4
1 changed files with 13 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
services/sitemaps/extractUrlsFromSearch.ts
View File

@ -56,7 +56,9 @@ function get(resultsPerUrl) {
const url = resultsUrlPrefix + "&fq=" + key + " " + fieldIdsMap[key].equalityOperator + " \"" + encodeURIComponent(value.id) + "\"" + "&type=results&page=0&size=" + resultsPerUrl; const url = resultsUrlPrefix + "&fq=" + key + " " + fieldIdsMap[key].equalityOperator + " \"" + encodeURIComponent(value.id) + "\"" + "&type=results&page=0&size=" + resultsPerUrl;
promiseArray.push(new Promise((resolve, reject) => { await new Promise(resolve => setTimeout(resolve, 500));
promiseArray.push(
new Promise((resolve, reject) => {
request.get(url, function (err: any, response: any) { request.get(url, function (err: any, response: any) {
if (!response && err) { if (!response && err) {
reject(err); reject(err);
@ -67,8 +69,8 @@ function get(resultsPerUrl) {
}) })
}).catch(error => { }).catch(error => {
console.error("Error getting results ", error); console.error("Error getting results ", error);
fs.appendFileSync("./"+errorFileName, "no response "+url); fs.appendFileSync("./"+errorFileName, "no response "+url+" ");
fs.appendFileSync("./"+errorFileName, err); fs.appendFileSync("./"+errorFileName, error);
fs.appendFileSync("./"+errorFileName, "\n"); fs.appendFileSync("./"+errorFileName, "\n");
})); }));
} }
@ -129,7 +131,7 @@ function parseAllUrls(response: any, allUrls: any) {
let pid: any = Identifier.getResultPIDFromIdentifiers(resultPreview.identifiers); let pid: any = Identifier.getResultPIDFromIdentifiers(resultPreview.identifiers);
let url; let url;
if(pid && pid.id) { if(pid && pid.id) {
url = getUrlByType(resultPreview.resultType, pid, pid.id); url = getUrlByType(resultPreview.resultType, pid, encodeURIComponent(pid.id));
} else { } else {
url = getUrlByType(resultPreview.resultType, null, resultPreview.id); url = getUrlByType(resultPreview.resultType, null, resultPreview.id);
} }
@ -137,10 +139,8 @@ function parseAllUrls(response: any, allUrls: any) {
alreadyin++; alreadyin++;
} else { } else {
allUrls.add(url); allUrls.add(url);
let urlPre = "<url>\n" + let urlPre = "\n<url><loc>";
" <loc>"; let urlSuf = "</loc></url>";
let urlSuf = "</loc>\n" +
" </url>";
fs.appendFileSync("./"+fileName, urlPre + url + urlSuf); fs.appendFileSync("./"+fileName, urlPre + url + urlSuf);
notin++; notin++;
} }
@ -225,7 +225,7 @@ function buildSiteMap(resultsPerUrl) {
console.log("Buiding sitemap in file: "+fileName+"\n"); console.log("Buiding sitemap in file: "+fileName+"\n");
let sitemap = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + let sitemap = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
"<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd\">"; "<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">";
fs.writeFile("./"+fileName, sitemap, function(err) { fs.writeFile("./"+fileName, sitemap, function(err) {
if(err) { if(err) {
@ -248,9 +248,9 @@ let notin= 0;
let communitiesPromise; let communitiesPromise;
let publicCommunities = []; let publicCommunities = [];
const refineUrl = "https://beta.services.openaire.eu/search/v2/api/resources2/?format=json&refine=true&fields=resultbestaccessright&fields=relfunder&fields=instancetypename&fields=resultlanguagename&fields=community&fields=resulthostingdatasource&fields=country&type=results&page=0&size=0"; const refineUrl = "https://services.openaire.eu/search/v2/api/resources2/?format=json&refine=true&fields=resultbestaccessright&fields=relfunder&fields=instancetypename&fields=resultlanguagename&fields=community&fields=resulthostingdatasource&fields=country&type=results&page=0&size=0";
const resultsUrlPrefix = "https://beta.services.openaire.eu/search/v2/api/resources2/?format=json"; const resultsUrlPrefix = "https://services.openaire.eu/search/v2/api/resources2/?format=json";
const landingPrefix = "https://beta.explore.openaire.eu/search/"; const landingPrefix = "https://explore.openaire.eu/search/";
const contextUrl = "https://beta.services.openaire.eu/openaire/contexts/"; const contextUrl = "https://services.openaire.eu/openaire/contexts/";
buildSiteMap(100); buildSiteMap(100);