[Trunk | Explore Services]: extractUrlsFromSearch.ts: Fixes in script | Services on production, not on beta.
git-svn-id: https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/uoa-services-portal/trunk@61367 d315682c-612b-4755-9ff5-7f18f6832af3
This commit is contained in:
parent
c2c81e7f77
commit
6dbdecafb4
|
@ -56,7 +56,9 @@ function get(resultsPerUrl) {
|
|||
|
||||
const url = resultsUrlPrefix + "&fq=" + key + " " + fieldIdsMap[key].equalityOperator + " \"" + encodeURIComponent(value.id) + "\"" + "&type=results&page=0&size=" + resultsPerUrl;
|
||||
|
||||
promiseArray.push(new Promise((resolve, reject) => {
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
promiseArray.push(
|
||||
new Promise((resolve, reject) => {
|
||||
request.get(url, function (err: any, response: any) {
|
||||
if (!response && err) {
|
||||
reject(err);
|
||||
|
@ -67,8 +69,8 @@ function get(resultsPerUrl) {
|
|||
})
|
||||
}).catch(error => {
|
||||
console.error("Error getting results ", error);
|
||||
fs.appendFileSync("./"+errorFileName, "no response "+url);
|
||||
fs.appendFileSync("./"+errorFileName, err);
|
||||
fs.appendFileSync("./"+errorFileName, "no response "+url+" ");
|
||||
fs.appendFileSync("./"+errorFileName, error);
|
||||
fs.appendFileSync("./"+errorFileName, "\n");
|
||||
}));
|
||||
}
|
||||
|
@ -129,7 +131,7 @@ function parseAllUrls(response: any, allUrls: any) {
|
|||
let pid: any = Identifier.getResultPIDFromIdentifiers(resultPreview.identifiers);
|
||||
let url;
|
||||
if(pid && pid.id) {
|
||||
url = getUrlByType(resultPreview.resultType, pid, pid.id);
|
||||
url = getUrlByType(resultPreview.resultType, pid, encodeURIComponent(pid.id));
|
||||
} else {
|
||||
url = getUrlByType(resultPreview.resultType, null, resultPreview.id);
|
||||
}
|
||||
|
@ -137,10 +139,8 @@ function parseAllUrls(response: any, allUrls: any) {
|
|||
alreadyin++;
|
||||
} else {
|
||||
allUrls.add(url);
|
||||
let urlPre = "<url>\n" +
|
||||
" <loc>";
|
||||
let urlSuf = "</loc>\n" +
|
||||
" </url>";
|
||||
let urlPre = "\n<url><loc>";
|
||||
let urlSuf = "</loc></url>";
|
||||
fs.appendFileSync("./"+fileName, urlPre + url + urlSuf);
|
||||
notin++;
|
||||
}
|
||||
|
@ -225,7 +225,7 @@ function buildSiteMap(resultsPerUrl) {
|
|||
console.log("Buiding sitemap in file: "+fileName+"\n");
|
||||
|
||||
let sitemap = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
|
||||
"<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd\">";
|
||||
"<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">";
|
||||
|
||||
fs.writeFile("./"+fileName, sitemap, function(err) {
|
||||
if(err) {
|
||||
|
@ -248,9 +248,9 @@ let notin= 0;
|
|||
let communitiesPromise;
|
||||
let publicCommunities = [];
|
||||
|
||||
const refineUrl = "https://beta.services.openaire.eu/search/v2/api/resources2/?format=json&refine=true&fields=resultbestaccessright&fields=relfunder&fields=instancetypename&fields=resultlanguagename&fields=community&fields=resulthostingdatasource&fields=country&type=results&page=0&size=0";
|
||||
const resultsUrlPrefix = "https://beta.services.openaire.eu/search/v2/api/resources2/?format=json";
|
||||
const landingPrefix = "https://beta.explore.openaire.eu/search/";
|
||||
const contextUrl = "https://beta.services.openaire.eu/openaire/contexts/";
|
||||
const refineUrl = "https://services.openaire.eu/search/v2/api/resources2/?format=json&refine=true&fields=resultbestaccessright&fields=relfunder&fields=instancetypename&fields=resultlanguagename&fields=community&fields=resulthostingdatasource&fields=country&type=results&page=0&size=0";
|
||||
const resultsUrlPrefix = "https://services.openaire.eu/search/v2/api/resources2/?format=json";
|
||||
const landingPrefix = "https://explore.openaire.eu/search/";
|
||||
const contextUrl = "https://services.openaire.eu/openaire/contexts/";
|
||||
|
||||
buildSiteMap(100);
|
||||
|
|
Loading…
Reference in New Issue