[Explore Service | Services]:

1. extractUrlsFromSearch.ts: changed results parsing (used landing parsing instead of searh) to call method "checkIfAllowed()" (no index filtering) | Added more numbers for statistics.
2. package.json: Upgrade node version.

services/sitemaps/extractUrlsFromSearch.ts

@@ -7,6 +7,9 @@ import {ResultPreview} from "../../explore/src/app/openaireLibrary/utils/result-
 import {Identifier} from "../../explore/src/app/openaireLibrary/utils/string-utils.class";
 import {SearchFields} from "../../explore/src/app/openaireLibrary/utils/properties/searchFields";
 import {ContextsService} from "../../explore/src/app/openaireLibrary/claims/claim-utils/service/contexts.service";
+import {ResultLandingComponent} from "../../explore/src/app/openaireLibrary/landingPages/result/resultLanding.component";
+import {ResultLandingService} from "../../explore/src/app/openaireLibrary/landingPages/result/resultLanding.service";
+import {ResultLandingInfo} from "../../explore/src/app/openaireLibrary/utils/entities/resultLandingInfo";
 
 const request = require('superagent');
 
@@ -27,6 +30,7 @@ function get(resultsPerUrl) {
         let promiseArray = [];
 
         for (let key of keys) {
+          // comment out for communities query check
           if(key == "community") {
             await communitiesPromise;
           }
@@ -48,7 +52,8 @@ function get(resultsPerUrl) {
                   valueId = idArray[0];
                 }
               }
-              if(!valueId || !publicCommunities.includes(valueId)) {
+              // if(!valueId) {
+              if(!valueId || !publicCommunities.includes(valueId)) { // comment out for communities query check
                 console.log("hidden community: "+valueId);
                 continue;
               }
@@ -64,7 +69,7 @@ function get(resultsPerUrl) {
                   reject(err);
                 } else {
                   parseAllUrls(response, allUrls);
-                  resolve();
+                  resolve(value => value);
                 }
               })
             }).catch(error => {
@@ -79,6 +84,9 @@ function get(resultsPerUrl) {
 
         await Promise.all(promiseArray);
         console.log("\nDuplicate urls: "+alreadyin + " vs unique urls: "+notin);
+        console.log("\nNo indexed urls: "+noIndexedUrls + " vs final urls: "+finalUrls);
+        console.log("\nPublications: "+publications + " - Datasets: "+datasets +
+          " - Software: "+software + " - Other: "+other + " --- urls with pid: "+urlsWithPid);
 
         fs.appendFile("./" + fileName, "\n</urlset>", function (err) {
           if (err) {
@@ -92,7 +100,7 @@ function get(resultsPerUrl) {
 }
 // });
 
-function parseAllUrls(response: any, allUrls: any) {
+function parseAllUrls_old(response: any, allUrls: any) {
   // let allUrls: any = [];
 
   let responses: any = response.body['results'];
@@ -123,7 +131,7 @@ function parseAllUrls(response: any, allUrls: any) {
   }
 
   if(searchResults.length == 0) {
-    fs.appendFileSync("./"+errorFileName, response.statusCode+"  "+response.request.url+"/n");
+    fs.appendFileSync("./"+errorFileName, response.statusCode+"  "+response.request.url+"\n");
   }
   for(let j=0; j<searchResults.length; j++) {
     let resultPreview: any = ResultPreview.searchResultConvert(searchResults[j], searchResults[j].entityType);
@@ -149,6 +157,93 @@ function parseAllUrls(response: any, allUrls: any) {
 }
 
 
+function parseAllUrls(response: any, allUrls: any) {
+  // let allUrls: any = [];
+
+  let responses: any = response.body['results'];
+  let length = responses ? (Array.isArray(responses) ? responses.length : 1) : 0;
+
+  if(length < 100 && length > 0) {
+    console.log("num of results: "+length + "   " + response.request.url);
+  }
+
+  if(length == 0) {
+    fs.appendFileSync("./"+errorFileName, response.statusCode+"  "+response.request.url+"\n");
+  }
+
+  for (let i = 0; i < length; i++) {
+    let curResponse = Array.isArray(responses) ? responses[i] : responses;
+    let resData = curResponse['result']['metadata']['oaf:entity'];
+
+    let param = [resData['oaf:result'], resData['oaf:result']['title'], resData['oaf:result']['rels']['rel'],
+                 resData['oaf:result']['children'], resData['oaf:result']['pid'], resData['oaf:result']['journal'],
+                 resData['oaf:result']['language'], resData['oaf:result']['subject'], resData['oaf:result']['context'],
+                 resData['oaf:result']['creator'], resData['oaf:result']['country'], resData['oaf:result']['programmingLanguage'],
+                 (resData['extraInfo'] !== undefined && resData['extraInfo']['citations'] !== undefined)
+                   ? resData['extraInfo']['citations']['citation'] : null,
+                 curResponse['result']['header']['dri:status'], curResponse];
+
+    let resultLandingService = new ResultLandingService(null);
+    let resultLandingInfo: ResultLandingInfo = resultLandingService.parseResultLandingInfo(param, null, properties);
+
+    let resultPreview: any = ResultPreview.resultLandingInfoConvert(resultLandingInfo, resultLandingInfo.resultType);
+
+    let pid: any = Identifier.getResultPIDFromIdentifiers(resultPreview.identifiers);
+    let url;
+    if(pid && pid.id) {
+      url = getUrlByType(resultPreview.resultType, pid, encodeURIComponent(pid.id));
+    } else {
+      url = getUrlByType(resultPreview.resultType, null, resultLandingInfo.relcanId);
+    }
+    if(allUrls.has(url)) {
+      alreadyin++;
+    } else {
+      let resultLandingComponent = new ResultLandingComponent(null, null, null,
+        null, null, null, null, null, null,
+        null, null, null, null, null);
+
+      if(!resultLandingComponent.checkIfAllowed(resultLandingInfo)) {
+        noIndexedUrls++;
+        fs.appendFileSync("./"+noIndexFileName, url+"\n");
+      } else {
+        finalUrls++;
+        let urlPre = "\n<url><loc>";
+        let urlSuf = "</loc></url>";
+        fs.appendFileSync("./"+fileName, urlPre + url + urlSuf);
+      }
+
+      allUrls.add(url);
+      notin++;
+    }
+
+  // .pipe(map(res => [res['result']['header']['dri:status'], res['result']['metadata']['oaf:entity'], res]))
+  //     .pipe(map(res => [
+  //       res[1]['oaf:result'],                 // 0
+  //       res[1]['oaf:result']['title'],        // 1
+  //       res[1]['oaf:result']['rels']['rel'],  // 2
+  //       res[1]['oaf:result']['children'],     // 3
+  //       res[1]['oaf:result']['pid'],          // 4
+  //       res[1]['oaf:result']['journal'],      // 5
+  //       res[1]['oaf:result']['language'],     // 6
+  //       res[1]['oaf:result']['subject'],      // 7
+  //       res[1]['oaf:result']['context'],      // 8
+  //       res[1]['oaf:result']['creator'],      // 9
+  //       res[1]['oaf:result']['country'] ,     // 10
+  //       res[1]['oaf:result']['programmingLanguage'],              // 11 - software
+  //       //res[1]['oaf:result']['resulttype'],
+  //       (res[1]['extraInfo'] !== undefined && res[1]['extraInfo']['citations'] !== undefined)
+  //         ? res[1]['extraInfo']['citations']['citation'] : null,  // 12
+  //       res[0],                               // 13
+  //       res[2]                                // 14
+  //     ]))
+  //     .pipe(map(res => this.parseResultLandingInfo(res, provenanceActionVocabulary, properties)));
+
+  }
+
+  return allUrls;
+}
+
+
 // function parseAllUrls1(response) {
 //   let allUrls = [];
 //
@@ -186,22 +281,28 @@ function getUrlByType(type: any, pid: any, id: any) {
   let parameter: any = "";
 
   if (type === "publication") {
+    publications++;
     parameter = "articleId";
   } else if (type === "dataset") {
+    datasets++;
     parameter = "datasetId";
   } else if (type === "software") {
+    software++;
     parameter = "softwareId";
   } else if (type === "other") {
+    other++;
     parameter = "orpId";
   } else {
     parameter = "id";
   }
   if(pid) {
+    urlsWithPid++;
     parameter = "pid";
   }
   return landingPrefix+type+"?"+parameter+"="+id;
 }
 
+// comment out for communities query check
 function getCommunities() {
   communitiesPromise = new Promise((resolve, reject) => {
     request.get(contextUrl, async function (err: any, communitiesResponse: any) {
@@ -210,7 +311,7 @@ function getCommunities() {
       } else {
         const contextsService = new ContextsService();
         publicCommunities = contextsService.parseCommunities(communitiesResponse.body, false).map(value => value.id);
-        resolve();
+        resolve(value => value);
       }
     })
   }).catch(error => console.error("Error getting communities ", error));
@@ -222,6 +323,7 @@ function buildSiteMap(resultsPerUrl) {
   let date = new Date();
   fileName = "sitemap_"+date.getFullYear()+"_"+(date.getMonth()+1)+"_"+date.getDate()+".xml";//+"_"+date.getTime();
   errorFileName = "error_"+date.getFullYear()+"_"+(date.getMonth()+1)+"_"+date.getDate()+".txt";//+"_"+date.getTime();
+  noIndexFileName = "noIndex_"+date.getFullYear()+"_"+(date.getMonth()+1)+"_"+date.getDate()+".txt";//+"_"+date.getTime();
   console.log("Buiding sitemap in file: "+fileName+"\n");
 
   let sitemap = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
@@ -233,24 +335,37 @@ function buildSiteMap(resultsPerUrl) {
     }
   });
 
-  getCommunities();
+  getCommunities();  // comment out for communities query check
   get(resultsPerUrl);
 }
 
 
 let fileName;
 let errorFileName;
+let noIndexFileName;
 const fs = require('fs');
 
 let alreadyin = 0;  // duplicate urls
 let notin= 0;
+let finalUrls = 0;
+let noIndexedUrls = 0;
+let publications = 0;
+let datasets = 0;
+let software = 0;
+let other = 0;
+let urlsWithPid = 0;
 
+// comment out for communities query check
 let communitiesPromise;
 let publicCommunities = [];
 
-const refineUrl = "https://services.openaire.eu/search/v2/api/resources2/?format=json&refine=true&fields=resultbestaccessright&fields=relfunder&fields=instancetypename&fields=resultlanguagename&fields=community&fields=resulthostingdatasource&fields=country&type=results&page=0&size=0";
+
 const resultsUrlPrefix = "https://services.openaire.eu/search/v2/api/resources2/?format=json";
 const landingPrefix = "https://explore.openaire.eu/search/";
+// const refineUrl = "https://services.openaire.eu/search/v2/api/resources2/?format=json&refine=true&fields=resultbestaccessright&fields=relfunder&fields=instancetypename&fields=resultlanguagename&fields=resulthostingdatasource&fields=country&type=results&page=0&size=0";
+
+// comment out for communities query check
+const refineUrl = "https://services.openaire.eu/search/v2/api/resources2/?format=json&refine=true&fields=resultbestaccessright&fields=relfunder&fields=instancetypename&fields=resultlanguagename&fields=community&fields=resulthostingdatasource&fields=country&type=results&page=0&size=0";
 const contextUrl = "https://services.openaire.eu/openaire/contexts/";
 
-buildSiteMap(100);
+buildSiteMap(150);

services/sitemaps/package.json

@@ -14,7 +14,7 @@
     "@types/node": "^8.0.30"
   },
   "engines": {
-    "node": "8.1.x"
+    "node": "16.3.0"
   },
   "author": "Konstantina Galouni <kgalouni@di.uoa.gr>",
   "license": "NKUA"