diff --git a/explore/src/app/app.component.ts b/explore/src/app/app.component.ts index 7de1b674..a17526e3 100644 --- a/explore/src/app/app.component.ts +++ b/explore/src/app/app.component.ts @@ -167,15 +167,15 @@ export class AppComponent { this.userMenuItems.push(new MenuItem("", "My profile", "", "", false, [], [], {})); this.userMenuItems.push(new MenuItem("", "My ORCID links", "", "/my-orcid-links", false, [], [""], {})); this.userMenuItems.push(new MenuItem("", "My links", "", "/myclaims", false, [], ["/myclaims"], {})); - let researchOutcomesMenu = new MenuItem("", OpenaireEntities.RESULTS, "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {resultbestaccessright: '"' + encodeURIComponent("Open Access") + '"'}); + let researchOutcomesMenu = new MenuItem("", OpenaireEntities.RESULTS, "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {}); researchOutcomesMenu.items = [ - new MenuItem("", OpenaireEntities.PUBLICATIONS, "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {type: '"' + encodeURIComponent("publications") + '"', resultbestaccessright: '"' + encodeURIComponent("Open Access") + '"'}), - new MenuItem("", OpenaireEntities.DATASETS, "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {type: '"' + encodeURIComponent("datasets") + '"', resultbestaccessright: '"' + encodeURIComponent("Open Access") + '"'}), - new MenuItem("", OpenaireEntities.SOFTWARE, "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {type: '"' + encodeURIComponent("software") + '"', resultbestaccessright: '"' + encodeURIComponent("Open Access") + '"'}), - new MenuItem("", OpenaireEntities.OTHER, "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {type: '"' + encodeURIComponent("other") + '"', resultbestaccessright: '"' + encodeURIComponent("Open Access") + '"'})]; + new MenuItem("", OpenaireEntities.PUBLICATIONS, "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {type: '"' + encodeURIComponent("publications") + '"'}), + new MenuItem("", OpenaireEntities.DATASETS, "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {type: '"' + encodeURIComponent("datasets") + '"'}), + new MenuItem("", OpenaireEntities.SOFTWARE, "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {type: '"' + encodeURIComponent("software") + '"'}), + new MenuItem("", OpenaireEntities.OTHER, "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {type: '"' + encodeURIComponent("other") + '"'})]; //TODO add check for research results route this.menuItems = [ - new MenuItem("search", "Search", "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {resultbestaccessright: '"' + encodeURIComponent("Open Access") + '"'}, + new MenuItem("search", "Search", "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {}, null, null, null, null, "_blank", "internal", false, [ researchOutcomesMenu, diff --git a/explore/src/app/home/home.component.ts b/explore/src/app/home/home.component.ts index 5bbbbfb7..f5cab70c 100644 --- a/explore/src/app/home/home.component.ts +++ b/explore/src/app/home/home.component.ts @@ -125,12 +125,12 @@ export class HomeComponent implements OnInit, OnDestroy, AfterViewInit { @ViewChild('contact') contact: ElementRef; subscriptions: any[] = []; @ViewChildren('scrolling_element') elements: QueryList; - resultsQuickFilter: { filter: Filter, selected: boolean, filterId: string, value: string } = { - filter: null, - selected: true, - filterId: "resultbestaccessright", - value: "Open Access" - }; + resultsQuickFilter: { filter: Filter, selected: boolean, filterId: string, value: string } = null;//{ + // filter: null, + // selected: true, + // filterId: "resultbestaccessright", + // value: "Open Access" + // }; selectedEntity = "all"; selectedEntitySimpleUrl; selectedEntityAdvancedUrl; diff --git a/services/cache/mecache/beta-properties.file b/services/cache/mecache/beta-properties.file index 471fad82..d39459e8 100644 --- a/services/cache/mecache/beta-properties.file +++ b/services/cache/mecache/beta-properties.file @@ -1 +1,2 @@ port = 4000 +utilsService = https://beta.explore.openaire.eu/utils-service diff --git a/services/cache/mecache/cache.js b/services/cache/mecache/cache.js index 8dba6912..435763a8 100644 --- a/services/cache/mecache/cache.js +++ b/services/cache/mecache/cache.js @@ -1,5 +1,5 @@ 'use strict'; - +const axios = require('axios'); let express = require('express'); let app = express(); let mcache = require('memory-cache'); @@ -8,13 +8,14 @@ const prom = require('prom-client'); const URL = require('url'); var PropertiesReader = require('properties-reader'); var properties = PropertiesReader('./properties.file'); -const expireShort = 2 * 60 * 1000; //2mins +const expireShort = 60 * 60 * 1000 // 1 hour //2 * 60 * 1000; //2mins const expireLong = 24 * 60 * 60 * 1000; //24 hours const cacheMaxSize = 500; const longCachingRequests = ["/communityFull", "/full", "/pagehelpcontent", "/provision/mvc/vocabularies/", "/pages?page_route=", "/allmetrics", "/countryusagestats/", "/openaire/info", - "/api/communities/", "/openaire/contexts/"]; + "/api/communities/", "/openaire/contexts/", + "/utils-service/explore/home", "/utils-service/explore/search", "/utils-service/explore/funders"]; let cors = require('cors'); app.use(cors()); @@ -142,6 +143,7 @@ const server = app.listen(properties.get('port'), function () { console.log(`Example app listening on port`, server.address().port) //run the timer resetAtMidnight(); + initCache(); }); function getResponse(code, message) { @@ -155,6 +157,18 @@ function clearCache() { console.log("cache is cleared!"); mcache.clear(); entries.set(mcache.size()); + initCache(); +} + +async function initCache() { + try { + const requests = await axios.get(properties.get('utilsService') + '/grouped-requests'); + const additionalDataPromises = requests.data.map((url) => axios.get('http://localhost:'+properties.get('port') + '/get?url=' + properties.get('utilsService') + url)); + const additionalDataResponses = await Promise.all(additionalDataPromises); + console.log("Cache initialized!") + } catch (error) { + console.error('Error fetching data: Cache initialize failed', error.message); + } } function checkForLongCachedRequests(url) { diff --git a/services/cache/mecache/package.json b/services/cache/mecache/package.json index e7530a97..1118af1a 100644 --- a/services/cache/mecache/package.json +++ b/services/cache/mecache/package.json @@ -10,6 +10,7 @@ "prepare-prod": " npm run prepare-dist; cp production-properties.file ./dist/properties.file" }, "dependencies": { + "axios": "^1.6.7", "cors": "^2.8.5", "express": "^4.18.2", "memory-cache": "^0.2.0", diff --git a/services/cache/mecache/production-properties.file b/services/cache/mecache/production-properties.file index 471fad82..40248189 100644 --- a/services/cache/mecache/production-properties.file +++ b/services/cache/mecache/production-properties.file @@ -1 +1,2 @@ port = 4000 +utilsService = https://explore.openaire.eu/utils-service diff --git a/services/cache/mecache/properties.file b/services/cache/mecache/properties.file index 74f6260b..6734892a 100644 --- a/services/cache/mecache/properties.file +++ b/services/cache/mecache/properties.file @@ -1 +1,2 @@ port = 3200 +utilsService= http://scoobydoo.di.uoa.gr:8000 diff --git a/services/utils-service/beta-properties.file b/services/utils-service/beta-properties.file index 7e37b386..e66a9f36 100644 --- a/services/utils-service/beta-properties.file +++ b/services/utils-service/beta-properties.file @@ -1,4 +1,6 @@ userInfoUrl = https://beta.services.openaire.eu/login-service/userInfo +searchServiceAPIUrl = https://beta.services.openaire.eu/search/v2/api/ +monitorAPIUrl = https://beta.services.openaire.eu/uoa-monitor-service/ ssl = true localPath = false # photo size in KB diff --git a/services/utils-service/package.json b/services/utils-service/package.json index 5ef016df..b816d04a 100644 --- a/services/utils-service/package.json +++ b/services/utils-service/package.json @@ -13,6 +13,7 @@ "author": "", "license": "ISC", "dependencies": { + "axios": "^1.6.7", "body-parser": "^1.20.2", "cookie-parser": "^1.4.6", "cors": "^2.8.5", diff --git a/services/utils-service/production-properties.file b/services/utils-service/production-properties.file index 93f676c1..539e3cd1 100644 --- a/services/utils-service/production-properties.file +++ b/services/utils-service/production-properties.file @@ -1,4 +1,6 @@ userInfoUrl = https://services.openaire.eu/login-service/userInfo +searchServiceAPIUrl = https://services.openaire.eu/search/v2/api/ +monitorAPIUrl = https://services.openaire.eu/uoa-monitor-service/ ssl = true localPath = false # photo size in KB diff --git a/services/utils-service/properties.file b/services/utils-service/properties.file index 4f43d8fc..8ab7aa3b 100644 --- a/services/utils-service/properties.file +++ b/services/utils-service/properties.file @@ -1,4 +1,6 @@ userInfoUrl = http://mpagasas.di.uoa.gr:19080/login-service/userInfo +searchServiceAPIUrl = https://beta.services.openaire.eu/search/v2/api/ +monitorAPIUrl = http://duffy.di.uoa.gr:19380/uoa-monitor-service/ ssl = false localPath = true # photo size in KB diff --git a/services/utils-service/uploadService.js b/services/utils-service/uploadService.js index c929c2e2..c26704a5 100644 --- a/services/utils-service/uploadService.js +++ b/services/utils-service/uploadService.js @@ -1,3 +1,4 @@ +const axios = require('axios'); var express = require("express"); var bodyParser = require("body-parser"); var cookieParser = require('cookie-parser'); @@ -12,6 +13,8 @@ if (properties.get('ssl')) { } else { http = require("http"); } +var searchServiceAPIUrl = properties.get('searchServiceAPIUrl'); +var monitorServiceAPIUrl = properties.get('monitorAPIUrl'); var auth = properties.get('userInfoUrl'); /** @deprecated*/ var authDeprecated = auth.includes("accessToken"); @@ -97,6 +100,181 @@ app.delete(['/delete/:filename', '/delete/stakeholder/:filename', '/delete/:type }); }); +app.get('/explore/home', async function (req, res) { + try { + // Make requests to multiple APIs + let requests= [ + searchServiceAPIUrl + 'publications/count?format=json', + searchServiceAPIUrl + 'datasets/count?format=json', + searchServiceAPIUrl + 'software/count?format=json', + searchServiceAPIUrl + 'other/count?format=json', + searchServiceAPIUrl +'results/?fields=relfunder&sf=relfunder&format=json&size=0', + searchServiceAPIUrl + 'datasources/count?format=json', + searchServiceAPIUrl + 'resources2/?format=json&size=0&type=organizations&fq=(reldatasourcecompatibilityid exact driver or reldatasourcecompatibilityid exact driver-openaire2.0 or reldatasourcecompatibilityid exact openaire2.0 or reldatasourcecompatibilityid exact openaire3.0 or reldatasourcecompatibilityid exact openaire4.0 or reldatasourcecompatibilityid exact openaire-cris_1.1 or reldatasourcecompatibilityid exact openaire2.0_data or reldatasourcecompatibilityid exact hostedBy or relproject=*)', + searchServiceAPIUrl + 'projects/?fields=funder&sf=funder&format=json&size=0', + searchServiceAPIUrl + 'resources?query=(%20(oaftype%20exact%20result)%20and%20(resulttypeid%20exact%20dataset)%20and%20(relresulttype%3Dpublication)%20%20)&page=0&size=0&format=json', + searchServiceAPIUrl + 'resources?query=(%20(oaftype%20exact%20result)%20and%20(resulttypeid%20exact%20software)%20and%20(relresulttype%3Dpublication)%20%20)&page=0&size=0&format=json' + ]; + const dataPromises = requests.map((url) => axios.get( url)); + const dataResponses = await Promise.all(dataPromises); + // Determine if all additional requests were successful + const allRequestsSuccessful = dataResponses.every((response) => response.status === 200); + // Combine the data + const aggregatedData = { + publications: dataResponses[0].data.total, + datasets: dataResponses[1].data.total, + software: dataResponses[2].data.total, + other: dataResponses[3].data.total, + results: dataResponses[4].data.meta.total, + /*resultFunders:resultRES.data.refineResults.relfunder.length,*/ + datasources: dataResponses[5].data.total, + organizations: dataResponses[6].data.meta.total, + projects:dataResponses[7].data.meta.total, + /*projectFunders:projectsRES.data.refineResults.funder.length,*/ + funders: parseNoOfFunders(dataResponses[4].data, dataResponses[7].data), + datasetsInterlinked:dataResponses[8].data.meta.total, + softwareInterlinked:dataResponses[9].data.meta.total, + success:allRequestsSuccessful + + }; + // Send the aggregated data as the response + res.status(allRequestsSuccessful?200:207).json(aggregatedData); + } catch (error) { + console.error('Error fetching data:', error.message); + res.status(500).send('Internal Server Error'); + } +}); + +app.get('/explore/search', async function (req, res) { + let aggregatedData = {}; + try { + // Make requests to multiple APIs + let requests= [ + searchServiceAPIUrl +'resources2/?format=json&size=0&type=results', + searchServiceAPIUrl + 'datasources/count?format=json', + searchServiceAPIUrl + 'resources2/?format=json&size=0&type=organizations&fq=(reldatasourcecompatibilityid exact driver or reldatasourcecompatibilityid exact driver-openaire2.0 or reldatasourcecompatibilityid exact openaire2.0 or reldatasourcecompatibilityid exact openaire3.0 or reldatasourcecompatibilityid exact openaire4.0 or reldatasourcecompatibilityid exact openaire-cris_1.1 or reldatasourcecompatibilityid exact openaire2.0_data or reldatasourcecompatibilityid exact hostedBy or relproject=*)', + searchServiceAPIUrl + 'projects/count?format=json' + ] + const dataPromises = requests.map((url) => axios.get( url)); + const dataResponses = await Promise.all(dataPromises); + // Determine if all additional requests were successful + const allRequestsSuccessful = dataResponses.every((response) => response.status === 200); + // Combine the data + aggregatedData = { + results: dataResponses[0].data.meta.total, + datasources: dataResponses[1].data.total, + organizations: dataResponses[2].data.meta.total, + projects:dataResponses[3].data.total, + success:allRequestsSuccessful + + }; + + // Send the aggregated data as the response + res.status(allRequestsSuccessful?200:207).json(aggregatedData); + } catch (error) { + console.log(aggregatedData) + console.error('Error fetching data:', error); + res.status(500).send('Internal Server Error'); + } +}); + +app.get('/explore/funders', async function (req, res) { + let aggregatedData = {}; + try { + // Make requests to multiple APIs + let requests= [ + searchServiceAPIUrl + 'resources2/?format=json&type=results&fq=relfunder=*&refine=true&fields=relfunder&sf=relfunder&page=0&size=0', + searchServiceAPIUrl + 'resources2/?format=json&type=results&fq=relfunder=*&refine=true&fields=relfunder&sf=relfunder&page=0&size=0&fq=resultbestaccessright%20exact%20%22Open%20Access%22', + searchServiceAPIUrl + 'resources2/?format=json&type=projects&fq=funder=*&fq=projecttitle<>"unidentified"&refine=true&fields=funder&sf=funder&page=0&size=0', + monitorServiceAPIUrl + 'stakeholder?type=funder', + + ] + const dataPromises = requests.map((url) => axios.get( url)); + const dataResponses = await Promise.all(dataPromises); + // Determine if all additional requests were successful + const allRequestsSuccessful = dataResponses.every((response) => response.status === 200); + let fundersMap = new Map(); + + let resultsFunders = dataResponses[0].data.refineResults.relfunder; + resultsFunders.forEach(queriedFunder => { + if (!fundersMap.has(queriedFunder.id)) { + fundersMap.set(queriedFunder.id,{name: queriedFunder.name.split("||")[0], id: queriedFunder.id, results:queriedFunder.count, openResults: null, projects:null, stakeholder:null}); + } + }); + let openResultsFunders = dataResponses[1].data.refineResults.relfunder; + openResultsFunders.forEach(queriedFunder => { + if (!fundersMap.has(queriedFunder.id)) { + fundersMap.set(queriedFunder.id,{name: queriedFunder.name.split("||")[0], id: queriedFunder.id, results:null, openResults: queriedFunder.count, projects:null, stakeholder:null}); + }else{ + fundersMap.get(queriedFunder.id).openResults = queriedFunder.count; + } + }); + let projectFunders = dataResponses[2].data.refineResults.funder; + projectFunders.forEach(queriedFunder => { + if (!fundersMap.has(queriedFunder.id) ) { + fundersMap.set(queriedFunder.id,{name: queriedFunder.name.split("||")[0], id: queriedFunder.id, results:null, openResults: null, projects:queriedFunder.count, stakeholder:null}); + }else{ + fundersMap.get(queriedFunder.id).projects = queriedFunder.count; + } + }); + let stakeholders = dataResponses[3].data; + stakeholders.forEach(stakeholder => { + let id = stakeholder.index_id + "||" + stakeholder.index_name + "||" + stakeholder.index_shortName; + // console.log(id); + if (fundersMap.has(id)) { + let ministakeholder = {id:id, name:stakeholder.name, alias: stakeholder.alias, visibility: stakeholder.visibility, + logoUrl:stakeholder.logoUrl, isUpload: stakeholder.isUpload} + fundersMap.get(id).stakeholder = ministakeholder; + } + }); + + // Combine the data + + // Send the aggregated data as the response + // console.log(fundersMap) + aggregatedData = { + count: fundersMap.size, + results: dataResponses[0].data.meta.total, + projects: dataResponses[2].data.meta.total, + funders: Array.from(fundersMap.values()), + success:allRequestsSuccessful + + }; + res.status(allRequestsSuccessful?200:207).json(aggregatedData); + } catch (error) { + // console.log(aggregatedData) + console.error('Error fetching data:', error); + res.status(500).send('Internal Server Error'); + } +}); +app.get('/grouped-requests', async function (req, res) { + res.json([ + "/explore/search", + "/explore/home", + "/explore/funders" + ]); +}); +function parseNoOfFunders(resultRES, projectsRES){ + // combines the refines qeries on funders field, the funders with results and the funders that have at least one project + let mergedFundersSet = new Set(); + let queriedFunders = resultRES.refineResults.relfunder; + queriedFunders.forEach(queriedFunder => { + if (!mergedFundersSet.has(queriedFunder.id)) { + mergedFundersSet.add(queriedFunder.id); + } + }); + queriedFunders = projectsRES.refineResults.funder; + queriedFunders.forEach(queriedFunder => { + if(+queriedFunder.count > 1) { + if (!mergedFundersSet.has(queriedFunder.id)) { + mergedFundersSet.add(queriedFunder.id); + } + } + }); + return mergedFundersSet.size; +} + + const server = app.listen(properties.get('port'), function () { console.log("Listening on port %s...", server.address().port); });