From 3678f7c5eae274f40079a96d3301bbab10d3efdf Mon Sep 17 00:00:00 2001 From: "konstantina.galouni" Date: Fri, 2 Feb 2024 12:12:57 +0200 Subject: [PATCH 1/9] [cache | develop | DONE | CHANGED]: cache.js: Increased time of short cache from 2 minutes to 1 hour. --- services/cache/mecache/cache.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/cache/mecache/cache.js b/services/cache/mecache/cache.js index 8dba6912..90d5d947 100644 --- a/services/cache/mecache/cache.js +++ b/services/cache/mecache/cache.js @@ -8,7 +8,7 @@ const prom = require('prom-client'); const URL = require('url'); var PropertiesReader = require('properties-reader'); var properties = PropertiesReader('./properties.file'); -const expireShort = 2 * 60 * 1000; //2mins +const expireShort = 60 * 60 * 1000 // 1 hour //2 * 60 * 1000; //2mins const expireLong = 24 * 60 * 60 * 1000; //24 hours const cacheMaxSize = 500; const longCachingRequests = ["/communityFull", "/full", "/pagehelpcontent", From 109be1c6f633a4cf5ee5523a5caf1295adb117eb Mon Sep 17 00:00:00 2001 From: "konstantina.galouni" Date: Fri, 2 Feb 2024 12:44:54 +0200 Subject: [PATCH 2/9] [cache | develop | DONE | ADDED]: cache.js: Added in "longCachingRequests", the endpoint "/utils-service/explore/home" to be used as a grouped response for the numbers in explore home page. --- services/cache/mecache/cache.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/cache/mecache/cache.js b/services/cache/mecache/cache.js index 90d5d947..73d1e472 100644 --- a/services/cache/mecache/cache.js +++ b/services/cache/mecache/cache.js @@ -14,7 +14,7 @@ const cacheMaxSize = 500; const longCachingRequests = ["/communityFull", "/full", "/pagehelpcontent", "/provision/mvc/vocabularies/", "/pages?page_route=", "/allmetrics", "/countryusagestats/", "/openaire/info", - "/api/communities/", "/openaire/contexts/"]; + "/api/communities/", "/openaire/contexts/", "/utils-service/explore/home"]; let cors = require('cors'); app.use(cors()); From 102f41722edbafa036d01d75e1ef2f2533531894 Mon Sep 17 00:00:00 2001 From: "konstantina.galouni" Date: Mon, 5 Feb 2024 11:55:23 +0200 Subject: [PATCH 3/9] Updating openaireLibrary & common-assets --- explore/src/app/openaireLibrary | 2 +- explore/src/assets/common-assets | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/explore/src/app/openaireLibrary b/explore/src/app/openaireLibrary index 243e7363..b12877df 160000 --- a/explore/src/app/openaireLibrary +++ b/explore/src/app/openaireLibrary @@ -1 +1 @@ -Subproject commit 243e73636289970338f3edf5c5b23fe4fa9df6b3 +Subproject commit b12877df2a5fb4d9d136edec330b1c5c82e8eb34 diff --git a/explore/src/assets/common-assets b/explore/src/assets/common-assets index ee22e97b..6cef9ebe 160000 --- a/explore/src/assets/common-assets +++ b/explore/src/assets/common-assets @@ -1 +1 @@ -Subproject commit ee22e97b114be823b7ab4e6c41a9cbff93b24e3a +Subproject commit 6cef9ebed0db7ecbcca023bae824239106907960 From a9d06e61fab65851a7b40c539c2ebaacf8581b5f Mon Sep 17 00:00:00 2001 From: argirok Date: Mon, 5 Feb 2024 12:07:34 +0200 Subject: [PATCH 4/9] [ develop | DONE | ADDED] Utils services: group common requests in a single request --- services/utils-service/beta-properties.file | 1 + services/utils-service/package.json | 1 + .../utils-service/production-properties.file | 1 + services/utils-service/properties.file | 1 + services/utils-service/uploadService.js | 93 +++++++++++++++++++ 5 files changed, 97 insertions(+) diff --git a/services/utils-service/beta-properties.file b/services/utils-service/beta-properties.file index 7e37b386..1a473047 100644 --- a/services/utils-service/beta-properties.file +++ b/services/utils-service/beta-properties.file @@ -1,4 +1,5 @@ userInfoUrl = https://beta.services.openaire.eu/login-service/userInfo +searchServiceAPIUrl = https://beta.services.openaire.eu/search/v2/api/ ssl = true localPath = false # photo size in KB diff --git a/services/utils-service/package.json b/services/utils-service/package.json index 5ef016df..b816d04a 100644 --- a/services/utils-service/package.json +++ b/services/utils-service/package.json @@ -13,6 +13,7 @@ "author": "", "license": "ISC", "dependencies": { + "axios": "^1.6.7", "body-parser": "^1.20.2", "cookie-parser": "^1.4.6", "cors": "^2.8.5", diff --git a/services/utils-service/production-properties.file b/services/utils-service/production-properties.file index 93f676c1..c2f029a3 100644 --- a/services/utils-service/production-properties.file +++ b/services/utils-service/production-properties.file @@ -1,4 +1,5 @@ userInfoUrl = https://services.openaire.eu/login-service/userInfo +searchServiceAPIUrl = https://services.openaire.eu/search/v2/api/ ssl = true localPath = false # photo size in KB diff --git a/services/utils-service/properties.file b/services/utils-service/properties.file index 4f43d8fc..fd1565a4 100644 --- a/services/utils-service/properties.file +++ b/services/utils-service/properties.file @@ -1,4 +1,5 @@ userInfoUrl = http://mpagasas.di.uoa.gr:19080/login-service/userInfo +searchServiceAPIUrl = https://beta.services.openaire.eu/search/v2/api/ ssl = false localPath = true # photo size in KB diff --git a/services/utils-service/uploadService.js b/services/utils-service/uploadService.js index c929c2e2..ab5e7de6 100644 --- a/services/utils-service/uploadService.js +++ b/services/utils-service/uploadService.js @@ -1,3 +1,4 @@ +const axios = require('axios'); var express = require("express"); var bodyParser = require("body-parser"); var cookieParser = require('cookie-parser'); @@ -12,6 +13,7 @@ if (properties.get('ssl')) { } else { http = require("http"); } +var searchServiceAPIUrl = properties.get('searchServiceAPIUrl'); var auth = properties.get('userInfoUrl'); /** @deprecated*/ var authDeprecated = auth.includes("accessToken"); @@ -97,6 +99,97 @@ app.delete(['/delete/:filename', '/delete/stakeholder/:filename', '/delete/:type }); }); +app.get('/explore/home', async function (req, res) { + try { + // Make requests to multiple APIs + const publicationsRES = await axios.get(searchServiceAPIUrl + 'publications/count?format=json'); + const datasetsRES = await axios.get(searchServiceAPIUrl + 'datasets/count?format=json'); + const softwareRES = await axios.get(searchServiceAPIUrl + 'software/count?format=json'); + const otherRES = await axios.get(searchServiceAPIUrl + 'other/count?format=json'); + const resultRES = await axios.get(searchServiceAPIUrl +'results/?fields=relfunder&sf=relfunder&format=json&size=0'); + const datasourcesRES = await axios.get(searchServiceAPIUrl + 'datasources/count?format=json'); + const organizationsRES = await axios.get(searchServiceAPIUrl + 'resources2/?format=json&size=0&type=organizations&fq=(reldatasourcecompatibilityid exact driver or reldatasourcecompatibilityid exact driver-openaire2.0 or reldatasourcecompatibilityid exact openaire2.0 or reldatasourcecompatibilityid exact openaire3.0 or reldatasourcecompatibilityid exact openaire4.0 or reldatasourcecompatibilityid exact openaire-cris_1.1 or reldatasourcecompatibilityid exact openaire2.0_data or reldatasourcecompatibilityid exact hostedBy or relproject=*)'); + const projectsRES = await axios.get(searchServiceAPIUrl + 'projects/?fields=funder&sf=funder&format=json&size=0'); + const datasetsInterlinkedRES = await axios.get(searchServiceAPIUrl + 'resources?query=(%20(oaftype%20exact%20result)%20and%20(resulttypeid%20exact%20dataset)%20and%20(relresulttype%3Dpublication)%20%20)&page=0&size=0&format=json'); + const softwareInterlinkedRES = await axios.get(searchServiceAPIUrl + 'resources?query=(%20(oaftype%20exact%20result)%20and%20(resulttypeid%20exact%20software)%20and%20(relresulttype%3Dpublication)%20%20)&page=0&size=0&format=json'); + + // Combine the data + const aggregatedData = { + publications: publicationsRES.data.total, + datasets: datasetsRES.data.total, + software: softwareRES.data.total, + other: otherRES.data.total, + results: resultRES.data.meta.total, + /*resultFunders:resultRES.data.refineResults.relfunder.length,*/ + datasources: datasourcesRES.data.total, + organizations: organizationsRES.data.meta.total?organizationsRES.data.meta.total:'--', + projects:projectsRES.data.meta.total, + /*projectFunders:projectsRES.data.refineResults.funder.length,*/ + funders: parseNoOfFunders(resultRES, projectsRES), + datasetsInterlinked:datasetsInterlinkedRES.data.meta.total, + softwareInterlinked:softwareInterlinkedRES.data.meta.total, + + }; + + // Send the aggregated data as the response + res.json(aggregatedData); + } catch (error) { + console.error('Error fetching data:', error.message); + res.status(500).send('Internal Server Error'); + } +}); + +app.get('/explore/search', async function (req, res) { + try { + // Make requests to multiple APIs + const resultRES = await axios.get(searchServiceAPIUrl +'resources2/?format=json&size=0&type=results&fq=resultbestaccessright%20exact%20%22Open%20Access%22'); + const datasourcesRES = await axios.get(searchServiceAPIUrl + 'datasources/count?format=json'); + const organizationsRES = await axios.get(searchServiceAPIUrl + 'resources2/?format=json&size=0&type=organizations&fq=(reldatasourcecompatibilityid exact driver or reldatasourcecompatibilityid exact driver-openaire2.0 or reldatasourcecompatibilityid exact openaire2.0 or reldatasourcecompatibilityid exact openaire3.0 or reldatasourcecompatibilityid exact openaire4.0 or reldatasourcecompatibilityid exact openaire-cris_1.1 or reldatasourcecompatibilityid exact openaire2.0_data or reldatasourcecompatibilityid exact hostedBy or relproject=*)'); + const projectsRES = await axios.get(searchServiceAPIUrl + 'projects/count?format=json'); + + // Combine the data + const aggregatedData = { + results: resultRES.data.meta.total, + datasources: datasourcesRES.data.total, + organizations: organizationsRES.data.meta.total?organizationsRES.data.meta.total:'--', + projects:projectsRES.data.total + + }; + + // Send the aggregated data as the response + res.json(aggregatedData); + } catch (error) { + console.error('Error fetching data:', error.message); + res.status(500).send('Internal Server Error'); + } +}); +app.get('/grouped-requests', async function (req, res) { + res.json([ + "/explore/search", + "/explore/home" + ]); +}); +function parseNoOfFunders(resultRES, projectsRES){ + // combines the refines qeries on funders field, the funders with results and the funders that have at least one project + let mergedFundersSet = new Set(); + let queriedFunders = resultRES.data.refineResults.relfunder; + queriedFunders.forEach(queriedFunder => { + if (!mergedFundersSet.has(queriedFunder.id)) { + mergedFundersSet.add(queriedFunder.id); + } + }); + queriedFunders = projectsRES.data.refineResults.funder; + queriedFunders.forEach(queriedFunder => { + if(+queriedFunder.count > 1) { + if (!mergedFundersSet.has(queriedFunder.id)) { + mergedFundersSet.add(queriedFunder.id); + } + } + }); + return mergedFundersSet.size; +} + + const server = app.listen(properties.get('port'), function () { console.log("Listening on port %s...", server.address().port); }); From 629d3efb082abd5cfe30202922197a80c244bc4e Mon Sep 17 00:00:00 2001 From: argirok Date: Mon, 5 Feb 2024 12:08:34 +0200 Subject: [PATCH 5/9] [ develop | DONE | ADDED] Cache services: add initialization method to initiate grouped requests from utils service. Init after clear and when cache starts --- services/cache/mecache/beta-properties.file | 1 + services/cache/mecache/cache.js | 15 ++++++++++++++- services/cache/mecache/package.json | 1 + services/cache/mecache/production-properties.file | 1 + services/cache/mecache/properties.file | 1 + 5 files changed, 18 insertions(+), 1 deletion(-) diff --git a/services/cache/mecache/beta-properties.file b/services/cache/mecache/beta-properties.file index 471fad82..d39459e8 100644 --- a/services/cache/mecache/beta-properties.file +++ b/services/cache/mecache/beta-properties.file @@ -1 +1,2 @@ port = 4000 +utilsService = https://beta.explore.openaire.eu/utils-service diff --git a/services/cache/mecache/cache.js b/services/cache/mecache/cache.js index 73d1e472..11710b4e 100644 --- a/services/cache/mecache/cache.js +++ b/services/cache/mecache/cache.js @@ -1,5 +1,5 @@ 'use strict'; - +const axios = require('axios'); let express = require('express'); let app = express(); let mcache = require('memory-cache'); @@ -142,6 +142,7 @@ const server = app.listen(properties.get('port'), function () { console.log(`Example app listening on port`, server.address().port) //run the timer resetAtMidnight(); + initCache(); }); function getResponse(code, message) { @@ -155,6 +156,18 @@ function clearCache() { console.log("cache is cleared!"); mcache.clear(); entries.set(mcache.size()); + initCache(); +} + +async function initCache() { + try { + const requests = await axios.get(properties.get('utilsService') + '/grouped-requests'); + const additionalDataPromises = requests.data.map((url) => axios.get('http://localhost:'+properties.get('port') + '/get?url=' + properties.get('utilsService') + url)); + const additionalDataResponses = await Promise.all(additionalDataPromises); + console.log("Cache initialized!") + } catch (error) { + console.error('Error fetching data: Cache initialize failed', error.message); + } } function checkForLongCachedRequests(url) { diff --git a/services/cache/mecache/package.json b/services/cache/mecache/package.json index e7530a97..1118af1a 100644 --- a/services/cache/mecache/package.json +++ b/services/cache/mecache/package.json @@ -10,6 +10,7 @@ "prepare-prod": " npm run prepare-dist; cp production-properties.file ./dist/properties.file" }, "dependencies": { + "axios": "^1.6.7", "cors": "^2.8.5", "express": "^4.18.2", "memory-cache": "^0.2.0", diff --git a/services/cache/mecache/production-properties.file b/services/cache/mecache/production-properties.file index 471fad82..40248189 100644 --- a/services/cache/mecache/production-properties.file +++ b/services/cache/mecache/production-properties.file @@ -1 +1,2 @@ port = 4000 +utilsService = https://explore.openaire.eu/utils-service diff --git a/services/cache/mecache/properties.file b/services/cache/mecache/properties.file index 74f6260b..6734892a 100644 --- a/services/cache/mecache/properties.file +++ b/services/cache/mecache/properties.file @@ -1 +1,2 @@ port = 3200 +utilsService= http://scoobydoo.di.uoa.gr:8000 From 46ad6aef36c332468aa52d46f236eb2295720e14 Mon Sep 17 00:00:00 2001 From: argirok Date: Tue, 6 Feb 2024 11:31:47 +0200 Subject: [PATCH 6/9] [ develop | DONE | ADDED] Utils services: add /explore/funders method that groups requests for funder page --- services/utils-service/beta-properties.file | 1 + .../utils-service/production-properties.file | 1 + services/utils-service/properties.file | 1 + services/utils-service/uploadService.js | 163 +++++++++++++----- 4 files changed, 127 insertions(+), 39 deletions(-) diff --git a/services/utils-service/beta-properties.file b/services/utils-service/beta-properties.file index 1a473047..e66a9f36 100644 --- a/services/utils-service/beta-properties.file +++ b/services/utils-service/beta-properties.file @@ -1,5 +1,6 @@ userInfoUrl = https://beta.services.openaire.eu/login-service/userInfo searchServiceAPIUrl = https://beta.services.openaire.eu/search/v2/api/ +monitorAPIUrl = https://beta.services.openaire.eu/uoa-monitor-service/ ssl = true localPath = false # photo size in KB diff --git a/services/utils-service/production-properties.file b/services/utils-service/production-properties.file index c2f029a3..539e3cd1 100644 --- a/services/utils-service/production-properties.file +++ b/services/utils-service/production-properties.file @@ -1,5 +1,6 @@ userInfoUrl = https://services.openaire.eu/login-service/userInfo searchServiceAPIUrl = https://services.openaire.eu/search/v2/api/ +monitorAPIUrl = https://services.openaire.eu/uoa-monitor-service/ ssl = true localPath = false # photo size in KB diff --git a/services/utils-service/properties.file b/services/utils-service/properties.file index fd1565a4..8ab7aa3b 100644 --- a/services/utils-service/properties.file +++ b/services/utils-service/properties.file @@ -1,5 +1,6 @@ userInfoUrl = http://mpagasas.di.uoa.gr:19080/login-service/userInfo searchServiceAPIUrl = https://beta.services.openaire.eu/search/v2/api/ +monitorAPIUrl = http://duffy.di.uoa.gr:19380/uoa-monitor-service/ ssl = false localPath = true # photo size in KB diff --git a/services/utils-service/uploadService.js b/services/utils-service/uploadService.js index ab5e7de6..b4f3df69 100644 --- a/services/utils-service/uploadService.js +++ b/services/utils-service/uploadService.js @@ -14,6 +14,7 @@ if (properties.get('ssl')) { http = require("http"); } var searchServiceAPIUrl = properties.get('searchServiceAPIUrl'); +var monitorServiceAPIUrl = properties.get('monitorAPIUrl'); var auth = properties.get('userInfoUrl'); /** @deprecated*/ var authDeprecated = auth.includes("accessToken"); @@ -102,37 +103,42 @@ app.delete(['/delete/:filename', '/delete/stakeholder/:filename', '/delete/:type app.get('/explore/home', async function (req, res) { try { // Make requests to multiple APIs - const publicationsRES = await axios.get(searchServiceAPIUrl + 'publications/count?format=json'); - const datasetsRES = await axios.get(searchServiceAPIUrl + 'datasets/count?format=json'); - const softwareRES = await axios.get(searchServiceAPIUrl + 'software/count?format=json'); - const otherRES = await axios.get(searchServiceAPIUrl + 'other/count?format=json'); - const resultRES = await axios.get(searchServiceAPIUrl +'results/?fields=relfunder&sf=relfunder&format=json&size=0'); - const datasourcesRES = await axios.get(searchServiceAPIUrl + 'datasources/count?format=json'); - const organizationsRES = await axios.get(searchServiceAPIUrl + 'resources2/?format=json&size=0&type=organizations&fq=(reldatasourcecompatibilityid exact driver or reldatasourcecompatibilityid exact driver-openaire2.0 or reldatasourcecompatibilityid exact openaire2.0 or reldatasourcecompatibilityid exact openaire3.0 or reldatasourcecompatibilityid exact openaire4.0 or reldatasourcecompatibilityid exact openaire-cris_1.1 or reldatasourcecompatibilityid exact openaire2.0_data or reldatasourcecompatibilityid exact hostedBy or relproject=*)'); - const projectsRES = await axios.get(searchServiceAPIUrl + 'projects/?fields=funder&sf=funder&format=json&size=0'); - const datasetsInterlinkedRES = await axios.get(searchServiceAPIUrl + 'resources?query=(%20(oaftype%20exact%20result)%20and%20(resulttypeid%20exact%20dataset)%20and%20(relresulttype%3Dpublication)%20%20)&page=0&size=0&format=json'); - const softwareInterlinkedRES = await axios.get(searchServiceAPIUrl + 'resources?query=(%20(oaftype%20exact%20result)%20and%20(resulttypeid%20exact%20software)%20and%20(relresulttype%3Dpublication)%20%20)&page=0&size=0&format=json'); - + let requests= [ + searchServiceAPIUrl + 'publications/count?format=json', + searchServiceAPIUrl + 'datasets/count?format=json', + searchServiceAPIUrl + 'software/count?format=json', + searchServiceAPIUrl + 'other/count?format=json', + searchServiceAPIUrl +'results/?fields=relfunder&sf=relfunder&format=json&size=0', + searchServiceAPIUrl + 'datasources/count?format=json', + searchServiceAPIUrl + 'resources2/?format=json&size=0&type=organizations&fq=(reldatasourcecompatibilityid exact driver or reldatasourcecompatibilityid exact driver-openaire2.0 or reldatasourcecompatibilityid exact openaire2.0 or reldatasourcecompatibilityid exact openaire3.0 or reldatasourcecompatibilityid exact openaire4.0 or reldatasourcecompatibilityid exact openaire-cris_1.1 or reldatasourcecompatibilityid exact openaire2.0_data or reldatasourcecompatibilityid exact hostedBy or relproject=*)', + searchServiceAPIUrl + 'projects/?fields=funder&sf=funder&format=json&size=0', + searchServiceAPIUrl + 'resources?query=(%20(oaftype%20exact%20result)%20and%20(resulttypeid%20exact%20dataset)%20and%20(relresulttype%3Dpublication)%20%20)&page=0&size=0&format=json', + searchServiceAPIUrl + 'resources?query=(%20(oaftype%20exact%20result)%20and%20(resulttypeid%20exact%20software)%20and%20(relresulttype%3Dpublication)%20%20)&page=0&size=0&format=json' + ]; + const dataPromises = requests.map((url) => axios.get( url)); + const dataResponses = await Promise.all(dataPromises); + // Determine if all additional requests were successful + const allRequestsSuccessful = dataResponses.every((response) => response.status === 200); // Combine the data const aggregatedData = { - publications: publicationsRES.data.total, - datasets: datasetsRES.data.total, - software: softwareRES.data.total, - other: otherRES.data.total, - results: resultRES.data.meta.total, + publications: dataResponses[0].data.total, + datasets: dataResponses[1].data.total, + software: dataResponses[2].data.total, + other: dataResponses[3].data.total, + results: dataResponses[4].data.meta.total, /*resultFunders:resultRES.data.refineResults.relfunder.length,*/ - datasources: datasourcesRES.data.total, - organizations: organizationsRES.data.meta.total?organizationsRES.data.meta.total:'--', - projects:projectsRES.data.meta.total, + datasources: dataResponses[5].data.total, + organizations: dataResponses[6].data.meta.total, + projects:dataResponses[7].data.meta.total, /*projectFunders:projectsRES.data.refineResults.funder.length,*/ - funders: parseNoOfFunders(resultRES, projectsRES), - datasetsInterlinked:datasetsInterlinkedRES.data.meta.total, - softwareInterlinked:softwareInterlinkedRES.data.meta.total, + funders: parseNoOfFunders(dataResponses[4].data, dataResponses[7].data), + datasetsInterlinked:dataResponses[8].data.meta.total, + softwareInterlinked:dataResponses[9].data.meta.total, + success:allRequestsSuccessful }; - // Send the aggregated data as the response - res.json(aggregatedData); + res.status(allRequestsSuccessful?200:207).json(aggregatedData); } catch (error) { console.error('Error fetching data:', error.message); res.status(500).send('Internal Server Error'); @@ -140,45 +146,124 @@ app.get('/explore/home', async function (req, res) { }); app.get('/explore/search', async function (req, res) { + let aggregatedData = {}; try { // Make requests to multiple APIs - const resultRES = await axios.get(searchServiceAPIUrl +'resources2/?format=json&size=0&type=results&fq=resultbestaccessright%20exact%20%22Open%20Access%22'); - const datasourcesRES = await axios.get(searchServiceAPIUrl + 'datasources/count?format=json'); - const organizationsRES = await axios.get(searchServiceAPIUrl + 'resources2/?format=json&size=0&type=organizations&fq=(reldatasourcecompatibilityid exact driver or reldatasourcecompatibilityid exact driver-openaire2.0 or reldatasourcecompatibilityid exact openaire2.0 or reldatasourcecompatibilityid exact openaire3.0 or reldatasourcecompatibilityid exact openaire4.0 or reldatasourcecompatibilityid exact openaire-cris_1.1 or reldatasourcecompatibilityid exact openaire2.0_data or reldatasourcecompatibilityid exact hostedBy or relproject=*)'); - const projectsRES = await axios.get(searchServiceAPIUrl + 'projects/count?format=json'); - + let requests= [ + searchServiceAPIUrl +'resources2/?format=json&size=0&type=results&fq=resultbestaccessright%20exact%20%22Open%20Access%22', + searchServiceAPIUrl + 'datasources/count?format=json', + searchServiceAPIUrl + 'resources2/?format=json&size=0&type=organizations&fq=(reldatasourcecompatibilityid exact driver or reldatasourcecompatibilityid exact driver-openaire2.0 or reldatasourcecompatibilityid exact openaire2.0 or reldatasourcecompatibilityid exact openaire3.0 or reldatasourcecompatibilityid exact openaire4.0 or reldatasourcecompatibilityid exact openaire-cris_1.1 or reldatasourcecompatibilityid exact openaire2.0_data or reldatasourcecompatibilityid exact hostedBy or relproject=*)', + searchServiceAPIUrl + 'projects/count?format=json' + ] + const dataPromises = requests.map((url) => axios.get( url)); + const dataResponses = await Promise.all(dataPromises); + // Determine if all additional requests were successful + const allRequestsSuccessful = dataResponses.every((response) => response.status === 200); // Combine the data - const aggregatedData = { - results: resultRES.data.meta.total, - datasources: datasourcesRES.data.total, - organizations: organizationsRES.data.meta.total?organizationsRES.data.meta.total:'--', - projects:projectsRES.data.total + aggregatedData = { + results: dataResponses[0].data.meta.total, + datasources: dataResponses[1].data.total, + organizations: dataResponses[2].data.meta.total, + projects:dataResponses[3].data.total, + success:allRequestsSuccessful }; // Send the aggregated data as the response - res.json(aggregatedData); + res.status(allRequestsSuccessful?200:207).json(aggregatedData); } catch (error) { - console.error('Error fetching data:', error.message); + console.log(aggregatedData) + console.error('Error fetching data:', error); + res.status(500).send('Internal Server Error'); + } +}); + +app.get('/explore/funders', async function (req, res) { + let aggregatedData = {}; + try { + // Make requests to multiple APIs + let requests= [ + searchServiceAPIUrl + 'resources2/?format=json&type=results&fq=relfunder=*&refine=true&fields=relfunder&sf=relfunder&page=0&size=0', + searchServiceAPIUrl + 'resources2/?format=json&type=results&fq=relfunder=*&refine=true&fields=relfunder&sf=relfunder&page=0&size=0&fq=resultbestaccessright%20exact%20%22Open%20Access%22', + searchServiceAPIUrl + 'resources2/?format=json&type=projects&fq=funder=*&fq=projecttitle<>"unidentified"&refine=true&fields=funder&sf=funder&page=0&size=0', + monitorServiceAPIUrl + 'stakeholder?type=funder', + + ] + const dataPromises = requests.map((url) => axios.get( url)); + const dataResponses = await Promise.all(dataPromises); + // Determine if all additional requests were successful + const allRequestsSuccessful = dataResponses.every((response) => response.status === 200); + let fundersMap = new Map(); + + let resultsFunders = dataResponses[0].data.refineResults.relfunder; + resultsFunders.forEach(queriedFunder => { + if (!fundersMap.has(queriedFunder.id)) { + fundersMap.set(queriedFunder.id,{name: queriedFunder.name, id: queriedFunder.id, results:queriedFunder.count, openResults: null, projects:null, stakeholder:null}); + } + }); + let openResultsFunders = dataResponses[1].data.refineResults.relfunder; + openResultsFunders.forEach(queriedFunder => { + if (!fundersMap.has(queriedFunder.id)) { + fundersMap.set(queriedFunder.id,{name: queriedFunder.name, id: queriedFunder.id, results:null, openResults: queriedFunder.count, projects:null, stakeholder:null}); + }else{ + fundersMap.get(queriedFunder.id).openResults = queriedFunder.count; + } + }); + let projectFunders = dataResponses[2].data.refineResults.funder; + projectFunders.forEach(queriedFunder => { + if (!fundersMap.has(queriedFunder.id) ) { + fundersMap.set(queriedFunder.id,{name: queriedFunder.name, id: queriedFunder.id, results:null, openResults: null, projects:queriedFunder.count, stakeholder:null}); + }else{ + fundersMap.get(queriedFunder.id).projects = queriedFunder.count; + } + }); + let stakeholders = dataResponses[3].data; + stakeholders.forEach(stakeholder => { + let id = stakeholder.index_id + "||" + stakeholder.index_name + "||" + stakeholder.index_shortName; + // console.log(id); + if (!fundersMap.has(id)) { + fundersMap.set(id,{name: stakeholder.index_name, id: id, results:null, openResults: null, projects:null, stakeholder:stakeholder}); + }else{ + fundersMap.get(id).stakeholder = stakeholder; + } + }); + + // Combine the data + + // Send the aggregated data as the response + // console.log(fundersMap) + aggregatedData = { + count: fundersMap.size, + results: dataResponses[0].data.meta.total, + projects: dataResponses[2].data.meta.total, + funders: Array.from(fundersMap.values()), + success:allRequestsSuccessful + + }; + res.status(allRequestsSuccessful?200:207).json(aggregatedData); + } catch (error) { + // console.log(aggregatedData) + console.error('Error fetching data:', error); res.status(500).send('Internal Server Error'); } }); app.get('/grouped-requests', async function (req, res) { res.json([ "/explore/search", - "/explore/home" + "/explore/home", + "/explore/funders" ]); }); function parseNoOfFunders(resultRES, projectsRES){ // combines the refines qeries on funders field, the funders with results and the funders that have at least one project let mergedFundersSet = new Set(); - let queriedFunders = resultRES.data.refineResults.relfunder; + let queriedFunders = resultRES.refineResults.relfunder; queriedFunders.forEach(queriedFunder => { if (!mergedFundersSet.has(queriedFunder.id)) { mergedFundersSet.add(queriedFunder.id); } }); - queriedFunders = projectsRES.data.refineResults.funder; + queriedFunders = projectsRES.refineResults.funder; queriedFunders.forEach(queriedFunder => { if(+queriedFunder.count > 1) { if (!mergedFundersSet.has(queriedFunder.id)) { From 7761eeeedcf5c1ed2869d5d45de6fb5947c14dfd Mon Sep 17 00:00:00 2001 From: argirok Date: Wed, 7 Feb 2024 11:05:55 +0200 Subject: [PATCH 7/9] [ develop | DONE | CHANGED] Utils services: update funders response, remove open access filter from results query in /explore/search --- services/utils-service/uploadService.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/services/utils-service/uploadService.js b/services/utils-service/uploadService.js index b4f3df69..c26704a5 100644 --- a/services/utils-service/uploadService.js +++ b/services/utils-service/uploadService.js @@ -150,7 +150,7 @@ app.get('/explore/search', async function (req, res) { try { // Make requests to multiple APIs let requests= [ - searchServiceAPIUrl +'resources2/?format=json&size=0&type=results&fq=resultbestaccessright%20exact%20%22Open%20Access%22', + searchServiceAPIUrl +'resources2/?format=json&size=0&type=results', searchServiceAPIUrl + 'datasources/count?format=json', searchServiceAPIUrl + 'resources2/?format=json&size=0&type=organizations&fq=(reldatasourcecompatibilityid exact driver or reldatasourcecompatibilityid exact driver-openaire2.0 or reldatasourcecompatibilityid exact openaire2.0 or reldatasourcecompatibilityid exact openaire3.0 or reldatasourcecompatibilityid exact openaire4.0 or reldatasourcecompatibilityid exact openaire-cris_1.1 or reldatasourcecompatibilityid exact openaire2.0_data or reldatasourcecompatibilityid exact hostedBy or relproject=*)', searchServiceAPIUrl + 'projects/count?format=json' @@ -198,13 +198,13 @@ app.get('/explore/funders', async function (req, res) { let resultsFunders = dataResponses[0].data.refineResults.relfunder; resultsFunders.forEach(queriedFunder => { if (!fundersMap.has(queriedFunder.id)) { - fundersMap.set(queriedFunder.id,{name: queriedFunder.name, id: queriedFunder.id, results:queriedFunder.count, openResults: null, projects:null, stakeholder:null}); + fundersMap.set(queriedFunder.id,{name: queriedFunder.name.split("||")[0], id: queriedFunder.id, results:queriedFunder.count, openResults: null, projects:null, stakeholder:null}); } }); let openResultsFunders = dataResponses[1].data.refineResults.relfunder; openResultsFunders.forEach(queriedFunder => { if (!fundersMap.has(queriedFunder.id)) { - fundersMap.set(queriedFunder.id,{name: queriedFunder.name, id: queriedFunder.id, results:null, openResults: queriedFunder.count, projects:null, stakeholder:null}); + fundersMap.set(queriedFunder.id,{name: queriedFunder.name.split("||")[0], id: queriedFunder.id, results:null, openResults: queriedFunder.count, projects:null, stakeholder:null}); }else{ fundersMap.get(queriedFunder.id).openResults = queriedFunder.count; } @@ -212,7 +212,7 @@ app.get('/explore/funders', async function (req, res) { let projectFunders = dataResponses[2].data.refineResults.funder; projectFunders.forEach(queriedFunder => { if (!fundersMap.has(queriedFunder.id) ) { - fundersMap.set(queriedFunder.id,{name: queriedFunder.name, id: queriedFunder.id, results:null, openResults: null, projects:queriedFunder.count, stakeholder:null}); + fundersMap.set(queriedFunder.id,{name: queriedFunder.name.split("||")[0], id: queriedFunder.id, results:null, openResults: null, projects:queriedFunder.count, stakeholder:null}); }else{ fundersMap.get(queriedFunder.id).projects = queriedFunder.count; } @@ -221,10 +221,10 @@ app.get('/explore/funders', async function (req, res) { stakeholders.forEach(stakeholder => { let id = stakeholder.index_id + "||" + stakeholder.index_name + "||" + stakeholder.index_shortName; // console.log(id); - if (!fundersMap.has(id)) { - fundersMap.set(id,{name: stakeholder.index_name, id: id, results:null, openResults: null, projects:null, stakeholder:stakeholder}); - }else{ - fundersMap.get(id).stakeholder = stakeholder; + if (fundersMap.has(id)) { + let ministakeholder = {id:id, name:stakeholder.name, alias: stakeholder.alias, visibility: stakeholder.visibility, + logoUrl:stakeholder.logoUrl, isUpload: stakeholder.isUpload} + fundersMap.get(id).stakeholder = ministakeholder; } }); From 4131de601cb29a97873120ae3c99c6c203a8724b Mon Sep 17 00:00:00 2001 From: "konstantina.galouni" Date: Wed, 7 Feb 2024 12:00:16 +0200 Subject: [PATCH 8/9] [develop | DONE | ADDED]: cache.js: Added in "longCachingRequests", the endpoints "/utils-service/explore/search" and "/utils-service/explore/funders" to be used as a grouped responses in explore search and funders pages. --- services/cache/mecache/cache.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/cache/mecache/cache.js b/services/cache/mecache/cache.js index 11710b4e..435763a8 100644 --- a/services/cache/mecache/cache.js +++ b/services/cache/mecache/cache.js @@ -14,7 +14,8 @@ const cacheMaxSize = 500; const longCachingRequests = ["/communityFull", "/full", "/pagehelpcontent", "/provision/mvc/vocabularies/", "/pages?page_route=", "/allmetrics", "/countryusagestats/", "/openaire/info", - "/api/communities/", "/openaire/contexts/", "/utils-service/explore/home"]; + "/api/communities/", "/openaire/contexts/", + "/utils-service/explore/home", "/utils-service/explore/search", "/utils-service/explore/funders"]; let cors = require('cors'); app.use(cors()); From a3c79137aa30a57096a9f04c715d99bb4649f88b Mon Sep 17 00:00:00 2001 From: "konstantina.galouni" Date: Wed, 7 Feb 2024 21:28:45 +0200 Subject: [PATCH 9/9] [develop | DONE | CHANGED]: Remove preselected "Open Access" filter from search. 1. home.component.ts: Set field "resultsQuickFilter" to null. 2. searchAll.component.ts: Set field "quickFilter" to null | In method "entityChanged()" do not set parameter "resultbestaccessright" | [BUG FIX] Clear subscriptions from fetchOrps. 3. searchResearchResults.component.ts: Set field "quickFilter" to null. 4. navigationBar.component.ts: Set field "resultsQuickFilter" to null. 5. app.component.ts: Remove "resultbestaccessright" parameter from menu items of Search research products. --- explore/src/app/app.component.ts | 12 ++++++------ explore/src/app/home/home.component.ts | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/explore/src/app/app.component.ts b/explore/src/app/app.component.ts index 7de1b674..a17526e3 100644 --- a/explore/src/app/app.component.ts +++ b/explore/src/app/app.component.ts @@ -167,15 +167,15 @@ export class AppComponent { this.userMenuItems.push(new MenuItem("", "My profile", "", "", false, [], [], {})); this.userMenuItems.push(new MenuItem("", "My ORCID links", "", "/my-orcid-links", false, [], [""], {})); this.userMenuItems.push(new MenuItem("", "My links", "", "/myclaims", false, [], ["/myclaims"], {})); - let researchOutcomesMenu = new MenuItem("", OpenaireEntities.RESULTS, "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {resultbestaccessright: '"' + encodeURIComponent("Open Access") + '"'}); + let researchOutcomesMenu = new MenuItem("", OpenaireEntities.RESULTS, "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {}); researchOutcomesMenu.items = [ - new MenuItem("", OpenaireEntities.PUBLICATIONS, "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {type: '"' + encodeURIComponent("publications") + '"', resultbestaccessright: '"' + encodeURIComponent("Open Access") + '"'}), - new MenuItem("", OpenaireEntities.DATASETS, "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {type: '"' + encodeURIComponent("datasets") + '"', resultbestaccessright: '"' + encodeURIComponent("Open Access") + '"'}), - new MenuItem("", OpenaireEntities.SOFTWARE, "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {type: '"' + encodeURIComponent("software") + '"', resultbestaccessright: '"' + encodeURIComponent("Open Access") + '"'}), - new MenuItem("", OpenaireEntities.OTHER, "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {type: '"' + encodeURIComponent("other") + '"', resultbestaccessright: '"' + encodeURIComponent("Open Access") + '"'})]; + new MenuItem("", OpenaireEntities.PUBLICATIONS, "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {type: '"' + encodeURIComponent("publications") + '"'}), + new MenuItem("", OpenaireEntities.DATASETS, "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {type: '"' + encodeURIComponent("datasets") + '"'}), + new MenuItem("", OpenaireEntities.SOFTWARE, "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {type: '"' + encodeURIComponent("software") + '"'}), + new MenuItem("", OpenaireEntities.OTHER, "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {type: '"' + encodeURIComponent("other") + '"'})]; //TODO add check for research results route this.menuItems = [ - new MenuItem("search", "Search", "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {resultbestaccessright: '"' + encodeURIComponent("Open Access") + '"'}, + new MenuItem("search", "Search", "", "/search/find/research-outcomes", false, [], ["/search/find/research-outcomes"], {}, null, null, null, null, "_blank", "internal", false, [ researchOutcomesMenu, diff --git a/explore/src/app/home/home.component.ts b/explore/src/app/home/home.component.ts index 5bbbbfb7..f5cab70c 100644 --- a/explore/src/app/home/home.component.ts +++ b/explore/src/app/home/home.component.ts @@ -125,12 +125,12 @@ export class HomeComponent implements OnInit, OnDestroy, AfterViewInit { @ViewChild('contact') contact: ElementRef; subscriptions: any[] = []; @ViewChildren('scrolling_element') elements: QueryList; - resultsQuickFilter: { filter: Filter, selected: boolean, filterId: string, value: string } = { - filter: null, - selected: true, - filterId: "resultbestaccessright", - value: "Open Access" - }; + resultsQuickFilter: { filter: Filter, selected: boolean, filterId: string, value: string } = null;//{ + // filter: null, + // selected: true, + // filterId: "resultbestaccessright", + // value: "Open Access" + // }; selectedEntity = "all"; selectedEntitySimpleUrl; selectedEntityAdvancedUrl;