MaDgIK
/
explore-services
13
0
Fork 0
Code Issues Pull Requests Projects Releases 17 Wiki Activity

[ develop | DONE | ADDED] Utils services: add /explore/funders method that groups requests for funder page

Browse Source
grouped-queries
argirok 3 months ago
parent f64b410dd5
commit 46ad6aef36
4 changed files with 127 additions and 39 deletions
Show Stats Download Patch File Download Diff File

1
services/utils-service/beta-properties.file
Unescape Escape View File

@ -1,5 +1,6 @@
userInfoUrl = https://beta.services.openaire.eu/login-service/userInfo
searchServiceAPIUrl = https://beta.services.openaire.eu/search/v2/api/
monitorAPIUrl = https://beta.services.openaire.eu/uoa-monitor-service/
ssl = true
localPath = false
# photo size in KB

1
services/utils-service/production-properties.file
Unescape Escape View File

@ -1,5 +1,6 @@
userInfoUrl = https://services.openaire.eu/login-service/userInfo
searchServiceAPIUrl = https://services.openaire.eu/search/v2/api/
monitorAPIUrl = https://services.openaire.eu/uoa-monitor-service/
ssl = true
localPath = false
# photo size in KB

1
services/utils-service/properties.file
Unescape Escape View File

@ -1,5 +1,6 @@
userInfoUrl = http://mpagasas.di.uoa.gr:19080/login-service/userInfo
searchServiceAPIUrl = https://beta.services.openaire.eu/search/v2/api/
monitorAPIUrl = http://duffy.di.uoa.gr:19380/uoa-monitor-service/
ssl = false
localPath = true
# photo size in KB

163
services/utils-service/uploadService.js
Unescape Escape View File

@ -14,6 +14,7 @@ if (properties.get('ssl')) {
http = require("http");
}
var searchServiceAPIUrl = properties.get('searchServiceAPIUrl');
var monitorServiceAPIUrl = properties.get('monitorAPIUrl');
var auth = properties.get('userInfoUrl');
/** @deprecated*/
var authDeprecated = auth.includes("accessToken");
@ -102,37 +103,42 @@ app.delete(['/delete/:filename', '/delete/stakeholder/:filename', '/delete/:type
app.get('/explore/home', async function (req, res) {
try {
// Make requests to multiple APIs
const publicationsRES = await axios.get(searchServiceAPIUrl + 'publications/count?format=json');
const datasetsRES = await axios.get(searchServiceAPIUrl + 'datasets/count?format=json');
const softwareRES = await axios.get(searchServiceAPIUrl + 'software/count?format=json');
const otherRES = await axios.get(searchServiceAPIUrl + 'other/count?format=json');
const resultRES = await axios.get(searchServiceAPIUrl +'results/?fields=relfunder&sf=relfunder&format=json&size=0');
const datasourcesRES = await axios.get(searchServiceAPIUrl + 'datasources/count?format=json');
const organizationsRES = await axios.get(searchServiceAPIUrl + 'resources2/?format=json&size=0&type=organizations&fq=(reldatasourcecompatibilityid exact driver or reldatasourcecompatibilityid exact driver-openaire2.0 or reldatasourcecompatibilityid exact openaire2.0 or reldatasourcecompatibilityid exact openaire3.0 or reldatasourcecompatibilityid exact openaire4.0 or reldatasourcecompatibilityid exact openaire-cris_1.1 or reldatasourcecompatibilityid exact openaire2.0_data or reldatasourcecompatibilityid exact hostedBy or relproject=*)');
const projectsRES = await axios.get(searchServiceAPIUrl + 'projects/?fields=funder&sf=funder&format=json&size=0');
const datasetsInterlinkedRES = await axios.get(searchServiceAPIUrl + 'resources?query=(%20(oaftype%20exact%20result)%20and%20(resulttypeid%20exact%20dataset)%20and%20(relresulttype%3Dpublication)%20%20)&page=0&size=0&format=json');
const softwareInterlinkedRES = await axios.get(searchServiceAPIUrl + 'resources?query=(%20(oaftype%20exact%20result)%20and%20(resulttypeid%20exact%20software)%20and%20(relresulttype%3Dpublication)%20%20)&page=0&size=0&format=json');
let requests= [
searchServiceAPIUrl + 'publications/count?format=json',
searchServiceAPIUrl + 'datasets/count?format=json',
searchServiceAPIUrl + 'software/count?format=json',
searchServiceAPIUrl + 'other/count?format=json',
searchServiceAPIUrl +'results/?fields=relfunder&sf=relfunder&format=json&size=0',
searchServiceAPIUrl + 'datasources/count?format=json',
searchServiceAPIUrl + 'resources2/?format=json&size=0&type=organizations&fq=(reldatasourcecompatibilityid exact driver or reldatasourcecompatibilityid exact driver-openaire2.0 or reldatasourcecompatibilityid exact openaire2.0 or reldatasourcecompatibilityid exact openaire3.0 or reldatasourcecompatibilityid exact openaire4.0 or reldatasourcecompatibilityid exact openaire-cris_1.1 or reldatasourcecompatibilityid exact openaire2.0_data or reldatasourcecompatibilityid exact hostedBy or relproject=*)',
searchServiceAPIUrl + 'projects/?fields=funder&sf=funder&format=json&size=0',
searchServiceAPIUrl + 'resources?query=(%20(oaftype%20exact%20result)%20and%20(resulttypeid%20exact%20dataset)%20and%20(relresulttype%3Dpublication)%20%20)&page=0&size=0&format=json',
searchServiceAPIUrl + 'resources?query=(%20(oaftype%20exact%20result)%20and%20(resulttypeid%20exact%20software)%20and%20(relresulttype%3Dpublication)%20%20)&page=0&size=0&format=json'
];
const dataPromises = requests.map((url) => axios.get( url));
const dataResponses = await Promise.all(dataPromises);
// Determine if all additional requests were successful
const allRequestsSuccessful = dataResponses.every((response) => response.status === 200);
// Combine the data
const aggregatedData = {
publications: publicationsRES.data.total,
datasets: datasetsRES.data.total,
software: softwareRES.data.total,
other: otherRES.data.total,
results: resultRES.data.meta.total,
publications: dataResponses[0].data.total,
datasets: dataResponses[1].data.total,
software: dataResponses[2].data.total,
other: dataResponses[3].data.total,
results: dataResponses[4].data.meta.total,
/*resultFunders:resultRES.data.refineResults.relfunder.length,*/
datasources: datasourcesRES.data.total,
organizations: organizationsRES.data.meta.total?organizationsRES.data.meta.total:'--',
projects:projectsRES.data.meta.total,
datasources: dataResponses[5].data.total,
organizations: dataResponses[6].data.meta.total,
projects:dataResponses[7].data.meta.total,
/*projectFunders:projectsRES.data.refineResults.funder.length,*/
funders: parseNoOfFunders(resultRES, projectsRES),
datasetsInterlinked:datasetsInterlinkedRES.data.meta.total,
softwareInterlinked:softwareInterlinkedRES.data.meta.total,
funders: parseNoOfFunders(dataResponses[4].data, dataResponses[7].data),
datasetsInterlinked:dataResponses[8].data.meta.total,
softwareInterlinked:dataResponses[9].data.meta.total,
success:allRequestsSuccessful
};
// Send the aggregated data as the response
res.json(aggregatedData);
res.status(allRequestsSuccessful?200:207).json(aggregatedData);
} catch (error) {
console.error('Error fetching data:', error.message);
res.status(500).send('Internal Server Error');
@ -140,45 +146,124 @@ app.get('/explore/home', async function (req, res) {
});
app.get('/explore/search', async function (req, res) {
let aggregatedData = {};
try {
// Make requests to multiple APIs
const resultRES = await axios.get(searchServiceAPIUrl +'resources2/?format=json&size=0&type=results&fq=resultbestaccessright%20exact%20%22Open%20Access%22');
const datasourcesRES = await axios.get(searchServiceAPIUrl + 'datasources/count?format=json');
const organizationsRES = await axios.get(searchServiceAPIUrl + 'resources2/?format=json&size=0&type=organizations&fq=(reldatasourcecompatibilityid exact driver or reldatasourcecompatibilityid exact driver-openaire2.0 or reldatasourcecompatibilityid exact openaire2.0 or reldatasourcecompatibilityid exact openaire3.0 or reldatasourcecompatibilityid exact openaire4.0 or reldatasourcecompatibilityid exact openaire-cris_1.1 or reldatasourcecompatibilityid exact openaire2.0_data or reldatasourcecompatibilityid exact hostedBy or relproject=*)');
const projectsRES = await axios.get(searchServiceAPIUrl + 'projects/count?format=json');
let requests= [
searchServiceAPIUrl +'resources2/?format=json&size=0&type=results&fq=resultbestaccessright%20exact%20%22Open%20Access%22',
searchServiceAPIUrl + 'datasources/count?format=json',
searchServiceAPIUrl + 'resources2/?format=json&size=0&type=organizations&fq=(reldatasourcecompatibilityid exact driver or reldatasourcecompatibilityid exact driver-openaire2.0 or reldatasourcecompatibilityid exact openaire2.0 or reldatasourcecompatibilityid exact openaire3.0 or reldatasourcecompatibilityid exact openaire4.0 or reldatasourcecompatibilityid exact openaire-cris_1.1 or reldatasourcecompatibilityid exact openaire2.0_data or reldatasourcecompatibilityid exact hostedBy or relproject=*)',
searchServiceAPIUrl + 'projects/count?format=json'
]
const dataPromises = requests.map((url) => axios.get( url));
const dataResponses = await Promise.all(dataPromises);
// Determine if all additional requests were successful
const allRequestsSuccessful = dataResponses.every((response) => response.status === 200);
// Combine the data
const aggregatedData = {
results: resultRES.data.meta.total,
datasources: datasourcesRES.data.total,
organizations: organizationsRES.data.meta.total?organizationsRES.data.meta.total:'--',
projects:projectsRES.data.total
aggregatedData = {
results: dataResponses[0].data.meta.total,
datasources: dataResponses[1].data.total,
organizations: dataResponses[2].data.meta.total,
projects:dataResponses[3].data.total,
success:allRequestsSuccessful
};
// Send the aggregated data as the response
res.json(aggregatedData);
res.status(allRequestsSuccessful?200:207).json(aggregatedData);
} catch (error) {
console.error('Error fetching data:', error.message);
console.log(aggregatedData)
console.error('Error fetching data:', error);
res.status(500).send('Internal Server Error');
}
});
app.get('/explore/funders', async function (req, res) {
let aggregatedData = {};
try {
// Make requests to multiple APIs
let requests= [
searchServiceAPIUrl + 'resources2/?format=json&type=results&fq=relfunder=*&refine=true&fields=relfunder&sf=relfunder&page=0&size=0',
searchServiceAPIUrl + 'resources2/?format=json&type=results&fq=relfunder=*&refine=true&fields=relfunder&sf=relfunder&page=0&size=0&fq=resultbestaccessright%20exact%20%22Open%20Access%22',
searchServiceAPIUrl + 'resources2/?format=json&type=projects&fq=funder=*&fq=projecttitle<>"unidentified"&refine=true&fields=funder&sf=funder&page=0&size=0',
monitorServiceAPIUrl + 'stakeholder?type=funder',
]
const dataPromises = requests.map((url) => axios.get( url));
const dataResponses = await Promise.all(dataPromises);
// Determine if all additional requests were successful
const allRequestsSuccessful = dataResponses.every((response) => response.status === 200);
let fundersMap = new Map();
let resultsFunders = dataResponses[0].data.refineResults.relfunder;
resultsFunders.forEach(queriedFunder => {
if (!fundersMap.has(queriedFunder.id)) {
fundersMap.set(queriedFunder.id,{name: queriedFunder.name, id: queriedFunder.id, results:queriedFunder.count, openResults: null, projects:null, stakeholder:null});
}
});
let openResultsFunders = dataResponses[1].data.refineResults.relfunder;
openResultsFunders.forEach(queriedFunder => {
if (!fundersMap.has(queriedFunder.id)) {
fundersMap.set(queriedFunder.id,{name: queriedFunder.name, id: queriedFunder.id, results:null, openResults: queriedFunder.count, projects:null, stakeholder:null});
}else{
fundersMap.get(queriedFunder.id).openResults = queriedFunder.count;
}
});
let projectFunders = dataResponses[2].data.refineResults.funder;
projectFunders.forEach(queriedFunder => {
if (!fundersMap.has(queriedFunder.id) ) {
fundersMap.set(queriedFunder.id,{name: queriedFunder.name, id: queriedFunder.id, results:null, openResults: null, projects:queriedFunder.count, stakeholder:null});
}else{
fundersMap.get(queriedFunder.id).projects = queriedFunder.count;
}
});
let stakeholders = dataResponses[3].data;
stakeholders.forEach(stakeholder => {
let id = stakeholder.index_id + "||" + stakeholder.index_name + "||" + stakeholder.index_shortName;
// console.log(id);
if (!fundersMap.has(id)) {
fundersMap.set(id,{name: stakeholder.index_name, id: id, results:null, openResults: null, projects:null, stakeholder:stakeholder});
}else{
fundersMap.get(id).stakeholder = stakeholder;
}
});
// Combine the data
// Send the aggregated data as the response
// console.log(fundersMap)
aggregatedData = {
count: fundersMap.size,
results: dataResponses[0].data.meta.total,
projects: dataResponses[2].data.meta.total,
funders: Array.from(fundersMap.values()),
success:allRequestsSuccessful
};
res.status(allRequestsSuccessful?200:207).json(aggregatedData);
} catch (error) {
// console.log(aggregatedData)
console.error('Error fetching data:', error);
res.status(500).send('Internal Server Error');
}
});
app.get('/grouped-requests', async function (req, res) {
res.json([
"/explore/search",
"/explore/home"
"/explore/home",
"/explore/funders"
]);
});
function parseNoOfFunders(resultRES, projectsRES){
// combines the refines qeries on funders field, the funders with results and the funders that have at least one project
let mergedFundersSet = new Set();
let queriedFunders = resultRES.data.refineResults.relfunder;
let queriedFunders = resultRES.refineResults.relfunder;
queriedFunders.forEach(queriedFunder => {
if (!mergedFundersSet.has(queriedFunder.id)) {
mergedFundersSet.add(queriedFunder.id);
}
});
queriedFunders = projectsRES.data.refineResults.funder;
queriedFunders = projectsRES.refineResults.funder;
queriedFunders.forEach(queriedFunder => {
if(+queriedFunder.count > 1) {
if (!mergedFundersSet.has(queriedFunder.id)) {

Write Preview
Loading…
Cancel
Save