From bfa590ea5e3701d7d43a33df491651203fd648d6 Mon Sep 17 00:00:00 2001 From: "k.triantafyllou" Date: Wed, 10 Apr 2024 15:59:40 +0300 Subject: [PATCH 1/4] Fix StakeholderPublications to extend the new Stakeholder class --- .../irishmonitorservice/entities/StakeholderPublications.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/eu/dnetlib/irishmonitorservice/entities/StakeholderPublications.java b/src/main/java/eu/dnetlib/irishmonitorservice/entities/StakeholderPublications.java index 6560f5f..4d32551 100644 --- a/src/main/java/eu/dnetlib/irishmonitorservice/entities/StakeholderPublications.java +++ b/src/main/java/eu/dnetlib/irishmonitorservice/entities/StakeholderPublications.java @@ -2,7 +2,7 @@ package eu.dnetlib.irishmonitorservice.entities; import eu.dnetlib.uoamonitorservice.entities.Stakeholder; -public class StakeholderPublications extends Stakeholder { +public class StakeholderPublications extends Stakeholder { private Number publications; public StakeholderPublications(Stakeholder stakeholder, Number publications) { From d25c54ce5a8c4290ab5c25545d5c096606af3c97 Mon Sep 17 00:00:00 2001 From: "k.triantafyllou" Date: Wed, 10 Apr 2024 16:05:25 +0300 Subject: [PATCH 2/4] Fix global vars path variables --- .../dnetlib/irishmonitorservice/configuration/GlobalVars.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/eu/dnetlib/irishmonitorservice/configuration/GlobalVars.java b/src/main/java/eu/dnetlib/irishmonitorservice/configuration/GlobalVars.java index b3fadbf..63f8e72 100644 --- a/src/main/java/eu/dnetlib/irishmonitorservice/configuration/GlobalVars.java +++ b/src/main/java/eu/dnetlib/irishmonitorservice/configuration/GlobalVars.java @@ -4,7 +4,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties; import java.util.Date; -@ConfigurationProperties("irishservice.globalVars") +@ConfigurationProperties("irishmonitorservice.globalVars") public class GlobalVars { public static Date date = new Date(); private Date buildDate; From 3176a0448d139f3b9698a0b6e2215b4eac030fad Mon Sep 17 00:00:00 2001 From: "k.triantafyllou" Date: Fri, 12 Apr 2024 13:10:55 +0300 Subject: [PATCH 3/4] [master]: cache response from stats tool after clearing the cache. --- .../StakeholderPublicationsController.java | 29 +++++-------- .../{utils => entities}/Data.java | 2 +- .../services/CacheService.java | 41 +++++++++++++++++++ .../StatsToolService.java} | 17 ++------ 4 files changed, 55 insertions(+), 34 deletions(-) rename src/main/java/eu/dnetlib/irishmonitorservice/{utils => entities}/Data.java (80%) create mode 100644 src/main/java/eu/dnetlib/irishmonitorservice/services/CacheService.java rename src/main/java/eu/dnetlib/irishmonitorservice/{utils/StatsToolUtils.java => services/StatsToolService.java} (76%) diff --git a/src/main/java/eu/dnetlib/irishmonitorservice/controllers/StakeholderPublicationsController.java b/src/main/java/eu/dnetlib/irishmonitorservice/controllers/StakeholderPublicationsController.java index 5df66d4..c930e0c 100644 --- a/src/main/java/eu/dnetlib/irishmonitorservice/controllers/StakeholderPublicationsController.java +++ b/src/main/java/eu/dnetlib/irishmonitorservice/controllers/StakeholderPublicationsController.java @@ -1,46 +1,37 @@ package eu.dnetlib.irishmonitorservice.controllers; -import eu.dnetlib.irishmonitorservice.configuration.properties.StatsToolProperties; import eu.dnetlib.irishmonitorservice.entities.StakeholderPublications; -import eu.dnetlib.irishmonitorservice.utils.StatsToolUtils; +import eu.dnetlib.irishmonitorservice.services.CacheService; +import eu.dnetlib.irishmonitorservice.services.StatsToolService; import eu.dnetlib.uoamonitorservice.entities.Stakeholder; import eu.dnetlib.uoamonitorservice.service.StakeholderService; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; @RestController @CrossOrigin(origins = "*") @RequestMapping("publications") public class StakeholderPublicationsController { + private final StakeholderService service; + private final CacheService cacheService; @Autowired - private StakeholderService service; - - @Autowired - private StatsToolUtils utils; - - public List> getResponse(String type) throws UnsupportedEncodingException { - if(type.equals("funder")) { - return this.utils.getFunders(); - } else if(type.equals("organization")) { - return this.utils.getOrganizations(); - } else if(type.equals("datasource")) { - return this.utils.getDataSources(); - } else { - return null; - } + public StakeholderPublicationsController(StakeholderService service, CacheService cacheService) { + this.service = service; + this.cacheService = cacheService; } @RequestMapping(value = "/{type}", method = RequestMethod.GET) public List getStakeholders(@PathVariable String type) throws UnsupportedEncodingException, InterruptedException { List stakeholders = this.service.getStakeholdersByTypeAndRole(type, null, false); List stakeholderPublications = new ArrayList<>(); - List> data = this.getResponse(type); + List> data = this.cacheService.getResponse(type); List remain = new ArrayList<>(stakeholders); if (data != null) { data.forEach(entity -> { diff --git a/src/main/java/eu/dnetlib/irishmonitorservice/utils/Data.java b/src/main/java/eu/dnetlib/irishmonitorservice/entities/Data.java similarity index 80% rename from src/main/java/eu/dnetlib/irishmonitorservice/utils/Data.java rename to src/main/java/eu/dnetlib/irishmonitorservice/entities/Data.java index 40a58f2..773bf95 100644 --- a/src/main/java/eu/dnetlib/irishmonitorservice/utils/Data.java +++ b/src/main/java/eu/dnetlib/irishmonitorservice/entities/Data.java @@ -1,4 +1,4 @@ -package eu.dnetlib.irishmonitorservice.utils; +package eu.dnetlib.irishmonitorservice.entities; import java.util.List; import java.util.Map; diff --git a/src/main/java/eu/dnetlib/irishmonitorservice/services/CacheService.java b/src/main/java/eu/dnetlib/irishmonitorservice/services/CacheService.java new file mode 100644 index 0000000..0ceb03f --- /dev/null +++ b/src/main/java/eu/dnetlib/irishmonitorservice/services/CacheService.java @@ -0,0 +1,41 @@ +package eu.dnetlib.irishmonitorservice.services; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import java.io.UnsupportedEncodingException; +import java.time.LocalDateTime; +import java.util.List; + +@Service +public class CacheService { + private final Logger logger = LogManager.getLogger(CacheService.class); + + @Autowired + private StatsToolService statsToolService; + + public List> getResponse(String type) throws UnsupportedEncodingException { + if (type.equals("funder")) { + return this.statsToolService.getFunders(); + } else if (type.equals("organization")) { + return this.statsToolService.getOrganizations(); + } else if (type.equals("datasource")) { + return this.statsToolService.getDataSources(); + } else { + return null; + } + } + + @CacheEvict(allEntries = true, value = {"funders", "organizations", "datasources"}, beforeInvocation = true) + @Scheduled(cron = "0 0 0 * * *") // Reset cache every day at 00:00 + public void clearCacheAtMidnight() throws UnsupportedEncodingException { + logger.info("Cache cleared at " + LocalDateTime.now()); + this.statsToolService.getFunders(); + this.statsToolService.getOrganizations(); + this.statsToolService.getDataSources(); + } +} diff --git a/src/main/java/eu/dnetlib/irishmonitorservice/utils/StatsToolUtils.java b/src/main/java/eu/dnetlib/irishmonitorservice/services/StatsToolService.java similarity index 76% rename from src/main/java/eu/dnetlib/irishmonitorservice/utils/StatsToolUtils.java rename to src/main/java/eu/dnetlib/irishmonitorservice/services/StatsToolService.java index 505bb85..a4a4a06 100644 --- a/src/main/java/eu/dnetlib/irishmonitorservice/utils/StatsToolUtils.java +++ b/src/main/java/eu/dnetlib/irishmonitorservice/services/StatsToolService.java @@ -1,14 +1,11 @@ -package eu.dnetlib.irishmonitorservice.utils; +package eu.dnetlib.irishmonitorservice.services; import eu.dnetlib.irishmonitorservice.configuration.properties.StatsToolProperties; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import eu.dnetlib.irishmonitorservice.entities.Data; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cache.annotation.CacheEvict; import org.springframework.cache.annotation.Cacheable; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; @@ -17,12 +14,10 @@ import org.springframework.web.util.UriComponentsBuilder; import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URLDecoder; -import java.time.LocalDateTime; import java.util.List; @Service -public class StatsToolUtils { - private final Logger logger = LogManager.getLogger(StatsToolUtils.class); +public class StatsToolService { @Autowired private StatsToolProperties properties; @@ -64,10 +59,4 @@ public class StatsToolUtils { } return null; } - - @CacheEvict(allEntries = true, value = {"funders", "organizations", "datasources"}) - @Scheduled(cron = "0 0 0 * * *") // Reset cache every day at 00:00 - public void clearCacheAtMidnight() { - logger.info("Cache cleared at " + LocalDateTime.now()); - } } From 371ad263450e2ec2dc93d0bd3c8ecf107e490cbc Mon Sep 17 00:00:00 2001 From: "k.triantafyllou" Date: Mon, 15 Apr 2024 11:13:55 +0300 Subject: [PATCH 4/4] Update monitor service --- pom.xml | 2 +- scripts/2024-04-12.js | 66 ++++++++++++++++++++++++++++++++++++ scripts/datasourceProfile.js | 15 -------- 3 files changed, 67 insertions(+), 16 deletions(-) create mode 100644 scripts/2024-04-12.js delete mode 100644 scripts/datasourceProfile.js diff --git a/pom.xml b/pom.xml index 393fcdd..fc073a8 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ eu.dnetlib uoa-monitor-service - 1.1.10 + 1.1.11 library diff --git a/scripts/2024-04-12.js b/scripts/2024-04-12.js new file mode 100644 index 0000000..d647135 --- /dev/null +++ b/scripts/2024-04-12.js @@ -0,0 +1,66 @@ +function deleteIndexParameters() { + var indicators = db.indicator.find(); + indicators.forEach(indicator => { + indicator.indicatorPaths.forEach(path => { + if(path.parameters) { + delete path.parameters['index_id']; + delete path.parameters['index_name']; + delete path.parameters['index_shortName']; + } + db.indicator.save(indicator); + }); + }) +} + +function convertProfilesToReferences() { + var defaultStakeholders = db.stakeholder.find({defaultId: null}); + defaultStakeholders.forEach(stakeholder => { + stakeholder.copy = false; + db.stakeholder.save(stakeholder); + }) + var stakeholders = db.stakeholder.find({ defaultId: { $exists: true, $ne: null } }); + stakeholders.forEach(stakeholder => { + stakeholder.copy = false; + stakeholder.topics.forEach(topicId => { + var topic = db.topic.findOne({_id: ObjectId(topicId)}); + topic.categories.forEach(categoryId => { + var category = db.category.findOne({_id: ObjectId(categoryId)}); + category.subCategories.forEach(subCategoryId => { + var subCategory = db.subCategory.findOne({_id: ObjectId(subCategoryId)}); + subCategory.numbers.forEach(sectionId => { + var section = db.section.findOne({_id: ObjectId(sectionId)}); + section.indicators.forEach(indicatorId => { + db.indicator.deleteOne({_id: ObjectId(indicatorId)}); + }); + db.section.deleteOne({_id: ObjectId(sectionId)}); + }); + subCategory.charts.forEach(sectionId => { + var section = db.section.findOne({_id: ObjectId(sectionId)}); + section.indicators.forEach(indicatorId => { + db.indicator.deleteOne({_id: ObjectId(indicatorId)}); + }); + db.section.deleteOne({_id: ObjectId(sectionId)}); + }); + db.subCategory.deleteOne({_id: ObjectId(subCategoryId)}); + }); + db.category.deleteOne({_id: ObjectId(categoryId)}); + }); + db.topic.deleteOne({_id: ObjectId(topicId)}); + }); + stakeholder.topics = []; + db.stakeholder.save(stakeholder); + }); +} + +print('topic: ' + db.topic.find({ defaultId: { $exists: true, $ne: null } }).count()); +print('category: ' + db.category.find({ defaultId: { $exists: true, $ne: null } }).count()); +print('subCategory: ' + db.subCategory.find({ defaultId: { $exists: true, $ne: null } }).count()); +print('Section: ' + db.section.find({ defaultId: { $exists: true, $ne: null } }).count()); +print('Indicator: ' + db.indicator.find({ defaultId: { $exists: true, $ne: null } }).count()); +convertProfilesToReferences(); +deleteIndexParameters(); +print('topic: ' + db.topic.find({ defaultId: { $exists: true, $ne: null } }).count()); +print('category: ' + db.category.find({ defaultId: { $exists: true, $ne: null } }).count()); +print('subCategory: ' + db.subCategory.find({ defaultId: { $exists: true, $ne: null } }).count()); +print('Section: ' + db.section.find({ defaultId: { $exists: true, $ne: null } }).count()); +print('Indicator: ' + db.indicator.find({ defaultId: { $exists: true, $ne: null } }).count()); diff --git a/scripts/datasourceProfile.js b/scripts/datasourceProfile.js deleted file mode 100644 index 5416bf9..0000000 --- a/scripts/datasourceProfile.js +++ /dev/null @@ -1,15 +0,0 @@ -function updateDatasourceIndicators() { - var indicators = db.indicator.find(); - indicators.forEach(indicator => { - indicator.indicatorPaths.forEach(path => { - if(path.chartObject && path.chartObject.includes('opendoar____::115f89503138416a242f40fb7d7f338e')) { - while(path.chartObject.includes('opendoar____::115f89503138416a242f40fb7d7f338e')) { - path.chartObject = path.chartObject.replace('opendoar____::115f89503138416a242f40fb7d7f338e', '((__index_id__))'); - } - db.indicator.save(indicator); - } - }); - }); -} - -updateDatasourceIndicators();