From 87d0cecf387a426e5c1d037e317913a0fe948f66 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 18 Dec 2024 14:55:29 +0100 Subject: [PATCH] refactoring --- .../directindex/clients/CommunityClient.java | 46 ++++++------------- .../directindex/mapping/SolrRecordMapper.java | 15 +++++- .../directindex/tasks/ScheduledActions.java | 14 ++++-- .../clients/CommunityClientTest.java | 25 ++++++---- .../controllers/LegacyApiControllerTest.java | 5 +- 5 files changed, 56 insertions(+), 49 deletions(-) diff --git a/src/main/java/eu/dnetlib/app/directindex/clients/CommunityClient.java b/src/main/java/eu/dnetlib/app/directindex/clients/CommunityClient.java index 663e1e0..da5c6c1 100644 --- a/src/main/java/eu/dnetlib/app/directindex/clients/CommunityClient.java +++ b/src/main/java/eu/dnetlib/app/directindex/clients/CommunityClient.java @@ -19,46 +19,22 @@ import org.springframework.web.client.RestTemplate; @Component public class CommunityClient implements HasCache { - private static final String ZENODO_COMMUNITY = "zenodo.org/communities/"; + public static final String ZENODO_COMMUNITY = "zenodo.org/communities/"; private static final Log log = LogFactory.getLog(CommunityClient.class); @Value("${dnet.directindex.community.url}") private String communityApiUrl; - public Map findContexts(final List paths) { - - final Map res = new HashMap<>(); - - for (final String id : paths) { - final String context = id.substring(id.lastIndexOf("/") + 1); - - if (!id.contains(ZENODO_COMMUNITY)) { - res.putAll(filterNodes(id)); - } else { - final String url = communityApiUrl + "/" + context + "/openairecommunities"; - for (final String path : new RestTemplate().getForObject(url, ZenodoContextList.class).getOpenAirecommunitylist()) { - res.putAll(filterNodes(path)); - } - } - } - - return res; - - } - - private Map filterNodes(final String path) { - final String root = path.split("::")[0]; - - return findNodes(root) - .entrySet() - .stream() - .filter(e -> path.equals(e.getKey()) || path.startsWith(e.getKey() + "::")) - .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); + @Cacheable("oa_community_lists") + public List findOpenaireCommunities(final String context) { + final String url = communityApiUrl + "/" + context + "/openairecommunities"; + return new RestTemplate().getForObject(url, ZenodoContextList.class).getOpenAirecommunitylist(); } @Cacheable("contexts") - private Map findNodes(final String root) { + public Map findNodes(final String path) { + final String root = path.split("::")[0]; log.info("Preparing context: " + root); @@ -77,11 +53,15 @@ public class CommunityClient implements HasCache { } } - return res; + return res + .entrySet() + .stream() + .filter(e -> path.equals(e.getKey()) || path.startsWith(e.getKey() + "::")) + .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); } @Override - @CacheEvict(value = { "contexts" }, allEntries = true) + @CacheEvict(value = { "contexts", "oa_community_lists" }, allEntries = true) public void clearCache() {} public static class Community implements Serializable { diff --git a/src/main/java/eu/dnetlib/app/directindex/mapping/SolrRecordMapper.java b/src/main/java/eu/dnetlib/app/directindex/mapping/SolrRecordMapper.java index 17e4676..a2b715d 100644 --- a/src/main/java/eu/dnetlib/app/directindex/mapping/SolrRecordMapper.java +++ b/src/main/java/eu/dnetlib/app/directindex/mapping/SolrRecordMapper.java @@ -2,6 +2,7 @@ package eu.dnetlib.app.directindex.mapping; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -368,7 +369,19 @@ public class SolrRecordMapper { private List prepareDnetContext(final List list) { - final Map nodes = communityClient.findContexts(list); + final Map nodes = new HashMap<>(); + + for (final String id : list) { + final String context = id.substring(id.lastIndexOf("/") + 1); + + if (!id.contains(CommunityClient.ZENODO_COMMUNITY)) { + nodes.putAll(communityClient.findNodes(id)); + } else { + for (final String path : communityClient.findOpenaireCommunities(context)) { + nodes.putAll(communityClient.findNodes(path)); + } + } + } final List res = new ArrayList(); filterContextInfoByLevel(nodes, 0, null).forEach((k, v) -> { diff --git a/src/main/java/eu/dnetlib/app/directindex/tasks/ScheduledActions.java b/src/main/java/eu/dnetlib/app/directindex/tasks/ScheduledActions.java index 6d8c2ba..a80869d 100644 --- a/src/main/java/eu/dnetlib/app/directindex/tasks/ScheduledActions.java +++ b/src/main/java/eu/dnetlib/app/directindex/tasks/ScheduledActions.java @@ -1,5 +1,7 @@ package eu.dnetlib.app.directindex.tasks; +import java.time.Duration; +import java.time.Instant; import java.time.LocalDateTime; import java.util.HashMap; import java.util.Iterator; @@ -43,7 +45,7 @@ public class ScheduledActions { private PendingActionRepository pendingActionRepository; @Scheduled(initialDelay = 1, fixedDelay = 5, timeUnit = TimeUnit.MINUTES) - public void indexRecords() { + public synchronized void indexRecords() { if (!enabled) { log.info("SKIP"); return; @@ -51,6 +53,7 @@ public class ScheduledActions { try { log.info("Indexing new records..."); + final Instant start = Instant.now(); final List list = pendingActionRepository.findInsertOrUpdateOperations(); @@ -85,7 +88,10 @@ public class ScheduledActions { updateExecutionDate(list, invalids); } - log.info(String.format("Indexed records: %s", list.size())); + final Instant finish = Instant.now(); + final long timeElapsed = Duration.between(start, finish).toSeconds() + 1; // I ADD 1 TO AVOID DIVISION BY 0 + + log.info(String.format("Indexed %s records in %d seconds (%.3f records/s)", list.size(), timeElapsed, (float) list.size() / timeElapsed)); } catch (final Throwable e) { log.error("The scheduled task is failed", e); } @@ -93,7 +99,7 @@ public class ScheduledActions { } @Scheduled(initialDelay = 10, fixedDelay = 30, timeUnit = TimeUnit.MINUTES) - public void deleteRecords() { + public synchronized void deleteRecords() { if (!enabled) { log.info("SKIP"); return; @@ -142,7 +148,7 @@ public class ScheduledActions { return enabled; } - public void setEnabled(final boolean enabled) { + public synchronized void setEnabled(final boolean enabled) { this.enabled = enabled; } diff --git a/src/test/java/eu/dnetlib/app/directindex/clients/CommunityClientTest.java b/src/test/java/eu/dnetlib/app/directindex/clients/CommunityClientTest.java index 7caf53c..7b17092 100644 --- a/src/test/java/eu/dnetlib/app/directindex/clients/CommunityClientTest.java +++ b/src/test/java/eu/dnetlib/app/directindex/clients/CommunityClientTest.java @@ -3,7 +3,6 @@ package eu.dnetlib.app.directindex.clients; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.util.Arrays; import java.util.Map; import org.apache.commons.lang3.StringUtils; @@ -28,19 +27,25 @@ class CommunityClientTest { } @Test - public void testFindContexts() { - final Map ctx = communityClient.findContexts(Arrays.asList("sobigdata", "clarin::subcommunity::3", "clarin::subcommunity::4")); + public void testFindContexts_01() { + final Map ctx = communityClient.findNodes("sobigdata"); - assertEquals(5, ctx.size()); - - assertTrue(StringUtils.isNotBlank(ctx.get("clarin").getLabel())); - assertTrue(StringUtils.isNotBlank(ctx.get("clarin").getType())); - assertTrue(StringUtils.isNotBlank(ctx.get("clarin::subcommunity").getLabel())); - assertTrue(StringUtils.isNotBlank(ctx.get("clarin::subcommunity::3").getLabel())); - assertTrue(StringUtils.isNotBlank(ctx.get("clarin::subcommunity::4").getLabel())); + assertEquals(1, ctx.size()); assertTrue(StringUtils.isNotBlank(ctx.get("sobigdata").getLabel())); assertTrue(StringUtils.isNotBlank(ctx.get("sobigdata").getType())); } + @Test + public void testFindContexts_02() { + final Map ctx = communityClient.findNodes("clarin::subcommunity::3"); + + assertEquals(3, ctx.size()); + + assertTrue(StringUtils.isNotBlank(ctx.get("clarin").getLabel())); + assertTrue(StringUtils.isNotBlank(ctx.get("clarin").getType())); + assertTrue(StringUtils.isNotBlank(ctx.get("clarin::subcommunity").getLabel())); + assertTrue(StringUtils.isNotBlank(ctx.get("clarin::subcommunity::3").getLabel())); + } + } diff --git a/src/test/java/eu/dnetlib/app/directindex/controllers/LegacyApiControllerTest.java b/src/test/java/eu/dnetlib/app/directindex/controllers/LegacyApiControllerTest.java index 803c1fb..71aa9f1 100644 --- a/src/test/java/eu/dnetlib/app/directindex/controllers/LegacyApiControllerTest.java +++ b/src/test/java/eu/dnetlib/app/directindex/controllers/LegacyApiControllerTest.java @@ -20,6 +20,8 @@ import eu.dnetlib.app.directindex.tasks.ScheduledActions; @Disabled public class LegacyApiControllerTest { + private static final int MAX_RESULTS = 1000; + @Autowired private LegacyApiController controller; @@ -41,7 +43,7 @@ public class LegacyApiControllerTest { final HttpServletRequest req = Mockito.mock(HttpServletRequest.class); lenient().when(req.getRemoteAddr()).thenReturn("127.0.0.1"); - for (int i = 0; i < 10000; i++) { + for (int i = 0; i < MAX_RESULTS; i++) { result.setOpenaireId(null); result.setOriginalId("test::" + i); final String openaireId = controller.feedResult(result, req); @@ -49,6 +51,7 @@ public class LegacyApiControllerTest { } actions.setEnabled(true); + actions.indexRecords(); }