refactoring

src/main/java/eu/dnetlib/app/directindex/clients/CommunityClient.java

@@ -19,46 +19,22 @@ import org.springframework.web.client.RestTemplate;
 @Component
 public class CommunityClient implements HasCache {
 
-	private static final String ZENODO_COMMUNITY = "zenodo.org/communities/";
+	public static final String ZENODO_COMMUNITY = "zenodo.org/communities/";
 
 	private static final Log log = LogFactory.getLog(CommunityClient.class);
 
 	@Value("${dnet.directindex.community.url}")
 	private String communityApiUrl;
 
-	public Map<String, ContextInfo> findContexts(final List<String> paths) {
+	@Cacheable("oa_community_lists")
-
+	public List<String> findOpenaireCommunities(final String context) {
-		final Map<String, ContextInfo> res = new HashMap<>();
+		final String url = communityApiUrl + "/" + context + "/openairecommunities";
-
+		return new RestTemplate().getForObject(url, ZenodoContextList.class).getOpenAirecommunitylist();
-		for (final String id : paths) {
-			final String context = id.substring(id.lastIndexOf("/") + 1);
-
-			if (!id.contains(ZENODO_COMMUNITY)) {
-				res.putAll(filterNodes(id));
-			} else {
-				final String url = communityApiUrl + "/" + context + "/openairecommunities";
-				for (final String path : new RestTemplate().getForObject(url, ZenodoContextList.class).getOpenAirecommunitylist()) {
-					res.putAll(filterNodes(path));
-				}
-			}
-		}
-
-		return res;
-
-	}
-
-	private Map<String, ContextInfo> filterNodes(final String path) {
-		final String root = path.split("::")[0];
-
-		return findNodes(root)
-				.entrySet()
-				.stream()
-				.filter(e -> path.equals(e.getKey()) || path.startsWith(e.getKey() + "::"))
-				.collect(Collectors.toMap(Entry::getKey, Entry::getValue));
 	}
 
 	@Cacheable("contexts")
-	private Map<String, ContextInfo> findNodes(final String root) {
+	public Map<String, ContextInfo> findNodes(final String path) {
+		final String root = path.split("::")[0];
 
 		log.info("Preparing context: " + root);
 
@@ -77,11 +53,15 @@ public class CommunityClient implements HasCache {
 			}
 		}
 
-		return res;
+		return res
+				.entrySet()
+				.stream()
+				.filter(e -> path.equals(e.getKey()) || path.startsWith(e.getKey() + "::"))
+				.collect(Collectors.toMap(Entry::getKey, Entry::getValue));
 	}
 
 	@Override
-	@CacheEvict(value = { "contexts" }, allEntries = true)
+	@CacheEvict(value = { "contexts", "oa_community_lists" }, allEntries = true)
 	public void clearCache() {}
 
 	public static class Community implements Serializable {

src/main/java/eu/dnetlib/app/directindex/mapping/SolrRecordMapper.java

@@ -2,6 +2,7 @@ package eu.dnetlib.app.directindex.mapping;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -368,7 +369,19 @@ public class SolrRecordMapper {
 
 	private List<Context> prepareDnetContext(final List<String> list) {
 
-		final Map<String, ContextInfo> nodes = communityClient.findContexts(list);
+		final Map<String, ContextInfo> nodes = new HashMap<>();
+
+		for (final String id : list) {
+			final String context = id.substring(id.lastIndexOf("/") + 1);
+
+			if (!id.contains(CommunityClient.ZENODO_COMMUNITY)) {
+				nodes.putAll(communityClient.findNodes(id));
+			} else {
+				for (final String path : communityClient.findOpenaireCommunities(context)) {
+					nodes.putAll(communityClient.findNodes(path));
+				}
+			}
+		}
 
 		final List<Context> res = new ArrayList<Context>();
 		filterContextInfoByLevel(nodes, 0, null).forEach((k, v) -> {

src/main/java/eu/dnetlib/app/directindex/tasks/ScheduledActions.java

@@ -1,5 +1,7 @@
 package eu.dnetlib.app.directindex.tasks;
 
+import java.time.Duration;
+import java.time.Instant;
 import java.time.LocalDateTime;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -43,7 +45,7 @@ public class ScheduledActions {
 	private PendingActionRepository pendingActionRepository;
 
 	@Scheduled(initialDelay = 1, fixedDelay = 5, timeUnit = TimeUnit.MINUTES)
-	public void indexRecords() {
+	public synchronized void indexRecords() {
 		if (!enabled) {
 			log.info("SKIP");
 			return;
@@ -51,6 +53,7 @@ public class ScheduledActions {
 
 		try {
 			log.info("Indexing new records...");
+			final Instant start = Instant.now();
 
 			final List<PendingAction> list = pendingActionRepository.findInsertOrUpdateOperations();
 
@@ -85,7 +88,10 @@ public class ScheduledActions {
 				updateExecutionDate(list, invalids);
 			}
 
-			log.info(String.format("Indexed records: %s", list.size()));
+			final Instant finish = Instant.now();
+			final long timeElapsed = Duration.between(start, finish).toSeconds() + 1; // I ADD 1 TO AVOID DIVISION BY 0
+
+			log.info(String.format("Indexed %s records in %d seconds (%.3f records/s)", list.size(), timeElapsed, (float) list.size() / timeElapsed));
 		} catch (final Throwable e) {
 			log.error("The scheduled task is failed", e);
 		}
@@ -93,7 +99,7 @@ public class ScheduledActions {
 	}
 
 	@Scheduled(initialDelay = 10, fixedDelay = 30, timeUnit = TimeUnit.MINUTES)
-	public void deleteRecords() {
+	public synchronized void deleteRecords() {
 		if (!enabled) {
 			log.info("SKIP");
 			return;
@@ -142,7 +148,7 @@ public class ScheduledActions {
 		return enabled;
 	}
 
-	public void setEnabled(final boolean enabled) {
+	public synchronized void setEnabled(final boolean enabled) {
 		this.enabled = enabled;
 	}
 

src/test/java/eu/dnetlib/app/directindex/clients/CommunityClientTest.java

@@ -3,7 +3,6 @@ package eu.dnetlib.app.directindex.clients;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
-import java.util.Arrays;
 import java.util.Map;
 
 import org.apache.commons.lang3.StringUtils;
@@ -28,19 +27,25 @@ class CommunityClientTest {
 	}
 
 	@Test
-	public void testFindContexts() {
+	public void testFindContexts_01() {
-		final Map<String, ContextInfo> ctx = communityClient.findContexts(Arrays.asList("sobigdata", "clarin::subcommunity::3", "clarin::subcommunity::4"));
+		final Map<String, ContextInfo> ctx = communityClient.findNodes("sobigdata");
 
-		assertEquals(5, ctx.size());
+		assertEquals(1, ctx.size());
-
-		assertTrue(StringUtils.isNotBlank(ctx.get("clarin").getLabel()));
-		assertTrue(StringUtils.isNotBlank(ctx.get("clarin").getType()));
-		assertTrue(StringUtils.isNotBlank(ctx.get("clarin::subcommunity").getLabel()));
-		assertTrue(StringUtils.isNotBlank(ctx.get("clarin::subcommunity::3").getLabel()));
-		assertTrue(StringUtils.isNotBlank(ctx.get("clarin::subcommunity::4").getLabel()));
 
 		assertTrue(StringUtils.isNotBlank(ctx.get("sobigdata").getLabel()));
 		assertTrue(StringUtils.isNotBlank(ctx.get("sobigdata").getType()));
 	}
 
+	@Test
+	public void testFindContexts_02() {
+		final Map<String, ContextInfo> ctx = communityClient.findNodes("clarin::subcommunity::3");
+
+		assertEquals(3, ctx.size());
+
+		assertTrue(StringUtils.isNotBlank(ctx.get("clarin").getLabel()));
+		assertTrue(StringUtils.isNotBlank(ctx.get("clarin").getType()));
+		assertTrue(StringUtils.isNotBlank(ctx.get("clarin::subcommunity").getLabel()));
+		assertTrue(StringUtils.isNotBlank(ctx.get("clarin::subcommunity::3").getLabel()));
+	}
+
 }

src/test/java/eu/dnetlib/app/directindex/controllers/LegacyApiControllerTest.java

@@ -20,6 +20,8 @@ import eu.dnetlib.app.directindex.tasks.ScheduledActions;
 @Disabled
 public class LegacyApiControllerTest {
 
+	private static final int MAX_RESULTS = 1000;
+
 	@Autowired
 	private LegacyApiController controller;
 
@@ -41,7 +43,7 @@ public class LegacyApiControllerTest {
 		final HttpServletRequest req = Mockito.mock(HttpServletRequest.class);
 		lenient().when(req.getRemoteAddr()).thenReturn("127.0.0.1");
 
-		for (int i = 0; i < 10000; i++) {
+		for (int i = 0; i < MAX_RESULTS; i++) {
 			result.setOpenaireId(null);
 			result.setOriginalId("test::" + i);
 			final String openaireId = controller.feedResult(result, req);
@@ -49,6 +51,7 @@ public class LegacyApiControllerTest {
 		}
 
 		actions.setEnabled(true);
+
 		actions.indexRecords();
 	}