D-Net
/
dnet-directindex-application
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactoring

This commit is contained in:
Michele Artini 2024-12-18 14:55:29 +01:00
parent 9062fa060f
commit 87d0cecf38
5 changed files with 56 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
src/main/java/eu/dnetlib/app/directindex/clients/CommunityClient.java
View File

@ -19,46 +19,22 @@ import org.springframework.web.client.RestTemplate;
@Component
public class CommunityClient implements HasCache {
private static final String ZENODO_COMMUNITY = "zenodo.org/communities/";
public static final String ZENODO_COMMUNITY = "zenodo.org/communities/";
private static final Log log = LogFactory.getLog(CommunityClient.class);
@Value("${dnet.directindex.community.url}")
private String communityApiUrl;
public Map<String, ContextInfo> findContexts(final List<String> paths) {
final Map<String, ContextInfo> res = new HashMap<>();
for (final String id : paths) {
final String context = id.substring(id.lastIndexOf("/") + 1);
if (!id.contains(ZENODO_COMMUNITY)) {
res.putAll(filterNodes(id));
} else {
final String url = communityApiUrl + "/" + context + "/openairecommunities";
for (final String path : new RestTemplate().getForObject(url, ZenodoContextList.class).getOpenAirecommunitylist()) {
res.putAll(filterNodes(path));
}
}
}
return res;
}
private Map<String, ContextInfo> filterNodes(final String path) {
final String root = path.split("::")[0];
return findNodes(root)
.entrySet()
.stream()
.filter(e -> path.equals(e.getKey()) || path.startsWith(e.getKey() + "::"))
.collect(Collectors.toMap(Entry::getKey, Entry::getValue));
@Cacheable("oa_community_lists")
public List<String> findOpenaireCommunities(final String context) {
final String url = communityApiUrl + "/" + context + "/openairecommunities";
return new RestTemplate().getForObject(url, ZenodoContextList.class).getOpenAirecommunitylist();
}
@Cacheable("contexts")
private Map<String, ContextInfo> findNodes(final String root) {
public Map<String, ContextInfo> findNodes(final String path) {
final String root = path.split("::")[0];
log.info("Preparing context: " + root);
@ -77,11 +53,15 @@ public class CommunityClient implements HasCache {
}
}
return res;
return res
.entrySet()
.stream()
.filter(e -> path.equals(e.getKey()) || path.startsWith(e.getKey() + "::"))
.collect(Collectors.toMap(Entry::getKey, Entry::getValue));
}
@Override
@CacheEvict(value = { "contexts" }, allEntries = true)
@CacheEvict(value = { "contexts", "oa_community_lists" }, allEntries = true)
public void clearCache() {}
public static class Community implements Serializable {

15
src/main/java/eu/dnetlib/app/directindex/mapping/SolrRecordMapper.java
View File

@ -2,6 +2,7 @@ package eu.dnetlib.app.directindex.mapping;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@ -368,7 +369,19 @@ public class SolrRecordMapper {
private List<Context> prepareDnetContext(final List<String> list) {
final Map<String, ContextInfo> nodes = communityClient.findContexts(list);
final Map<String, ContextInfo> nodes = new HashMap<>();
for (final String id : list) {
final String context = id.substring(id.lastIndexOf("/") + 1);
if (!id.contains(CommunityClient.ZENODO_COMMUNITY)) {
nodes.putAll(communityClient.findNodes(id));
} else {
for (final String path : communityClient.findOpenaireCommunities(context)) {
nodes.putAll(communityClient.findNodes(path));
}
}
}
final List<Context> res = new ArrayList<Context>();
filterContextInfoByLevel(nodes, 0, null).forEach((k, v) -> {

14
src/main/java/eu/dnetlib/app/directindex/tasks/ScheduledActions.java
View File

@ -1,5 +1,7 @@
package eu.dnetlib.app.directindex.tasks;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.util.HashMap;
import java.util.Iterator;
@ -43,7 +45,7 @@ public class ScheduledActions {
private PendingActionRepository pendingActionRepository;
@Scheduled(initialDelay = 1, fixedDelay = 5, timeUnit = TimeUnit.MINUTES)
public void indexRecords() {
public synchronized void indexRecords() {
if (!enabled) {
log.info("SKIP");
return;
@ -51,6 +53,7 @@ public class ScheduledActions {
try {
log.info("Indexing new records...");
final Instant start = Instant.now();
final List<PendingAction> list = pendingActionRepository.findInsertOrUpdateOperations();
@ -85,7 +88,10 @@ public class ScheduledActions {
updateExecutionDate(list, invalids);
}
log.info(String.format("Indexed records: %s", list.size()));
final Instant finish = Instant.now();
final long timeElapsed = Duration.between(start, finish).toSeconds() + 1; // I ADD 1 TO AVOID DIVISION BY 0
log.info(String.format("Indexed %s records in %d seconds (%.3f records/s)", list.size(), timeElapsed, (float) list.size() / timeElapsed));
} catch (final Throwable e) {
log.error("The scheduled task is failed", e);
}
@ -93,7 +99,7 @@ public class ScheduledActions {
}
@Scheduled(initialDelay = 10, fixedDelay = 30, timeUnit = TimeUnit.MINUTES)
public void deleteRecords() {
public synchronized void deleteRecords() {
if (!enabled) {
log.info("SKIP");
return;
@ -142,7 +148,7 @@ public class ScheduledActions {
return enabled;
}
public void setEnabled(final boolean enabled) {
public synchronized void setEnabled(final boolean enabled) {
this.enabled = enabled;
}

25
src/test/java/eu/dnetlib/app/directindex/clients/CommunityClientTest.java
View File

@ -3,7 +3,6 @@ package eu.dnetlib.app.directindex.clients;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.Arrays;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
@ -28,19 +27,25 @@ class CommunityClientTest {
}
@Test
public void testFindContexts() {
final Map<String, ContextInfo> ctx = communityClient.findContexts(Arrays.asList("sobigdata", "clarin::subcommunity::3", "clarin::subcommunity::4"));
public void testFindContexts_01() {
final Map<String, ContextInfo> ctx = communityClient.findNodes("sobigdata");
assertEquals(5, ctx.size());
assertTrue(StringUtils.isNotBlank(ctx.get("clarin").getLabel()));
assertTrue(StringUtils.isNotBlank(ctx.get("clarin").getType()));
assertTrue(StringUtils.isNotBlank(ctx.get("clarin::subcommunity").getLabel()));
assertTrue(StringUtils.isNotBlank(ctx.get("clarin::subcommunity::3").getLabel()));
assertTrue(StringUtils.isNotBlank(ctx.get("clarin::subcommunity::4").getLabel()));
assertEquals(1, ctx.size());
assertTrue(StringUtils.isNotBlank(ctx.get("sobigdata").getLabel()));
assertTrue(StringUtils.isNotBlank(ctx.get("sobigdata").getType()));
}
@Test
public void testFindContexts_02() {
final Map<String, ContextInfo> ctx = communityClient.findNodes("clarin::subcommunity::3");
assertEquals(3, ctx.size());
assertTrue(StringUtils.isNotBlank(ctx.get("clarin").getLabel()));
assertTrue(StringUtils.isNotBlank(ctx.get("clarin").getType()));
assertTrue(StringUtils.isNotBlank(ctx.get("clarin::subcommunity").getLabel()));
assertTrue(StringUtils.isNotBlank(ctx.get("clarin::subcommunity::3").getLabel()));
}
}

5
src/test/java/eu/dnetlib/app/directindex/controllers/LegacyApiControllerTest.java
View File

@ -20,6 +20,8 @@ import eu.dnetlib.app.directindex.tasks.ScheduledActions;
@Disabled
public class LegacyApiControllerTest {
private static final int MAX_RESULTS = 1000;
@Autowired
private LegacyApiController controller;
@ -41,7 +43,7 @@ public class LegacyApiControllerTest {
final HttpServletRequest req = Mockito.mock(HttpServletRequest.class);
lenient().when(req.getRemoteAddr()).thenReturn("127.0.0.1");
for (int i = 0; i < 10000; i++) {
for (int i = 0; i < MAX_RESULTS; i++) {
result.setOpenaireId(null);
result.setOriginalId("test::" + i);
final String openaireId = controller.feedResult(result, req);
@ -49,6 +51,7 @@ public class LegacyApiControllerTest {
}
actions.setEnabled(true);
actions.indexRecords();
}