D-Net
/
dnet-directindex-application
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

minor

This commit is contained in:
Michele Artini 2024-11-14 14:58:04 +01:00
parent 6384af47c6
commit c30e4ccf10
6 changed files with 56 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
TODO.txt
View File

@ -17,11 +17,11 @@
- Uso dei record all'interno del wf di provision
5) Generazione del record SOLR
- Profilo del MetadataFormat
- Profilo del MetadataFormat
- XML / JSON
6) Interazione con l'indice SOLR
- TMF/DMF
- Versione di SOLR
- TMF/DMF (SOLUZIONE: si usa il meccanismo degli alias)
- Versione di SOLR (SOLUZIONE: si usa la versione con docker)
- ...

12
src/main/java/eu/dnetlib/app/directindex/controllers/AdminController.java
View File

@ -3,11 +3,13 @@ package eu.dnetlib.app.directindex.controllers;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.app.directindex.is.ISLookupClient;
import eu.dnetlib.app.directindex.tasks.ScheduledActions;
@RestController
@RequestMapping("/api/admin")
@ -16,9 +18,19 @@ public class AdminController {
@Autowired
private ISLookupClient isLookupClient;
@Autowired
private ScheduledActions scheduledActions;
@GetMapping("/evictCache")
@ResponseStatus(HttpStatus.OK)
public void evictCache() {
isLookupClient.evictCache();
}
@GetMapping("/scheduling/{enabled}")
@ResponseStatus(HttpStatus.OK)
public void updateScheduling(@PathVariable final boolean enabled) {
scheduledActions.setEnabled(enabled);
}
}

23
src/main/java/eu/dnetlib/app/directindex/is/ISLookupClient.java
View File

@ -4,8 +4,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.codec.Charsets;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -27,27 +25,6 @@ public class ISLookupClient {
@Autowired
private ISLookUpService lookupService;
@Cacheable("indexDsInfo")
public IndexDsInfo currentIndexDsInfo() {
try {
log.info("Not using cache");
final String queryUrl = IOUtils.toString(getClass().getResourceAsStream("/xquery/findSolrIndexUrl.xquery"), Charsets.UTF_8);
final String queryDs = IOUtils.toString(getClass().getResourceAsStream("/xquery/findIndexDsInfo.xquery"), Charsets.UTF_8);
final String indexBaseUrl = findOne(queryUrl);
final String idxDs = findOne(queryDs);
if (idxDs.isEmpty()) { throw new IllegalStateException(queryDs + "\n\nreturned no results, check IS profiles"); }
final String[] arr = idxDs.split("@@@");
return new IndexDsInfo(indexBaseUrl, arr[0].trim(), arr[1].trim(), arr[2].trim());
} catch (final Exception e) {
log.error(e.getMessage());
throw new RuntimeException(e);
}
}
@Cacheable("datasources")
public DatasourceEntry findDatasource(final String dsId) throws DirectIndexApiException {
final String query =

44
src/main/java/eu/dnetlib/app/directindex/is/IndexDsInfo.java
View File

@ -1,44 +0,0 @@
package eu.dnetlib.app.directindex.is;
public class IndexDsInfo {
private final String indexBaseUrl;
private final String indexDsId;
private final String format;
private final String coll;
public IndexDsInfo(final String indexBaseUrl, final String indexDsId, final String format, final String coll) {
this.indexBaseUrl = indexBaseUrl;
this.indexDsId = indexDsId;
this.format = format;
this.coll = coll;
}
public String getIndexBaseUrl() {
return indexBaseUrl;
}
public String getIndexDsId() {
return indexDsId;
}
public String getFormat() {
return format;
}
public String getColl() {
return coll;
}
@Override
public int hashCode() {
return getColl().hashCode();
}
@Override
public boolean equals(final Object other) {
if (!(other instanceof IndexDsInfo)) { return false; }
return getColl().equals(((IndexDsInfo) other).getColl());
}
}

19
src/main/java/eu/dnetlib/app/directindex/solr/SolrIndexClientFactory.java
View File

@ -8,10 +8,9 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import eu.dnetlib.app.directindex.is.IndexDsInfo;
@Component
public class SolrIndexClientFactory {
@ -19,20 +18,26 @@ public class SolrIndexClientFactory {
public static final String CHROOT_SEPARATOR = "/";
public SolrIndexClient getClient(final IndexDsInfo info) {
log.info(String.format("Initializing solr client (%s) with collection %s", info.getIndexBaseUrl(), info.getColl()));
@Value("${dnet.directindex.solr.url}")
public String solrIndexBaseUrl;
@Value("${dnet.directindex.solr.collection}")
public String solrCollection;
public SolrIndexClient getClient() {
log.info(String.format("Initializing solr client, url: %s, collection: %s", solrIndexBaseUrl, solrCollection));
// Example: quorum0:2182,quorum1:2182,quorum2:2182,quorum3:2182,quorum4:2182/solr-dev-openaire
final String s = StringUtils.substringAfterLast(info.getIndexBaseUrl(), CHROOT_SEPARATOR);
final String s = StringUtils.substringAfterLast(solrIndexBaseUrl, CHROOT_SEPARATOR);
final String chroot = StringUtils.isNotBlank(s) ? CHROOT_SEPARATOR + s : null;
final String urls = chroot != null ? info.getIndexBaseUrl().replace(chroot, "") : info.getIndexBaseUrl();
final String urls = chroot != null ? solrIndexBaseUrl.replace(chroot, "") : solrIndexBaseUrl;
final List<String> urlList = Arrays.stream(urls.split(",")).map(String::trim).filter(StringUtils::isNotBlank).toList();
final CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder(urlList, Optional.of(chroot))
.withParallelUpdates(true)
.withDefaultCollection(info.getColl())
.withDefaultCollection(solrCollection)
.build();
return new SolrIndexClient(cloudSolrClient);

36
src/main/java/eu/dnetlib/app/directindex/tasks/ScheduledActions.java
View File

@ -8,7 +8,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@ -17,7 +17,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.app.directindex.errors.DirectIndexApiException;
import eu.dnetlib.app.directindex.input.ResultEntry;
import eu.dnetlib.app.directindex.is.ISLookupClient;
import eu.dnetlib.app.directindex.is.IndexDsInfo;
import eu.dnetlib.app.directindex.mapping.OafMapper;
import eu.dnetlib.app.directindex.repo.PendingAction;
import eu.dnetlib.app.directindex.repo.PendingActionRepository;
@ -25,11 +24,13 @@ import eu.dnetlib.app.directindex.solr.SolrIndexClient;
import eu.dnetlib.app.directindex.solr.SolrIndexClientFactory;
@Component
@ConditionalOnProperty(value = "dnet.directindex.scheduling.enabled", havingValue = "true", matchIfMissing = false)
public class ScheduledActions {
private static final Log log = LogFactory.getLog(ScheduledActions.class);
@Value(value = "${dnet.directindex.scheduling.enabled}")
private boolean enabled;
@Autowired
private ISLookupClient isLookupClient;
@ -41,13 +42,17 @@ public class ScheduledActions {
@Scheduled(fixedDelay = 5 * 60 * 1000) // 5 minutes
public void indexNewRecords() throws DirectIndexApiException {
if (!enabled) {
log.info("SKIP");
return;
}
log.info("Indexing new records...");
final List<PendingAction> list = pendingActionRepository.recentActions();
if (list.size() > 0) {
final IndexDsInfo info = isLookupClient.currentIndexDsInfo();
final SolrIndexClient solr = solrIndexClientFactory.getClient(info);
final SolrIndexClient solr = solrIndexClientFactory.getClient();
final ObjectMapper objectMapper = new ObjectMapper();
@ -76,13 +81,17 @@ public class ScheduledActions {
@Scheduled(fixedDelay = 30 * 60 * 1000) // 30 minutes
public void deleteRecords() {
if (!enabled) {
log.info("SKIP");
return;
}
log.info("Deleting records from index...");
final List<PendingAction> list = pendingActionRepository.toDeleteRecords();
if (list.size() > 0) {
final IndexDsInfo info = isLookupClient.currentIndexDsInfo();
final SolrIndexClient solr = solrIndexClientFactory.getClient(info);
final SolrIndexClient solr = solrIndexClientFactory.getClient();
list.stream().map(PendingAction::getId).forEach(id -> {
try {
@ -107,8 +116,21 @@ public class ScheduledActions {
@Scheduled(fixedDelay = 24 * 60 * 60 * 1000) // 24 hours
public void removeOldRecordsFromDB() {
if (!enabled) {
log.info("SKIP");
return;
}
// TODO
}
public boolean isEnabled() {
return enabled;
}
public void setEnabled(final boolean enabled) {
this.enabled = enabled;
}
}