D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

logs and properties

This commit is contained in:
Michele Artini 2021-06-14 10:07:08 +02:00
parent ae3db17485
commit 743b68096b
5 changed files with 53 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
apps/dnet-directindex-api/src/main/java/eu/dnetlib/openaire/directindex/AdminController.java Normal file
View File

@ -0,0 +1,25 @@
package eu.dnetlib.openaire.directindex;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.openaire.directindex.is.ISLookupClient;
@RestController
@RequestMapping("/api/admin")
public class AdminController extends AbstractDnetController {
@Autowired
private ISLookupClient isLookupClient;
@GetMapping("/evictCache")
@ResponseStatus(HttpStatus.OK)
public void evictCache() {
isLookupClient.evictCache();
}
}

28
apps/dnet-directindex-api/src/main/java/eu/dnetlib/openaire/directindex/DirectIndexController.java
View File

@ -6,26 +6,22 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.openaire.directindex.input.ResultEntry;
import eu.dnetlib.openaire.directindex.is.ISLookupClient;
import eu.dnetlib.openaire.directindex.mapping.XmlRecordConverter;
import eu.dnetlib.openaire.directindex.repo.RecordInfo;
import eu.dnetlib.openaire.directindex.repo.RecordInfoRepository;
@RestController
@RequestMapping("/api")
@RequestMapping("/api/results")
public class DirectIndexController extends AbstractDnetController {
private static final Log log = LogFactory.getLog(DirectIndexController.class);
@ -36,10 +32,7 @@ public class DirectIndexController extends AbstractDnetController {
@Autowired
private XmlRecordConverter xmlRecordConverter;
@Autowired
private ISLookupClient isLookupClient;
@PostMapping("/results/feedObject")
@PostMapping("/")
public String feedResult(@RequestBody final ResultEntry pub) throws DirectIndexApiException {
log.debug(pub);
@ -66,16 +59,16 @@ public class DirectIndexController extends AbstractDnetController {
}
@DeleteMapping("/result/{openaireId}")
public void deleteResultWithOpenaireId(@PathVariable final String openaireId) throws DirectIndexApiException {
deleteResult(openaireId);
}
@DeleteMapping("/results")
@DeleteMapping("/")
public void deleteResultWithOriginalId(@RequestParam final String originalId, @RequestParam final String collectedFromId) throws Exception {
deleteResult(xmlRecordConverter.calculateOpenaireId(originalId, collectedFromId));
}
@DeleteMapping("/{openaireId}")
public void deleteResultWithOpenaireId(@PathVariable final String openaireId) throws DirectIndexApiException {
deleteResult(openaireId);
}
private void deleteResult(final String openaireId) throws DirectIndexApiException {
final RecordInfo info = new RecordInfo();
@ -88,9 +81,4 @@ public class DirectIndexController extends AbstractDnetController {
recordInfoRepository.save(info);
}
@GetMapping("/admin/evictCache")
@ResponseStatus(HttpStatus.OK)
public void evictCache() {
isLookupClient.evictCache();
}
}

4
apps/dnet-directindex-api/src/main/java/eu/dnetlib/openaire/directindex/repo/RecordInfoRepository.java
View File

@ -9,10 +9,10 @@ import org.springframework.stereotype.Repository;
@Repository
public interface RecordInfoRepository extends JpaRepository<RecordInfo, String> {
@Query(value = "select * from records where execution_date is null and (uc(operation) = 'INSERT' or uc(operation) = 'UPDATE')", nativeQuery = true)
@Query(value = "select * from records where execution_date is null and (upper(operation) = 'INSERT' or upper(operation) = 'UPDATE')", nativeQuery = true)
List<RecordInfo> newRecords();
@Query(value = "select * from records where execution_date is null and uc(operation)", nativeQuery = true)
@Query(value = "select * from records where execution_date is null and upper(operation) = 'DELETE'", nativeQuery = true)
List<RecordInfo> toDeleteRecords();
}

25
apps/dnet-directindex-api/src/main/java/eu/dnetlib/openaire/directindex/scheduled/ScheduledActions.java
View File

@ -8,6 +8,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@ -24,6 +25,7 @@ import eu.dnetlib.openaire.directindex.solr.SolrIndexClient;
import eu.dnetlib.openaire.directindex.solr.SolrIndexClientFactory;
@Component
@ConditionalOnProperty(value = "directindex.scheduling.enabled", havingValue = "true", matchIfMissing = false)
public class ScheduledActions {
private static final Log log = LogFactory.getLog(ScheduledActions.class);
@ -42,6 +44,7 @@ public class ScheduledActions {
@Scheduled(fixedDelay = 5 * 60 * 1000) // 5 minutes
public void indexNewRecords() throws DirectIndexApiException {
log.info("Indexing new records...");
final List<RecordInfo> list = recordInfoRepository.newRecords();
@ -69,16 +72,15 @@ public class ScheduledActions {
updateExecutionDate(list);
}
}
private void updateExecutionDate(final List<RecordInfo> list) {
final OffsetDateTime now = OffsetDateTime.now();
list.forEach(r -> r.setExecutionDate(now));
recordInfoRepository.saveAll(list);
log.info(String.format("done (indexed records: %s)", list.size()));
}
@Scheduled(fixedDelay = 30 * 60 * 1000) // 30 minutes
public void deleteRecords() {
log.info("Deleting records from index...");
final List<RecordInfo> list = recordInfoRepository.toDeleteRecords();
if (list.size() > 0) {
@ -96,6 +98,14 @@ public class ScheduledActions {
updateExecutionDate(list);
}
log.info(String.format("done (deleted records: %s)", list.size()));
}
private void updateExecutionDate(final List<RecordInfo> list) {
final OffsetDateTime now = OffsetDateTime.now();
list.forEach(r -> r.setExecutionDate(now));
recordInfoRepository.saveAll(list);
}
@Scheduled(fixedDelay = 24 * 60 * 60 * 1000) // 24 hours
@ -104,9 +114,4 @@ public class ScheduledActions {
}
@Scheduled(fixedDelay = 15 * 60 * 1000) // 15 minutes
public void evictCache() {
isLookupClient.evictCache();
}
}

5
apps/dnet-directindex-api/src/main/resources/application.properties
View File

@ -28,5 +28,6 @@ spring.jpa.properties.hibernate.show_sql=false
spring.jpa.properties.hibernate.use_sql_comments=false
spring.jpa.properties.hibernate.format_sql=false
openaire.api.community=http://${container.hostname}/openaire/community/
oaf.schema.location=https://www.openaire.eu/schema/1.0/oaf-1.0.xsd
openaire.api.community=http://XXXXXX/openaire/community/
oaf.schema.location=https://www.openaire.eu/schema/1.0/oaf-1.0.xsd
directindex.scheduling.enabled = false