diff --git a/src/main/java/eu/dnetlib/app/directindex/repo/OperationType.java b/src/main/java/eu/dnetlib/app/directindex/repo/OperationType.java new file mode 100644 index 0000000..fe7e8bf --- /dev/null +++ b/src/main/java/eu/dnetlib/app/directindex/repo/OperationType.java @@ -0,0 +1,5 @@ +package eu.dnetlib.app.directindex.repo; + +public enum OperationType { + DELETE, INSERT, UPDATE +} diff --git a/src/main/java/eu/dnetlib/app/directindex/repo/PendingAction.java b/src/main/java/eu/dnetlib/app/directindex/repo/PendingAction.java index 8198d82..a3439c1 100644 --- a/src/main/java/eu/dnetlib/app/directindex/repo/PendingAction.java +++ b/src/main/java/eu/dnetlib/app/directindex/repo/PendingAction.java @@ -1,9 +1,11 @@ package eu.dnetlib.app.directindex.repo; -import java.time.OffsetDateTime; +import java.time.LocalDateTime; import jakarta.persistence.Column; import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; import jakarta.persistence.Id; import jakarta.persistence.Table; @@ -18,11 +20,9 @@ public class PendingAction { @Column(name = "type") private String type; - /** - * One of: DELETE, INSERT, UPDATE - */ @Column(name = "operation") - private String operation; + @Enumerated(EnumType.STRING) + private OperationType operation; @Column(name = "body") private String body; @@ -31,10 +31,10 @@ public class PendingAction { private String createdBy; @Column(name = "creation_date") - private OffsetDateTime creationDate; + private LocalDateTime creationDate; @Column(name = "execution_date") - private OffsetDateTime executionDate; + private LocalDateTime executionDate; public String getId() { return id; @@ -52,11 +52,11 @@ public class PendingAction { this.type = type; } - public String getOperation() { + public OperationType getOperation() { return operation; } - public void setOperation(final String operation) { + public void setOperation(final OperationType operation) { this.operation = operation; } @@ -76,19 +76,19 @@ public class PendingAction { this.createdBy = createdBy; } - public OffsetDateTime getCreationDate() { + public LocalDateTime getCreationDate() { return creationDate; } - public void setCreationDate(final OffsetDateTime creationDate) { + public void setCreationDate(final LocalDateTime creationDate) { this.creationDate = creationDate; } - public OffsetDateTime getExecutionDate() { + public LocalDateTime getExecutionDate() { return executionDate; } - public void setExecutionDate(final OffsetDateTime executionDate) { + public void setExecutionDate(final LocalDateTime executionDate) { this.executionDate = executionDate; } diff --git a/src/main/java/eu/dnetlib/app/directindex/repo/PendingActionRepository.java b/src/main/java/eu/dnetlib/app/directindex/repo/PendingActionRepository.java index eef9a1a..563c12d 100644 --- a/src/main/java/eu/dnetlib/app/directindex/repo/PendingActionRepository.java +++ b/src/main/java/eu/dnetlib/app/directindex/repo/PendingActionRepository.java @@ -11,10 +11,10 @@ import org.springframework.stereotype.Repository; public interface PendingActionRepository extends JpaRepository { @Query(value = "select * from pending_actions where execution_date is null and (upper(operation) = 'INSERT' or upper(operation) = 'UPDATE')", nativeQuery = true) - List recentActions(); + List findInsertOrUpdateOperations(); @Query(value = "select * from pending_actions where execution_date is null and upper(operation) = 'DELETE'", nativeQuery = true) - List toDeleteRecords(); + List findDeleteOperations(); void deleteByCreationDateBefore(LocalDateTime datatime); diff --git a/src/main/java/eu/dnetlib/app/directindex/service/DirectIndexService.java b/src/main/java/eu/dnetlib/app/directindex/service/DirectIndexService.java index 204b131..6b23feb 100644 --- a/src/main/java/eu/dnetlib/app/directindex/service/DirectIndexService.java +++ b/src/main/java/eu/dnetlib/app/directindex/service/DirectIndexService.java @@ -1,7 +1,6 @@ package eu.dnetlib.app.directindex.service; import java.time.LocalDateTime; -import java.time.OffsetDateTime; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; @@ -12,6 +11,7 @@ import eu.dnetlib.app.directindex.clients.DatasourceManagerClient; import eu.dnetlib.app.directindex.errors.DirectIndexApiException; import eu.dnetlib.app.directindex.input.DatasourceEntry; import eu.dnetlib.app.directindex.input.ResultEntry; +import eu.dnetlib.app.directindex.repo.OperationType; import eu.dnetlib.app.directindex.repo.PendingAction; import eu.dnetlib.app.directindex.repo.PendingActionRepository; @@ -31,9 +31,9 @@ public class DirectIndexService { final PendingAction action = new PendingAction(); action.setId(openaireId); - action.setOperation("DELETE"); + action.setOperation(OperationType.DELETE); action.setCreatedBy(createdBy); - action.setCreationDate(OffsetDateTime.now()); + action.setCreationDate(LocalDateTime.now()); action.setExecutionDate(null); pendingActionRepository.save(action); @@ -46,10 +46,10 @@ public class DirectIndexService { if (!r.getOpenaireId().matches("^\\w{12}::\\w{32}$")) { throw new DirectIndexApiException("Invalid openaireId: " + r.getOpenaireId() + " - regex ^\\w{12}::\\w{32}$ not matched"); } - info.setOperation("UPDATE"); + info.setOperation(OperationType.UPDATE); } else if (StringUtils.isNoneBlank(r.getOriginalId(), r.getCollectedFromId())) { fixOpenaireId(r); - info.setOperation("INSERT"); + info.setOperation(OperationType.INSERT); } else { throw new DirectIndexApiException("Missing identifier fields: [openaireId] or [originalId, collectedFromId]"); } @@ -65,7 +65,7 @@ public class DirectIndexService { info.setBody(r.toJson()); info.setType(r.getType()); info.setCreatedBy(createdBy); - info.setCreationDate(OffsetDateTime.now()); + info.setCreationDate(LocalDateTime.now()); info.setExecutionDate(null); pendingActionRepository.save(info); diff --git a/src/main/java/eu/dnetlib/app/directindex/solr/SolrIndexClient.java b/src/main/java/eu/dnetlib/app/directindex/solr/SolrIndexClient.java index 0b9d451..894b1c6 100644 --- a/src/main/java/eu/dnetlib/app/directindex/solr/SolrIndexClient.java +++ b/src/main/java/eu/dnetlib/app/directindex/solr/SolrIndexClient.java @@ -32,15 +32,6 @@ public class SolrIndexClient { this.cloudSolrClient = cloudSolrClient; } - public void deleteByQuery(final String query) throws DirectIndexApiException { - try { - cloudSolrClient.deleteByQuery(query); - cloudSolrClient.commit(); - } catch (SolrServerException | IOException e) { - throw new DirectIndexApiException(e); - } - } - public void addRecords(final Stream records) throws DirectIndexApiException { try { cloudSolrClient.add(records.map(this::prepareSolrDocument).iterator()); @@ -54,6 +45,18 @@ public class SolrIndexClient { addRecords(Arrays.stream(records)); } + public void deleteRecord(final String id) throws DirectIndexApiException { + try { + // final String query = String.format("objidentifier:%s OR resultdupid:%s", ClientUtils.escapeQueryChars(id), + // ClientUtils.escapeQueryChars(id)); + final String query = String.format("objidentifier:\"%s\" OR resultdupid:\"%s\"", id, id); + cloudSolrClient.deleteByQuery(query); + cloudSolrClient.commit(); + } catch (SolrServerException | IOException e) { + throw new DirectIndexApiException(e); + } + } + private SolrInputDocument prepareSolrDocument(final SolrRecord record) { // TODO (usare classe condivisa preparata da Claudio) diff --git a/src/main/java/eu/dnetlib/app/directindex/tasks/ScheduledActions.java b/src/main/java/eu/dnetlib/app/directindex/tasks/ScheduledActions.java index 0ef4d46..085c64d 100644 --- a/src/main/java/eu/dnetlib/app/directindex/tasks/ScheduledActions.java +++ b/src/main/java/eu/dnetlib/app/directindex/tasks/ScheduledActions.java @@ -1,8 +1,9 @@ package eu.dnetlib.app.directindex.tasks; -import java.time.OffsetDateTime; +import java.time.LocalDateTime; import java.util.List; import java.util.Objects; +import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -38,7 +39,7 @@ public class ScheduledActions { @Autowired private PendingActionRepository pendingActionRepository; - @Scheduled(fixedDelay = 5 * 60 * 1000) // 5 minutes + @Scheduled(initialDelay = 5, fixedDelay = 5, timeUnit = TimeUnit.MINUTES) public void indexNewRecords() throws DirectIndexApiException { if (!enabled) { log.info("SKIP"); @@ -47,7 +48,7 @@ public class ScheduledActions { log.info("Indexing new records..."); - final List list = pendingActionRepository.recentActions(); + final List list = pendingActionRepository.findInsertOrUpdateOperations(); if (list.size() > 0) { final SolrIndexClient solr = solrIndexClientFactory.getClient(); @@ -75,7 +76,7 @@ public class ScheduledActions { } - @Scheduled(fixedDelay = 30 * 60 * 1000) // 30 minutes + @Scheduled(initialDelay = 10, fixedDelay = 30, timeUnit = TimeUnit.MINUTES) public void deleteRecords() { if (!enabled) { log.info("SKIP"); @@ -84,15 +85,14 @@ public class ScheduledActions { log.info("Deleting records from index..."); - final List list = pendingActionRepository.toDeleteRecords(); + final List list = pendingActionRepository.findDeleteOperations(); if (list.size() > 0) { final SolrIndexClient solr = solrIndexClientFactory.getClient(); list.stream().map(PendingAction::getId).forEach(id -> { try { - final String query = String.format("objidentifier:\"%s\" OR resultdupid:\"%s\"", id, id); - solr.deleteByQuery(query); + solr.deleteRecord(id); } catch (final DirectIndexApiException e) { log.error(e); } @@ -105,7 +105,7 @@ public class ScheduledActions { } private void updateExecutionDate(final List list) { - final OffsetDateTime now = OffsetDateTime.now(); + final LocalDateTime now = LocalDateTime.now(); list.forEach(r -> r.setExecutionDate(now)); pendingActionRepository.saveAll(list); } diff --git a/src/test/java/eu/dnetlib/app/directindex/solr/SolrIndexClientTest.java b/src/test/java/eu/dnetlib/app/directindex/solr/SolrIndexClientTest.java new file mode 100644 index 0000000..2f0b66a --- /dev/null +++ b/src/test/java/eu/dnetlib/app/directindex/solr/SolrIndexClientTest.java @@ -0,0 +1,20 @@ +package eu.dnetlib.app.directindex.solr; + +import org.apache.solr.client.solrj.util.ClientUtils; +import org.junit.jupiter.api.Test; + +class SolrIndexClientTest { + + @Test + void testParameters() { + final String id = "test________::1234"; + + final String query1 = String.format("objidentifier:\"%s\" OR resultdupid:\"%s\"", id, id); + final String query2 = String.format("objidentifier:%s OR resultdupid:%s", ClientUtils.escapeQueryChars(id), ClientUtils.escapeQueryChars(id)); + + System.out.println(query1); + System.out.println(query2); + + } + +}