D-Net
/
dnet-directindex-application
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

test

This commit is contained in:
Michele Artini 2024-11-22 15:31:14 +01:00
parent 9b762600b2
commit 4ca5f77fa0
7 changed files with 66 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/eu/dnetlib/app/directindex/repo/OperationType.java Normal file
View File

@ -0,0 +1,5 @@
package eu.dnetlib.app.directindex.repo;
public enum OperationType {
DELETE, INSERT, UPDATE
}

26
src/main/java/eu/dnetlib/app/directindex/repo/PendingAction.java
View File

@ -1,9 +1,11 @@
package eu.dnetlib.app.directindex.repo;
import java.time.OffsetDateTime;
import java.time.LocalDateTime;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.Id;
import jakarta.persistence.Table;
@ -18,11 +20,9 @@ public class PendingAction {
@Column(name = "type")
private String type;
/**
* One of: DELETE, INSERT, UPDATE
*/
@Column(name = "operation")
private String operation;
@Enumerated(EnumType.STRING)
private OperationType operation;
@Column(name = "body")
private String body;
@ -31,10 +31,10 @@ public class PendingAction {
private String createdBy;
@Column(name = "creation_date")
private OffsetDateTime creationDate;
private LocalDateTime creationDate;
@Column(name = "execution_date")
private OffsetDateTime executionDate;
private LocalDateTime executionDate;
public String getId() {
return id;
@ -52,11 +52,11 @@ public class PendingAction {
this.type = type;
}
public String getOperation() {
public OperationType getOperation() {
return operation;
}
public void setOperation(final String operation) {
public void setOperation(final OperationType operation) {
this.operation = operation;
}
@ -76,19 +76,19 @@ public class PendingAction {
this.createdBy = createdBy;
}
public OffsetDateTime getCreationDate() {
public LocalDateTime getCreationDate() {
return creationDate;
}
public void setCreationDate(final OffsetDateTime creationDate) {
public void setCreationDate(final LocalDateTime creationDate) {
this.creationDate = creationDate;
}
public OffsetDateTime getExecutionDate() {
public LocalDateTime getExecutionDate() {
return executionDate;
}
public void setExecutionDate(final OffsetDateTime executionDate) {
public void setExecutionDate(final LocalDateTime executionDate) {
this.executionDate = executionDate;
}

4
src/main/java/eu/dnetlib/app/directindex/repo/PendingActionRepository.java
View File

@ -11,10 +11,10 @@ import org.springframework.stereotype.Repository;
public interface PendingActionRepository extends JpaRepository<PendingAction, String> {
@Query(value = "select * from pending_actions where execution_date is null and (upper(operation) = 'INSERT' or upper(operation) = 'UPDATE')", nativeQuery = true)
List<PendingAction> recentActions();
List<PendingAction> findInsertOrUpdateOperations();
@Query(value = "select * from pending_actions where execution_date is null and upper(operation) = 'DELETE'", nativeQuery = true)
List<PendingAction> toDeleteRecords();
List<PendingAction> findDeleteOperations();
void deleteByCreationDateBefore(LocalDateTime datatime);

12
src/main/java/eu/dnetlib/app/directindex/service/DirectIndexService.java
View File

@ -1,7 +1,6 @@
package eu.dnetlib.app.directindex.service;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
@ -12,6 +11,7 @@ import eu.dnetlib.app.directindex.clients.DatasourceManagerClient;
import eu.dnetlib.app.directindex.errors.DirectIndexApiException;
import eu.dnetlib.app.directindex.input.DatasourceEntry;
import eu.dnetlib.app.directindex.input.ResultEntry;
import eu.dnetlib.app.directindex.repo.OperationType;
import eu.dnetlib.app.directindex.repo.PendingAction;
import eu.dnetlib.app.directindex.repo.PendingActionRepository;
@ -31,9 +31,9 @@ public class DirectIndexService {
final PendingAction action = new PendingAction();
action.setId(openaireId);
action.setOperation("DELETE");
action.setOperation(OperationType.DELETE);
action.setCreatedBy(createdBy);
action.setCreationDate(OffsetDateTime.now());
action.setCreationDate(LocalDateTime.now());
action.setExecutionDate(null);
pendingActionRepository.save(action);
@ -46,10 +46,10 @@ public class DirectIndexService {
if (!r.getOpenaireId().matches("^\\w{12}::\\w{32}$")) {
throw new DirectIndexApiException("Invalid openaireId: " + r.getOpenaireId() + " - regex ^\\w{12}::\\w{32}$ not matched");
}
info.setOperation("UPDATE");
info.setOperation(OperationType.UPDATE);
} else if (StringUtils.isNoneBlank(r.getOriginalId(), r.getCollectedFromId())) {
fixOpenaireId(r);
info.setOperation("INSERT");
info.setOperation(OperationType.INSERT);
} else {
throw new DirectIndexApiException("Missing identifier fields: [openaireId] or [originalId, collectedFromId]");
}
@ -65,7 +65,7 @@ public class DirectIndexService {
info.setBody(r.toJson());
info.setType(r.getType());
info.setCreatedBy(createdBy);
info.setCreationDate(OffsetDateTime.now());
info.setCreationDate(LocalDateTime.now());
info.setExecutionDate(null);
pendingActionRepository.save(info);

21
src/main/java/eu/dnetlib/app/directindex/solr/SolrIndexClient.java
View File

@ -32,15 +32,6 @@ public class SolrIndexClient {
this.cloudSolrClient = cloudSolrClient;
}
public void deleteByQuery(final String query) throws DirectIndexApiException {
try {
cloudSolrClient.deleteByQuery(query);
cloudSolrClient.commit();
} catch (SolrServerException | IOException e) {
throw new DirectIndexApiException(e);
}
}
public void addRecords(final Stream<SolrRecord> records) throws DirectIndexApiException {
try {
cloudSolrClient.add(records.map(this::prepareSolrDocument).iterator());
@ -54,6 +45,18 @@ public class SolrIndexClient {
addRecords(Arrays.stream(records));
}
public void deleteRecord(final String id) throws DirectIndexApiException {
try {
// final String query = String.format("objidentifier:%s OR resultdupid:%s", ClientUtils.escapeQueryChars(id),
// ClientUtils.escapeQueryChars(id));
final String query = String.format("objidentifier:\"%s\" OR resultdupid:\"%s\"", id, id);
cloudSolrClient.deleteByQuery(query);
cloudSolrClient.commit();
} catch (SolrServerException | IOException e) {
throw new DirectIndexApiException(e);
}
}
private SolrInputDocument prepareSolrDocument(final SolrRecord record) {
// TODO (usare classe condivisa preparata da Claudio)

16
src/main/java/eu/dnetlib/app/directindex/tasks/ScheduledActions.java
View File

@ -1,8 +1,9 @@
package eu.dnetlib.app.directindex.tasks;
import java.time.OffsetDateTime;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -38,7 +39,7 @@ public class ScheduledActions {
@Autowired
private PendingActionRepository pendingActionRepository;
@Scheduled(fixedDelay = 5 * 60 * 1000) // 5 minutes
@Scheduled(initialDelay = 5, fixedDelay = 5, timeUnit = TimeUnit.MINUTES)
public void indexNewRecords() throws DirectIndexApiException {
if (!enabled) {
log.info("SKIP");
@ -47,7 +48,7 @@ public class ScheduledActions {
log.info("Indexing new records...");
final List<PendingAction> list = pendingActionRepository.recentActions();
final List<PendingAction> list = pendingActionRepository.findInsertOrUpdateOperations();
if (list.size() > 0) {
final SolrIndexClient solr = solrIndexClientFactory.getClient();
@ -75,7 +76,7 @@ public class ScheduledActions {
}
@Scheduled(fixedDelay = 30 * 60 * 1000) // 30 minutes
@Scheduled(initialDelay = 10, fixedDelay = 30, timeUnit = TimeUnit.MINUTES)
public void deleteRecords() {
if (!enabled) {
log.info("SKIP");
@ -84,15 +85,14 @@ public class ScheduledActions {
log.info("Deleting records from index...");
final List<PendingAction> list = pendingActionRepository.toDeleteRecords();
final List<PendingAction> list = pendingActionRepository.findDeleteOperations();
if (list.size() > 0) {
final SolrIndexClient solr = solrIndexClientFactory.getClient();
list.stream().map(PendingAction::getId).forEach(id -> {
try {
final String query = String.format("objidentifier:\"%s\" OR resultdupid:\"%s\"", id, id);
solr.deleteByQuery(query);
solr.deleteRecord(id);
} catch (final DirectIndexApiException e) {
log.error(e);
}
@ -105,7 +105,7 @@ public class ScheduledActions {
}
private void updateExecutionDate(final List<PendingAction> list) {
final OffsetDateTime now = OffsetDateTime.now();
final LocalDateTime now = LocalDateTime.now();
list.forEach(r -> r.setExecutionDate(now));
pendingActionRepository.saveAll(list);
}

20
src/test/java/eu/dnetlib/app/directindex/solr/SolrIndexClientTest.java Normal file
View File

@ -0,0 +1,20 @@
package eu.dnetlib.app.directindex.solr;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.junit.jupiter.api.Test;
class SolrIndexClientTest {
@Test
void testParameters() {
final String id = "test________::1234";
final String query1 = String.format("objidentifier:\"%s\" OR resultdupid:\"%s\"", id, id);
final String query2 = String.format("objidentifier:%s OR resultdupid:%s", ClientUtils.escapeQueryChars(id), ClientUtils.escapeQueryChars(id));
System.out.println(query1);
System.out.println(query2);
}
}