D-Net
/
dnet-directindex-application
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

solr record

This commit is contained in:
Michele Artini 2024-11-15 15:48:52 +01:00
parent 2028c658c4
commit 81da6cfd58
8 changed files with 58 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
TODO.txt
View File

@ -17,6 +17,7 @@
- Uso dei record all'interno del wf di provision
5) Generazione del record SOLR
- Usare classe eu.dnetlib.dhp.schema.solr.SolrRecord di dhp-schemas 9.0.0
- Profilo del MetadataFormat
- XML / JSON

84
pom.xml
View File

@ -155,7 +155,20 @@
<artifactId>cxf-rt-transports-http</artifactId>
<version>3.1.5</version>
</dependency>
<!-- DHP SCHEMAS -->
<dependency>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dhp-schemas</artifactId>
<version>${dhp-schemas-version}</version>
<exclusions>
<exclusion>
<groupId>io.swagger.core.v3</groupId>
<artifactId>swagger-annotations</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- SOLR -->
<dependency>
<groupId>org.apache.solr</groupId>
@ -279,13 +292,6 @@
<version>2.3.2</version>
</dependency>
<!-- Mail -->
<dependency>
<groupId>javax.mail</groupId>
<artifactId>mail</artifactId>
<version>1.4.7</version>
</dependency>
<!-- Swagger -->
<dependency>
<groupId>org.springdoc</groupId>
@ -293,67 +299,7 @@
<version>1.6.10</version>
</dependency>
<!-- Hadoop -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.6.0-cdh5.9.2</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- Bioschemas BMUSE -->
<dependency>
<groupId>hwu.elixir</groupId>
<artifactId>bmuse-core</artifactId>
<version>0.5.4</version>
</dependency>
<dependency>
<groupId>org.apache.any23</groupId>
<artifactId>apache-any23-core</artifactId>
<version>2.3</version>
</dependency>
<dependency>
<groupId>org.eclipse.rdf4j</groupId>
<artifactId>rdf4j-rio-rdfxml</artifactId>
<version>3.7.1</version>
</dependency>
<dependency>
<groupId>org.eclipse.rdf4j</groupId>
<artifactId>rdf4j-model</artifactId>
<version>3.7.1</version>
</dependency>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.13.1</version>
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
<artifactId>selenium-java</artifactId>
<version>3.141.59</version>
</dependency>
<dependency>
<groupId>commons-validator</groupId>
<artifactId>commons-validator</artifactId>
<version>1.6</version>
</dependency>
</dependencies>
</dependencyManagement>
@ -488,7 +434,7 @@
<maven.compiler.plugin.version>3.6.0</maven.compiler.plugin.version>
<java.version>17</java.version>
<cnr-rmi-api.version>2.6.1</cnr-rmi-api.version>
<dhp-schemas-version>2.14.0</dhp-schemas-version>
<dhp-schemas-version>9.0.0</dhp-schemas-version>
<apache.solr.version>9.7.0</apache.solr.version>
<mongodb.driver.version>3.4.2</mongodb.driver.version>
<prometheus.version>0.10.0</prometheus.version>

6
src/main/java/eu/dnetlib/app/directindex/controllers/LegacyApiController.java
View File

@ -18,7 +18,7 @@ import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.app.directindex.errors.DirectIndexApiException;
import eu.dnetlib.app.directindex.input.ResultEntry;
import eu.dnetlib.app.directindex.mapping.OafMapper;
import eu.dnetlib.app.directindex.mapping.SolrRecordMapper;
import eu.dnetlib.app.directindex.service.DirectIndexService;
@RestController("/api")
@ -31,7 +31,7 @@ public class LegacyApiController {
private DirectIndexService service;
@Autowired
private OafMapper oafMapper;
private SolrRecordMapper solrRecordMapper;
@PostMapping("/results/feedObject")
public String feedResult(@RequestBody final ResultEntry pub, @RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit)
@ -56,7 +56,7 @@ public class LegacyApiController {
@RequestParam(value = "collectedFromId", required = true) final String collectedFromId,
@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws DirectIndexApiException {
final String openaireId = oafMapper.calculateOpenaireId(originalId, collectedFromId);
final String openaireId = solrRecordMapper.calculateOpenaireId(originalId, collectedFromId);
service.prepareMetadataDeletion(openaireId);

12
src/main/java/eu/dnetlib/app/directindex/controllers/SwordApiController.java
View File

@ -2,7 +2,6 @@ package eu.dnetlib.app.directindex.controllers;
import java.io.IOException;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@ -29,12 +28,13 @@ import eu.dnetlib.app.directindex.errors.SwordError;
import eu.dnetlib.app.directindex.errors.SwordErrorType;
import eu.dnetlib.app.directindex.errors.SwordException;
import eu.dnetlib.app.directindex.input.ResultEntry;
import eu.dnetlib.app.directindex.mapping.OafMapper;
import eu.dnetlib.app.directindex.mapping.SolrRecordMapper;
import eu.dnetlib.app.directindex.service.DirectIndexService;
import eu.dnetlib.app.directindex.solr.SolrIndexClient;
import eu.dnetlib.app.directindex.solr.SolrIndexClientFactory;
import eu.dnetlib.app.directindex.sword.model.SwordServiceDocument;
import eu.dnetlib.app.directindex.sword.model.SwordStatusDocument;
import eu.dnetlib.dhp.schema.solr.SolrRecord;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
@ -49,7 +49,7 @@ public class SwordApiController {
private SolrIndexClientFactory solrIndexClientFactory;
@Autowired
private OafMapper oafMapper;
private SolrRecordMapper solRecordMapper;
@Value("${dnet.directindex.baseurl}")
private String baseUrl;
@ -105,11 +105,11 @@ public class SwordApiController {
final SolrIndexClient solr = solrIndexClientFactory.getClient();
final String metadata = solr.findRecord(id);
final SolrRecord metadata = solr.findRecord(id);
if (StringUtils.isBlank(metadata)) { throw new SwordException(SwordErrorType.NotFound); }
if (metadata == null) { throw new SwordException(SwordErrorType.NotFound); }
return new ResponseEntity<>(oafMapper.toResultEntry(metadata), HttpStatus.OK);
return new ResponseEntity<>(solRecordMapper.toResultEntry(metadata), HttpStatus.OK);
}
@PutMapping("/objects/{id}/metadata")

25
src/main/java/eu/dnetlib/app/directindex/mapping/OafMapper.java → src/main/java/eu/dnetlib/app/directindex/mapping/SolrRecordMapper.java
View File

@ -8,20 +8,37 @@ import eu.dnetlib.app.directindex.clients.DatasourceManagerClient;
import eu.dnetlib.app.directindex.errors.DirectIndexApiException;
import eu.dnetlib.app.directindex.input.DatasourceEntry;
import eu.dnetlib.app.directindex.input.ResultEntry;
import eu.dnetlib.dhp.schema.solr.SolrRecord;
@Component
public class OafMapper {
public class SolrRecordMapper {
@Autowired
private DatasourceManagerClient dsmClient;
public String toOAF(final ResultEntry result) {
public SolrRecord toSolrRecord(final ResultEntry result) {
// TODO
return null;
final SolrRecord sr = new SolrRecord();
sr.setHeader(null);
sr.setResult(null);
sr.setDatasource(null); // NULL for Result Entry
sr.setProject(null); // NULL for Result Entry
sr.setOrganization(null); // NULL for Result Entry
sr.setPerson(null); // NULL for Result Entry
sr.setLinks(null);
sr.setCollectedfrom(null);
sr.setPid(null);
sr.setContext(null);
sr.setMeasures(null);
return sr;
}
public ResultEntry toResultEntry(final String oaf) {
public ResultEntry toResultEntry(final SolrRecord r) {
// TODO Auto-generated method stub
return null;
}

23
src/main/java/eu/dnetlib/app/directindex/solr/SolrIndexClient.java
View File

@ -1,9 +1,7 @@
package eu.dnetlib.app.directindex.solr;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.Iterator;
import java.util.stream.Stream;
@ -14,6 +12,7 @@ import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.common.SolrInputDocument;
import eu.dnetlib.app.directindex.errors.DirectIndexApiException;
import eu.dnetlib.dhp.schema.solr.SolrRecord;
public class SolrIndexClient {
@ -42,7 +41,7 @@ public class SolrIndexClient {
}
}
public void addRecords(final Stream<String> records) throws DirectIndexApiException {
public void addRecords(final Stream<SolrRecord> records) throws DirectIndexApiException {
try {
final Iterator<SolrInputDocument> iter = records
.map(this::prepareSolrDocument)
@ -54,24 +53,18 @@ public class SolrIndexClient {
}
}
public void addRecords(final String... xmlRecords) throws Exception {
addRecords(Arrays.stream(xmlRecords));
public void addRecords(final SolrRecord... records) throws Exception {
addRecords(Arrays.stream(records));
}
protected SolrInputDocument prepareSolrDocument(final String record) {
protected SolrInputDocument prepareSolrDocument(final SolrRecord record) {
// TODO (usare classe ccondivisa preparata da Claudio)
final String version = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss'Z'").format(new Date());
final StreamingInputDocumentFactory documentFactory = new StreamingInputDocumentFactory(version);
final String indexRecord = SolrRecordMappper.toSolr(record);
if (log.isDebugEnabled()) {
log.debug("***************************************\nSubmitting index record:\n" + indexRecord +
"\n***************************************\n");
}
return documentFactory.parseDocument(indexRecord);
return null;
}
public String findRecord(final String id) {
public SolrRecord findRecord(final String id) {
// TODO Auto-generated method stub
return null;
}

10
src/main/java/eu/dnetlib/app/directindex/solr/SolrRecordMappper.java
View File

@ -1,10 +0,0 @@
package eu.dnetlib.app.directindex.solr;
public class SolrRecordMappper {
public static String toSolr(final String record) {
// TODO Auto-generated method stub
return null;
}
}

9
src/main/java/eu/dnetlib/app/directindex/tasks/ScheduledActions.java
View File

@ -4,7 +4,6 @@ import java.time.OffsetDateTime;
import java.util.List;
import java.util.Objects;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
@ -16,7 +15,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.app.directindex.errors.DirectIndexApiException;
import eu.dnetlib.app.directindex.input.ResultEntry;
import eu.dnetlib.app.directindex.mapping.OafMapper;
import eu.dnetlib.app.directindex.mapping.SolrRecordMapper;
import eu.dnetlib.app.directindex.repo.PendingAction;
import eu.dnetlib.app.directindex.repo.PendingActionRepository;
import eu.dnetlib.app.directindex.solr.SolrIndexClient;
@ -34,7 +33,7 @@ public class ScheduledActions {
private SolrIndexClientFactory solrIndexClientFactory;
@Autowired
private OafMapper oafMapper;
private SolrRecordMapper solrRecordMapper;
@Autowired
private PendingActionRepository pendingActionRepository;
@ -66,8 +65,8 @@ public class ScheduledActions {
}
})
.filter(Objects::nonNull)
.map(oafMapper::toOAF)
.filter(StringUtils::isNotBlank));
.map(solrRecordMapper::toSolrRecord)
.filter(Objects::nonNull));
solr.commit();