D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

[graph provision] fixed serialization of the instancetypes

This commit is contained in:
Claudio Atzori 2024-06-28 14:21:12 +02:00
parent 023099a921
commit 157cc8be87
3 changed files with 27 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/StreamingInputDocumentFactory.java
View File

@ -213,7 +213,7 @@ public class StreamingInputDocumentFactory implements Serializable {
} }
writer.close(); writer.close();
indexDocument.addField(INDEX_RESULT, results.toString()); indexDocument.addField(INDEX_RESULT, results.toString());
//indexDocument.addField(INDEX_JSON_RESULT, json); // indexDocument.addField(INDEX_JSON_RESULT, json);
} finally { } finally {
outputFactory.remove(); outputFactory.remove();
eventFactory.remove(); eventFactory.remove();

21
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java
View File

@ -1315,7 +1315,7 @@ public class XmlRecordFactory implements Serializable {
instance instance
.getCollectedfrom() .getCollectedfrom()
.stream() .stream()
.filter(cf -> kvNotBlank(cf)) .filter(XmlRecordFactory::kvNotBlank)
.map(cf -> XmlSerializationUtils.mapKeyValue("collectedfrom", cf)) .map(cf -> XmlSerializationUtils.mapKeyValue("collectedfrom", cf))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -1326,7 +1326,7 @@ public class XmlRecordFactory implements Serializable {
instance instance
.getHostedby() .getHostedby()
.stream() .stream()
.filter(hb -> kvNotBlank(hb)) .filter(XmlRecordFactory::kvNotBlank)
.map(hb -> XmlSerializationUtils.mapKeyValue("hostedby", hb)) .map(hb -> XmlSerializationUtils.mapKeyValue("hostedby", hb))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -1336,7 +1336,7 @@ public class XmlRecordFactory implements Serializable {
instance instance
.getDateofacceptance() .getDateofacceptance()
.stream() .stream()
.filter(d -> isNotBlank(d)) .filter(StringUtils::isNotBlank)
.map(d -> XmlSerializationUtils.asXmlElement("dateofacceptance", d)) .map(d -> XmlSerializationUtils.asXmlElement("dateofacceptance", d))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -1346,7 +1346,7 @@ public class XmlRecordFactory implements Serializable {
instance instance
.getInstancetype() .getInstancetype()
.stream() .stream()
.filter(t -> !StringUtils.isNotBlank(t.getClassid())) .filter(t -> StringUtils.isNotBlank(t.getClassid()))
.map(t -> XmlSerializationUtils.mapQualifier("instancetype", t)) .map(t -> XmlSerializationUtils.mapQualifier("instancetype", t))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -1356,7 +1356,7 @@ public class XmlRecordFactory implements Serializable {
instance instance
.getDistributionlocation() .getDistributionlocation()
.stream() .stream()
.filter(d -> isNotBlank(d)) .filter(StringUtils::isNotBlank)
.map(d -> XmlSerializationUtils.asXmlElement("distributionlocation", d)) .map(d -> XmlSerializationUtils.asXmlElement("distributionlocation", d))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -1409,7 +1409,7 @@ public class XmlRecordFactory implements Serializable {
instance instance
.getLicense() .getLicense()
.stream() .stream()
.filter(d -> isNotBlank(d)) .filter(StringUtils::isNotBlank)
.map(d -> XmlSerializationUtils.asXmlElement("license", d)) .map(d -> XmlSerializationUtils.asXmlElement("license", d))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@ -1540,11 +1540,16 @@ public class XmlRecordFactory implements Serializable {
.min(new RefereedComparator()) .min(new RefereedComparator())
.orElse(XmlInstance.UNKNOWN_REVIEW_LEVEL)); .orElse(XmlInstance.UNKNOWN_REVIEW_LEVEL));
Map<String, Qualifier> instanceTypes = Maps.newHashMap();
instances.forEach(p -> { instances.forEach(p -> {
final Instance i = p.getRight(); final Instance i = p.getRight();
instance.getCollectedfrom().add(i.getCollectedfrom()); instance.getCollectedfrom().add(i.getCollectedfrom());
instance.getHostedby().add(i.getHostedby()); instance.getHostedby().add(i.getHostedby());
instance.getInstancetype().add(i.getInstancetype());
if (Optional.ofNullable(i.getInstancetype()).map(Qualifier::getClassid).isPresent()) {
instanceTypes.putIfAbsent(i.getInstancetype().getClassid(), i.getInstancetype());
}
instance instance
.setProcessingchargeamount( .setProcessingchargeamount(
Optional.ofNullable(i.getProcessingchargeamount()).map(apc -> apc.getValue()).orElse(null)); Optional.ofNullable(i.getProcessingchargeamount()).map(apc -> apc.getValue()).orElse(null));
@ -1571,6 +1576,8 @@ public class XmlRecordFactory implements Serializable {
.ifPresent(instance::setFulltext); .ifPresent(instance::setFulltext);
}); });
instance.getInstancetype().addAll(instanceTypes.values());
if (instance.getHostedby().size() > 1 if (instance.getHostedby().size() > 1
&& instance.getHostedby().stream().anyMatch(hb -> ModelConstants.UNKNOWN_REPOSITORY.equals(hb))) { && instance.getHostedby().stream().anyMatch(hb -> ModelConstants.UNKNOWN_REPOSITORY.equals(hb))) {
instance.getHostedby().remove(ModelConstants.UNKNOWN_REPOSITORY); instance.getHostedby().remove(ModelConstants.UNKNOWN_REPOSITORY);

17
dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java
View File

@ -5,6 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotNull;
import java.io.IOException; import java.io.IOException;
import java.io.StringReader;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.util.Arrays; import java.util.Arrays;
@ -16,6 +17,9 @@ import javax.xml.transform.TransformerException;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputDocument;
import org.dom4j.Document;
import org.dom4j.DocumentException;
import org.dom4j.io.SAXReader;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@ -34,7 +38,6 @@ import eu.dnetlib.dhp.utils.saxon.SaxonTransformerFactory;
/** /**
* This test can be used to produce a record that can be manually fed to Solr in XML format. * This test can be used to produce a record that can be manually fed to Solr in XML format.
*
* The input is a JoinedEntity, i.e. a json representation of an OpenAIRE entity that embeds all the linked entities. * The input is a JoinedEntity, i.e. a json representation of an OpenAIRE entity that embeds all the linked entities.
*/ */
public class IndexRecordTransformerTest { public class IndexRecordTransformerTest {
@ -54,7 +57,7 @@ public class IndexRecordTransformerTest {
} }
@Test @Test
public void testPublicationRecordTransformation() throws IOException, TransformerException { public void testPublicationRecordTransformation() throws IOException, TransformerException, DocumentException {
final XmlRecordFactory xmlRecordFactory = new XmlRecordFactory(contextMapper, false, final XmlRecordFactory xmlRecordFactory = new XmlRecordFactory(contextMapper, false,
PayloadConverterJob.schemaLocation); PayloadConverterJob.schemaLocation);
@ -71,11 +74,15 @@ public class IndexRecordTransformerTest {
new RelatedEntityWrapper(rel, new RelatedEntityWrapper(rel,
CreateRelatedEntitiesJob_phase1.asRelatedEntity(pj, Project.class)))); CreateRelatedEntitiesJob_phase1.asRelatedEntity(pj, Project.class))));
final String record = xmlRecordFactory.build(je); final String xmlRecord = xmlRecordFactory.build(je);
assertNotNull(record); assertNotNull(xmlRecord);
testRecordTransformation(record); Document doc = new SAXReader().read(new StringReader(xmlRecord));
assertEquals("Article", doc.valueOf("//children/instance/instancetype/@classname"));
testRecordTransformation(xmlRecord);
} }
@Test @Test