Merge pull request 'XML serialisation of instances with the same URLs' (#167) from instance_group_by_url into beta

Reviewed-on: D-Net/dnet-hadoop#167

dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/XmlInstance.java

@@ -0,0 +1,158 @@
+
+package eu.dnetlib.dhp.oa.provision.model;
+
+import java.util.Set;
+
+import com.google.common.collect.Sets;
+
+import eu.dnetlib.dhp.schema.common.ModelConstants;
+import eu.dnetlib.dhp.schema.oaf.*;
+import scala.Serializable;
+
+public class XmlInstance implements Serializable {
+
+	public static final AccessRight UNKNOWN_ACCESS_RIGHT;
+
+	static {
+		UNKNOWN_ACCESS_RIGHT = new AccessRight();
+		UNKNOWN_ACCESS_RIGHT.setClassid(ModelConstants.UNKNOWN);
+		UNKNOWN_ACCESS_RIGHT.setClassname(ModelConstants.UNKNOWN);
+		UNKNOWN_ACCESS_RIGHT.setSchemeid(ModelConstants.DNET_ACCESS_MODES);
+		UNKNOWN_ACCESS_RIGHT.setSchemename(ModelConstants.DNET_ACCESS_MODES);
+	}
+
+	private String url;
+
+	private AccessRight accessright;
+
+	private Set<KeyValue> collectedfrom = Sets.newHashSet();
+
+	private Set<KeyValue> hostedby = Sets.newHashSet();
+
+	private Set<Qualifier> instancetype = Sets.newHashSet();
+
+	private Set<String> license = Sets.newHashSet();
+
+	// other research products specifc
+	private Set<String> distributionlocation = Sets.newHashSet();
+
+	private Set<StructuredProperty> pid = Sets.newHashSet();
+
+	private Set<StructuredProperty> alternateIdentifier = Sets.newHashSet();
+
+	private Set<String> dateofacceptance = Sets.newHashSet();
+
+	// ( article | book ) processing charges. Defined here to cope with possible wrongly typed
+	// results
+	private String processingchargeamount;
+
+	// currency - alphabetic code describe in ISO-4217. Defined here to cope with possible wrongly
+	// typed results
+	private String processingchargecurrency;
+
+	private Set<Qualifier> refereed = Sets.newHashSet();; // peer-review status
+
+	public String getUrl() {
+		return url;
+	}
+
+	public void setUrl(String url) {
+		this.url = url;
+	}
+
+	public AccessRight getAccessright() {
+		return accessright;
+	}
+
+	public void setAccessright(AccessRight accessright) {
+		this.accessright = accessright;
+	}
+
+	public Set<KeyValue> getCollectedfrom() {
+		return collectedfrom;
+	}
+
+	public void setCollectedfrom(Set<KeyValue> collectedfrom) {
+		this.collectedfrom = collectedfrom;
+	}
+
+	public Set<KeyValue> getHostedby() {
+		return hostedby;
+	}
+
+	public void setHostedby(Set<KeyValue> hostedby) {
+		this.hostedby = hostedby;
+	}
+
+	public Set<Qualifier> getInstancetype() {
+		return instancetype;
+	}
+
+	public void setInstancetype(Set<Qualifier> instancetype) {
+		this.instancetype = instancetype;
+	}
+
+	public Set<String> getLicense() {
+		return license;
+	}
+
+	public void setLicense(Set<String> license) {
+		this.license = license;
+	}
+
+	public Set<String> getDistributionlocation() {
+		return distributionlocation;
+	}
+
+	public void setDistributionlocation(Set<String> distributionlocation) {
+		this.distributionlocation = distributionlocation;
+	}
+
+	public Set<StructuredProperty> getPid() {
+		return pid;
+	}
+
+	public void setPid(Set<StructuredProperty> pid) {
+		this.pid = pid;
+	}
+
+	public Set<StructuredProperty> getAlternateIdentifier() {
+		return alternateIdentifier;
+	}
+
+	public void setAlternateIdentifier(Set<StructuredProperty> alternateIdentifier) {
+		this.alternateIdentifier = alternateIdentifier;
+	}
+
+	public Set<String> getDateofacceptance() {
+		return dateofacceptance;
+	}
+
+	public void setDateofacceptance(Set<String> dateofacceptance) {
+		this.dateofacceptance = dateofacceptance;
+	}
+
+	public String getProcessingchargeamount() {
+		return processingchargeamount;
+	}
+
+	public void setProcessingchargeamount(String processingchargeamount) {
+		this.processingchargeamount = processingchargeamount;
+	}
+
+	public String getProcessingchargecurrency() {
+		return processingchargecurrency;
+	}
+
+	public void setProcessingchargecurrency(String processingchargecurrency) {
+		this.processingchargecurrency = processingchargecurrency;
+	}
+
+	public Set<Qualifier> getRefereed() {
+		return refereed;
+	}
+
+	public void setRefereed(Set<Qualifier> refereed) {
+		this.refereed = refereed;
+	}
+}

dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/TemplateFactory.java

@@ -8,11 +8,16 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
+import javax.swing.text.html.Option;
+
 import org.apache.commons.lang3.StringUtils;
 import org.stringtemplate.v4.ST;
 
+import com.google.common.collect.Lists;
+
 import eu.dnetlib.dhp.schema.oaf.DataInfo;
 import eu.dnetlib.dhp.schema.oaf.OafEntity;
 
@@ -94,13 +99,15 @@ public class TemplateFactory {
 	}
 
 	public String getInstance(
-		final String resultId, final List<String> instancemetadata, final List<String> webresources) {
+		final List<String> instancemetadata, final String url) {
 		return getTemplate(resources.getInstance())
-			.add("instanceId", escapeXml(removePrefix(resultId)))
 			.add("metadata", instancemetadata)
 			.add(
 				"webresources",
-				(webresources != null ? webresources : new ArrayList<String>())
+				Optional
+					.ofNullable(url)
+					.map(u -> Lists.newArrayList(url))
+					.orElse(Lists.newArrayList())
 					.stream()
 					.filter(StringUtils::isNotBlank)
 					.map(this::getWebResource)

dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java

@@ -10,6 +10,7 @@ import java.io.IOException;
 import java.io.Serializable;
 import java.io.StringReader;
 import java.io.StringWriter;
+import java.net.URL;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -17,7 +18,10 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collector;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import javax.xml.transform.OutputKeys;
 import javax.xml.transform.Transformer;
@@ -27,7 +31,11 @@ import javax.xml.transform.TransformerFactory;
 import javax.xml.transform.dom.DOMSource;
 import javax.xml.transform.stream.StreamResult;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.solr.common.util.URLUtil;
 import org.apache.spark.util.LongAccumulator;
 import org.dom4j.Document;
 import org.dom4j.DocumentException;
@@ -49,25 +57,9 @@ import com.mycila.xmltool.XMLTag;
 import eu.dnetlib.dhp.oa.provision.model.JoinedEntity;
 import eu.dnetlib.dhp.oa.provision.model.RelatedEntity;
 import eu.dnetlib.dhp.oa.provision.model.RelatedEntityWrapper;
-import eu.dnetlib.dhp.schema.common.EntityType;
+import eu.dnetlib.dhp.oa.provision.model.XmlInstance;
-import eu.dnetlib.dhp.schema.common.MainEntityType;
+import eu.dnetlib.dhp.schema.common.*;
-import eu.dnetlib.dhp.schema.common.ModelConstants;
+import eu.dnetlib.dhp.schema.oaf.*;
-import eu.dnetlib.dhp.schema.common.ModelSupport;
-import eu.dnetlib.dhp.schema.oaf.Dataset;
-import eu.dnetlib.dhp.schema.oaf.Datasource;
-import eu.dnetlib.dhp.schema.oaf.ExternalReference;
-import eu.dnetlib.dhp.schema.oaf.ExtraInfo;
-import eu.dnetlib.dhp.schema.oaf.Instance;
-import eu.dnetlib.dhp.schema.oaf.Journal;
-import eu.dnetlib.dhp.schema.oaf.KeyValue;
-import eu.dnetlib.dhp.schema.oaf.OafEntity;
-import eu.dnetlib.dhp.schema.oaf.Organization;
-import eu.dnetlib.dhp.schema.oaf.OtherResearchProduct;
-import eu.dnetlib.dhp.schema.oaf.Project;
-import eu.dnetlib.dhp.schema.oaf.Publication;
-import eu.dnetlib.dhp.schema.oaf.Relation;
-import eu.dnetlib.dhp.schema.oaf.Result;
-import eu.dnetlib.dhp.schema.oaf.Software;
 import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory;
 
 public class XmlRecordFactory implements Serializable {
@@ -1112,37 +1104,64 @@ public class XmlRecordFactory implements Serializable {
 		if (MainEntityType.result.toString().equals(ModelSupport.getMainType(entityType))) {
 			final List<Instance> instances = ((Result) entity).getInstance();
 			if (instances != null) {
-				for (final Instance instance : ((Result) entity).getInstance()) {
 
+				groupInstancesByUrl(((Result) entity).getInstance()).forEach(instance -> {
 					final List<String> fields = Lists.newArrayList();
 
 					if (instance.getAccessright() != null && !instance.getAccessright().isBlank()) {
 						fields
 							.add(XmlSerializationUtils.mapQualifier("accessright", instance.getAccessright()));
 					}
-					if (instance.getCollectedfrom() != null && kvNotBlank(instance.getCollectedfrom())) {
+					if (instance.getCollectedfrom() != null) {
 						fields
-							.add(XmlSerializationUtils.mapKeyValue("collectedfrom", instance.getCollectedfrom()));
+							.addAll(
+								instance
+									.getCollectedfrom()
+									.stream()
+									.filter(cf -> kvNotBlank(cf))
+									.map(cf -> XmlSerializationUtils.mapKeyValue("collectedfrom", cf))
+									.collect(Collectors.toList()));
 					}
-					if (instance.getHostedby() != null && kvNotBlank(instance.getHostedby())) {
+
-						fields.add(XmlSerializationUtils.mapKeyValue("hostedby", instance.getHostedby()));
+					if (instance.getHostedby() != null) {
-					}
-					if (instance.getDateofacceptance() != null
-						&& isNotBlank(instance.getDateofacceptance().getValue())) {
 						fields
-							.add(
+							.addAll(
-								XmlSerializationUtils
+								instance
-									.asXmlElement("dateofacceptance", instance.getDateofacceptance().getValue()));
+									.getHostedby()
+									.stream()
+									.filter(hb -> kvNotBlank(hb))
+									.map(hb -> XmlSerializationUtils.mapKeyValue("hostedby", hb))
+									.collect(Collectors.toList()));
 					}
-					if (instance.getInstancetype() != null && !instance.getInstancetype().isBlank()) {
+					if (instance.getDateofacceptance() != null) {
 						fields
-							.add(XmlSerializationUtils.mapQualifier("instancetype", instance.getInstancetype()));
+							.addAll(
+								instance
+									.getDateofacceptance()
+									.stream()
+									.filter(d -> isNotBlank(d))
+									.map(d -> XmlSerializationUtils.asXmlElement("dateofacceptance", d))
+									.collect(Collectors.toList()));
 					}
-					if (isNotBlank(instance.getDistributionlocation())) {
+					if (instance.getInstancetype() != null) {
 						fields
-							.add(
+							.addAll(
-								XmlSerializationUtils
+								instance
-									.asXmlElement("distributionlocation", instance.getDistributionlocation()));
+									.getInstancetype()
+									.stream()
+									.filter(t -> !t.isBlank())
+									.map(t -> XmlSerializationUtils.mapQualifier("instancetype", t))
+									.collect(Collectors.toList()));
+					}
+					if (instance.getDistributionlocation() != null) {
+						fields
+							.addAll(
+								instance
+									.getDistributionlocation()
+									.stream()
+									.filter(d -> isNotBlank(d))
+									.map(d -> XmlSerializationUtils.asXmlElement("distributionlocation", d))
+									.collect(Collectors.toList()));
 					}
 					if (instance.getPid() != null) {
 						fields
@@ -1165,32 +1184,39 @@ public class XmlRecordFactory implements Serializable {
 									.collect(Collectors.toList()));
 					}
 
-					if (instance.getRefereed() != null && !instance.getRefereed().isBlank()) {
+					if (instance.getRefereed() != null) {
 						fields
-							.add(XmlSerializationUtils.mapQualifier("refereed", instance.getRefereed()));
+							.addAll(
+								instance
+									.getRefereed()
+									.stream()
+									.filter(Objects::nonNull)
+									.filter(r -> !r.isBlank())
+									.map(r -> XmlSerializationUtils.mapQualifier("refereed", r))
+									.collect(Collectors.toList()));
 					}
 					if (instance.getProcessingchargeamount() != null
-						&& isNotBlank(instance.getProcessingchargeamount().getValue())) {
+						&& isNotBlank(instance.getProcessingchargeamount())) {
 						fields
 							.add(
 								XmlSerializationUtils
 									.asXmlElement(
-										"processingchargeamount", instance.getProcessingchargeamount().getValue()));
+										"processingchargeamount", instance.getProcessingchargeamount()));
 					}
 					if (instance.getProcessingchargecurrency() != null
-						&& isNotBlank(instance.getProcessingchargecurrency().getValue())) {
+						&& isNotBlank(instance.getProcessingchargecurrency())) {
 						fields
 							.add(
 								XmlSerializationUtils
 									.asXmlElement(
-										"processingchargecurrency", instance.getProcessingchargecurrency().getValue()));
+										"processingchargecurrency", instance.getProcessingchargecurrency()));
 					}
 
 					children
 						.add(
 							templateFactory
-								.getInstance(instance.getHostedby().getKey(), fields, instance.getUrl()));
+								.getInstance(fields, instance.getUrl()));
-				}
+				});
 			}
 			final List<ExternalReference> ext = ((Result) entity).getExternalReference();
 			if (ext != null) {
@@ -1234,6 +1260,85 @@ public class XmlRecordFactory implements Serializable {
 		return children;
 	}
 
+	private Stream<XmlInstance> groupInstancesByUrl(List<Instance> instance) {
+		return instance
+			.stream()
+			.map(i -> {
+				i
+					.setUrl(
+						i
+							.getUrl()
+							.stream()
+							.filter(this::isValidUrl)
+							.collect(Collectors.toList()));
+				return i;
+			})
+			.filter(
+				i -> Optional
+					.ofNullable(i.getUrl())
+					.map(u -> !u.isEmpty())
+					.orElse(false))
+			.map(this::pickByUrl)
+			.collect(Collectors.groupingBy(ImmutablePair::getLeft))
+			.values()
+			.stream()
+			.map(this::mergeInstances);
+	}
+
+	private boolean isValidUrl(String url) {
+		try {
+			new URL(url).toURI();
+			return true;
+		} catch (Exception e) {
+			return false;
+		}
+	}
+
+	private ImmutablePair<String, Instance> pickByUrl(Instance i) {
+		return new ImmutablePair<>(i.getUrl().get(0), i);
+	}
+
+	private XmlInstance mergeInstances(List<ImmutablePair<String, Instance>> instances) {
+
+		final XmlInstance instance = new XmlInstance();
+
+		instance.setUrl(instances.get(0).getLeft());
+		instance
+			.setAccessright(
+				instances
+					.stream()
+					.map(Pair::getValue)
+					.map(Instance::getAccessright)
+					.min(new AccessRightComparator<AccessRight>())
+					.orElse(XmlInstance.UNKNOWN_ACCESS_RIGHT));
+
+		instances.forEach(p -> {
+			final Instance i = p.getRight();
+			instance.getCollectedfrom().add(i.getCollectedfrom());
+			instance.getHostedby().add(i.getHostedby());
+			instance.getInstancetype().add(i.getInstancetype());
+			instance.getLicense().add(i.getLicense().getValue());
+			instance.getDistributionlocation().add(i.getDistributionlocation());
+			instance.getPid().addAll(i.getPid());
+			instance.getAlternateIdentifier().addAll(i.getAlternateIdentifier());
+			instance.getDateofacceptance().add(i.getDateofacceptance().getValue());
+			instance
+				.setProcessingchargeamount(
+					Optional.ofNullable(i.getProcessingchargeamount()).map(apc -> apc.getValue()).orElse(null));
+			instance
+				.setProcessingchargecurrency(
+					Optional.ofNullable(i.getProcessingchargecurrency()).map(c -> c.getValue()).orElse(null));
+			instance.getRefereed().add(i.getRefereed());
+		});
+
+		if (instance.getHostedby().size() > 1
+			&& instance.getHostedby().stream().anyMatch(hb -> ModelConstants.UNKNOWN_REPOSITORY.equals(hb))) {
+			instance.getHostedby().remove(ModelConstants.UNKNOWN_REPOSITORY);
+		}
+
+		return instance;
+	}
+
 	private boolean isDuplicate(final RelatedEntityWrapper link) {
 		return ModelConstants.DEDUP.equalsIgnoreCase(link.getRelation().getSubRelType());
 	}

dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/template/instance.st

@@ -1,4 +1,4 @@
-<instance id="$instanceId$">
+<instance>
     $metadata:{ it | $it$ }$
 	$webresources:{ it | $it$ }$
 </instance>

dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java

@@ -12,7 +12,6 @@ import org.apache.commons.io.IOUtils;
 import org.dom4j.Document;
 import org.dom4j.DocumentException;
 import org.dom4j.io.SAXReader;
-import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
 import com.fasterxml.jackson.databind.DeserializationFeature;
@@ -54,18 +53,19 @@ public class XmlRecordFactoryTest {
 
 		System.out.println(doc.asXML());
 
-		Assertions.assertEquals("0000-0001-9613-6638", doc.valueOf("//creator[@rank = '1']/@orcid"));
+		assertEquals("0000-0001-9613-6638", doc.valueOf("//creator[@rank = '1']/@orcid"));
-		Assertions.assertEquals("0000-0001-9613-6639", doc.valueOf("//creator[@rank = '1']/@orcid_pending"));
+		assertEquals("0000-0001-9613-6639", doc.valueOf("//creator[@rank = '1']/@orcid_pending"));
 
-		Assertions.assertEquals("0000-0001-9613-9956", doc.valueOf("//creator[@rank = '2']/@orcid"));
+		assertEquals("0000-0001-9613-9956", doc.valueOf("//creator[@rank = '2']/@orcid"));
-		Assertions.assertEquals("", doc.valueOf("//creator[@rank = '2']/@orcid_pending"));
+		assertEquals("", doc.valueOf("//creator[@rank = '2']/@orcid_pending"));
 
-		Assertions.assertEquals("doi", doc.valueOf("//instance/pid/@classid"));
+		assertEquals("doi", doc.valueOf("//instance/pid/@classid"));
-		Assertions.assertEquals("10.1109/TED.2018.2853550", doc.valueOf("//instance/pid/text()"));
+		assertEquals("10.1109/TED.2018.2853550", doc.valueOf("//instance/pid/text()"));
 
-		Assertions.assertEquals("doi", doc.valueOf("//instance/alternateidentifier/@classid"));
+		assertEquals("doi", doc.valueOf("//instance/alternateidentifier/@classid"));
-		Assertions.assertEquals("10.5689/LIB.2018.2853550", doc.valueOf("//instance/alternateidentifier/text()"));
+		assertEquals("10.5689/LIB.2018.2853550", doc.valueOf("//instance/alternateidentifier/text()"));
-		// TODO add assertions based of values extracted from the XML record
+
+		assertEquals(3, doc.selectNodes("//instance").size());
 	}
 
 	@Test
@@ -96,7 +96,7 @@ public class XmlRecordFactoryTest {
 		final Document doc = new SAXReader().read(new StringReader(xml));
 		assertNotNull(doc);
 		System.out.println(doc.asXML());
-		Assertions.assertEquals("2021-01-01", doc.valueOf("//validated/@date"));
+		assertEquals("2021-01-01", doc.valueOf("//validated/@date"));
 	}
 
 	@Test

dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/publication.json

@@ -412,9 +412,800 @@
         "value": ""
       },
       "url": [
-        "http://juuli.fi/Record/0331473718",
         "http://dx.doi.org/10.1109/TED.2018.2853550"
       ]
+    },
+    {
+      "pid": [
+        {
+          "dataInfo": {
+            "deletedbyinference": false,
+            "inferenceprovenance": "",
+            "inferred": false,
+            "invisible": false,
+            "provenanceaction": {
+              "classid": "",
+              "classname": "",
+              "schemeid": "",
+              "schemename": ""
+            },
+            "trust": ""
+          },
+          "qualifier": {
+            "classid": "doi",
+            "classname": "doi",
+            "schemeid": "dnet:pid_types",
+            "schemename": "dnet:pid_types"
+          },
+          "value": "10.1109/TED.2018.2853550"
+        }
+      ],
+      "alternateIdentifier": [
+        {
+          "dataInfo": {
+            "deletedbyinference": false,
+            "inferenceprovenance": "",
+            "inferred": false,
+            "invisible": false,
+            "provenanceaction": {
+              "classid": "",
+              "classname": "",
+              "schemeid": "",
+              "schemename": ""
+            },
+            "trust": ""
+          },
+          "qualifier": {
+            "classid": "doi",
+            "classname": "doi",
+            "schemeid": "dnet:pid_types",
+            "schemename": "dnet:pid_types"
+          },
+          "value": "10.5689/LIB.2018.2853550"
+        }
+      ],
+      "accessright": {
+        "classid": "CLOSED",
+        "classname": "Closed Access",
+        "schemeid": "dnet:access_modes",
+        "schemename": "dnet:access_modes"
+      },
+      "collectedfrom": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "key": "10|od______3456::b2b9ce8435390bcbfc05f3cae3948567",
+        "value": "A wonderful repository"
+      },
+      "dateofacceptance": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "value": "2020-01-01"
+      },
+      "distributionlocation": "",
+      "hostedby": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "key": "10|openaire____::55045bd2a65019fd8e6741a755395c8c",
+        "value": "Unknown Repository"
+      },
+      "instancetype": {
+        "classid": "0001",
+        "classname": "Article",
+        "schemeid": "dnet:dataCite_resource",
+        "schemename": "dnet:dataCite_resource"
+      },
+      "license": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "value": ""
+      },
+      "url": [
+        "http://dx.doi.org/10.1109/TED.2018.2853550"
+      ]
+    },
+    {
+      "pid": [
+        {
+          "dataInfo": {
+            "deletedbyinference": false,
+            "inferenceprovenance": "",
+            "inferred": false,
+            "invisible": false,
+            "provenanceaction": {
+              "classid": "",
+              "classname": "",
+              "schemeid": "",
+              "schemename": ""
+            },
+            "trust": ""
+          },
+          "qualifier": {
+            "classid": "doi",
+            "classname": "doi",
+            "schemeid": "dnet:pid_types",
+            "schemename": "dnet:pid_types"
+          },
+          "value": "10.1109/TED.2018.2853550"
+        }
+      ],
+      "alternateIdentifier": [
+        {
+          "dataInfo": {
+            "deletedbyinference": false,
+            "inferenceprovenance": "",
+            "inferred": false,
+            "invisible": false,
+            "provenanceaction": {
+              "classid": "",
+              "classname": "",
+              "schemeid": "",
+              "schemename": ""
+            },
+            "trust": ""
+          },
+          "qualifier": {
+            "classid": "doi",
+            "classname": "doi",
+            "schemeid": "dnet:pid_types",
+            "schemename": "dnet:pid_types"
+          },
+          "value": "10.5689/LIB.2018.2853550"
+        }
+      ],
+      "accessright": {
+        "classid": "OPEN",
+        "classname": "Open Access",
+        "schemeid": "dnet:access_modes",
+        "schemename": "dnet:access_modes"
+      },
+      "collectedfrom": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "key": "10|od______2367::dfb9c4r4353ghjcbfbnhf3cyu79484rf",
+        "value": "Another repository"
+      },
+      "dateofacceptance": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "value": "2018-01-01"
+      },
+      "distributionlocation": "",
+      "hostedby": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "key": "10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747",
+        "value": "VIRTA"
+      },
+      "instancetype": {
+        "classid": "0001",
+        "classname": "Article",
+        "schemeid": "dnet:dataCite_resource",
+        "schemename": "dnet:dataCite_resource"
+      },
+      "license": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "value": ""
+      },
+      "url": [
+        "http://dx.doi.org/10.1109/TED.2018.2853551"
+      ]
+    },
+    {
+      "pid": [
+        {
+          "dataInfo": {
+            "deletedbyinference": false,
+            "inferenceprovenance": "",
+            "inferred": false,
+            "invisible": false,
+            "provenanceaction": {
+              "classid": "",
+              "classname": "",
+              "schemeid": "",
+              "schemename": ""
+            },
+            "trust": ""
+          },
+          "qualifier": {
+            "classid": "doi",
+            "classname": "doi",
+            "schemeid": "dnet:pid_types",
+            "schemename": "dnet:pid_types"
+          },
+          "value": "10.1109/TED.2018.2853550"
+        }
+      ],
+      "alternateIdentifier": [
+        {
+          "dataInfo": {
+            "deletedbyinference": false,
+            "inferenceprovenance": "",
+            "inferred": false,
+            "invisible": false,
+            "provenanceaction": {
+              "classid": "",
+              "classname": "",
+              "schemeid": "",
+              "schemename": ""
+            },
+            "trust": ""
+          },
+          "qualifier": {
+            "classid": "doi",
+            "classname": "doi",
+            "schemeid": "dnet:pid_types",
+            "schemename": "dnet:pid_types"
+          },
+          "value": "10.5689/LIB.2018.2853550"
+        }
+      ],
+      "accessright": {
+        "classid": "OPEN",
+        "classname": "Open Access",
+        "schemeid": "dnet:access_modes",
+        "schemename": "dnet:access_modes"
+      },
+      "collectedfrom": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "key": "10|od______2367::dfb9c4r4353ghjcbfbnhf3cyu79484rf",
+        "value": "Another repository"
+      },
+      "dateofacceptance": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "value": "2018-01-01"
+      },
+      "distributionlocation": "",
+      "hostedby": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "key": "10|openaire____::55045bd2a65019fd8e6741a755395c8c",
+        "value": "Unknown Repository"
+      },
+      "instancetype": {
+        "classid": "0001",
+        "classname": "Article",
+        "schemeid": "dnet:dataCite_resource",
+        "schemename": "dnet:dataCite_resource"
+      },
+      "license": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "value": ""
+      },
+      "url": [
+        "http://dx.doi.org/10.1109/TED.2018.2853552",
+        "http://dx.doi.org/10.1109/TED.2018.2853554"
+      ]
+    },
+    {
+      "pid": [
+        {
+          "dataInfo": {
+            "deletedbyinference": false,
+            "inferenceprovenance": "",
+            "inferred": false,
+            "invisible": false,
+            "provenanceaction": {
+              "classid": "",
+              "classname": "",
+              "schemeid": "",
+              "schemename": ""
+            },
+            "trust": ""
+          },
+          "qualifier": {
+            "classid": "doi",
+            "classname": "doi",
+            "schemeid": "dnet:pid_types",
+            "schemename": "dnet:pid_types"
+          },
+          "value": "10.1109/TED.2018.2853550"
+        }
+      ],
+      "alternateIdentifier": [
+        {
+          "dataInfo": {
+            "deletedbyinference": false,
+            "inferenceprovenance": "",
+            "inferred": false,
+            "invisible": false,
+            "provenanceaction": {
+              "classid": "",
+              "classname": "",
+              "schemeid": "",
+              "schemename": ""
+            },
+            "trust": ""
+          },
+          "qualifier": {
+            "classid": "doi",
+            "classname": "doi",
+            "schemeid": "dnet:pid_types",
+            "schemename": "dnet:pid_types"
+          },
+          "value": "10.5689/LIB.2018.2853550"
+        }
+      ],
+      "accessright": {
+        "classid": "OPEN",
+        "classname": "Open Access",
+        "schemeid": "dnet:access_modes",
+        "schemename": "dnet:access_modes"
+      },
+      "collectedfrom": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "key": "10|od______2367::dfb9c4r4353ghjcbfbnhf3cyu79484rf",
+        "value": "Another repository"
+      },
+      "dateofacceptance": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "value": "2018-01-01"
+      },
+      "distributionlocation": "",
+      "hostedby": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "key": "10|openaire____::55045bd2a65019fd8e6741a755395c8c",
+        "value": "Unknown Repository"
+      },
+      "instancetype": {
+        "classid": "0001",
+        "classname": "Article",
+        "schemeid": "dnet:dataCite_resource",
+        "schemename": "dnet:dataCite_resource"
+      },
+      "license": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "value": ""
+      },
+      "url": [
+      ]
+    },
+    {
+      "pid": [
+        {
+          "dataInfo": {
+            "deletedbyinference": false,
+            "inferenceprovenance": "",
+            "inferred": false,
+            "invisible": false,
+            "provenanceaction": {
+              "classid": "",
+              "classname": "",
+              "schemeid": "",
+              "schemename": ""
+            },
+            "trust": ""
+          },
+          "qualifier": {
+            "classid": "doi",
+            "classname": "doi",
+            "schemeid": "dnet:pid_types",
+            "schemename": "dnet:pid_types"
+          },
+          "value": "10.1109/TED.2018.2853550"
+        }
+      ],
+      "alternateIdentifier": [
+        {
+          "dataInfo": {
+            "deletedbyinference": false,
+            "inferenceprovenance": "",
+            "inferred": false,
+            "invisible": false,
+            "provenanceaction": {
+              "classid": "",
+              "classname": "",
+              "schemeid": "",
+              "schemename": ""
+            },
+            "trust": ""
+          },
+          "qualifier": {
+            "classid": "doi",
+            "classname": "doi",
+            "schemeid": "dnet:pid_types",
+            "schemename": "dnet:pid_types"
+          },
+          "value": "10.5689/LIB.2018.2853550"
+        }
+      ],
+      "accessright": {
+        "classid": "OPEN",
+        "classname": "Open Access",
+        "schemeid": "dnet:access_modes",
+        "schemename": "dnet:access_modes"
+      },
+      "collectedfrom": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "key": "10|od______2367::dfb9c4r4353ghjcbfbnhf3cyu79484rf",
+        "value": "Another repository"
+      },
+      "dateofacceptance": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "value": "2018-01-01"
+      },
+      "distributionlocation": "",
+      "hostedby": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "key": "10|openaire____::55045bd2a65019fd8e6741a755395c8c",
+        "value": "Unknown Repository"
+      },
+      "instancetype": {
+        "classid": "0001",
+        "classname": "Article",
+        "schemeid": "dnet:dataCite_resource",
+        "schemename": "dnet:dataCite_resource"
+      },
+      "license": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "value": ""
+      },
+      "url": [
+        ""
+      ]
+    },
+    {
+      "pid": [
+        {
+          "dataInfo": {
+            "deletedbyinference": false,
+            "inferenceprovenance": "",
+            "inferred": false,
+            "invisible": false,
+            "provenanceaction": {
+              "classid": "",
+              "classname": "",
+              "schemeid": "",
+              "schemename": ""
+            },
+            "trust": ""
+          },
+          "qualifier": {
+            "classid": "doi",
+            "classname": "doi",
+            "schemeid": "dnet:pid_types",
+            "schemename": "dnet:pid_types"
+          },
+          "value": "10.1109/TED.2018.2853550"
+        }
+      ],
+      "alternateIdentifier": [
+        {
+          "dataInfo": {
+            "deletedbyinference": false,
+            "inferenceprovenance": "",
+            "inferred": false,
+            "invisible": false,
+            "provenanceaction": {
+              "classid": "",
+              "classname": "",
+              "schemeid": "",
+              "schemename": ""
+            },
+            "trust": ""
+          },
+          "qualifier": {
+            "classid": "doi",
+            "classname": "doi",
+            "schemeid": "dnet:pid_types",
+            "schemename": "dnet:pid_types"
+          },
+          "value": "10.5689/LIB.2018.2853550"
+        }
+      ],
+      "accessright": {
+        "classid": "OPEN",
+        "classname": "Open Access",
+        "schemeid": "dnet:access_modes",
+        "schemename": "dnet:access_modes"
+      },
+      "collectedfrom": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "key": "10|od______2367::dfb9c4r4353ghjcbfbnhf3cyu79484rf",
+        "value": "Another repository"
+      },
+      "dateofacceptance": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "value": "2018-01-01"
+      },
+      "distributionlocation": "",
+      "hostedby": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "key": "10|openaire____::55045bd2a65019fd8e6741a755395c8c",
+        "value": "Unknown Repository"
+      },
+      "instancetype": {
+        "classid": "0001",
+        "classname": "Article",
+        "schemeid": "dnet:dataCite_resource",
+        "schemename": "dnet:dataCite_resource"
+      },
+      "license": {
+        "dataInfo": {
+          "deletedbyinference": false,
+          "inferenceprovenance": "",
+          "inferred": false,
+          "invisible": false,
+          "provenanceaction": {
+            "classid": "",
+            "classname": "",
+            "schemeid": "",
+            "schemename": ""
+          },
+          "trust": ""
+        },
+        "value": ""
+      },
+      "url": [
+        "asdasd://not a URL"
+      ]
     }
   ],
   "journal": {