Merge remote-tracking branch 'origin/stable_ids' into stable_ids

dhp-common/src/main/java/eu/dnetlib/dhp/common/Constants.java

@@ -42,6 +42,8 @@ public class Constants {
 	public static final String RETRY_DELAY = "retryDelay";
 	public static final String CONNECT_TIMEOUT = "connectTimeOut";
 	public static final String READ_TIMEOUT = "readTimeOut";
+	public static final String FROM_DATE_OVERRIDE = "fromDateOverride";
+	public static final String UNTIL_DATE_OVERRIDE = "untilDateOverride";
 
 	public static final String CONTENT_TOTALITEMS = "TotalItems";
 	public static final String CONTENT_INVALIDRECORDS = "InvalidRecords";

dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/CleaningFunctions.java

@@ -4,6 +4,7 @@ package eu.dnetlib.dhp.schema.oaf;
 import java.util.*;
 import java.util.function.Function;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.apache.commons.lang3.StringUtils;
 
@@ -22,6 +23,9 @@ public class CleaningFunctions {
 	public static final String CLEANING_REGEX = "(?:\\n|\\r|\\t)";
 
 	public static final Set<String> PID_BLACKLIST = new HashSet<>();
+	public static final String INVALID_AUTHOR_REGEX = ".*deactivated.*";
+	public static final String TITLE_FILTER_REGEX = "[.*test.*\\W\\d]";
+	public static final int TITLE_FILTER_RESIDUAL_LENGTH = 10;
 
 	static {
 		PID_BLACKLIST.add("none");
@@ -80,6 +84,36 @@ public class CleaningFunctions {
 		return value;
 	}
 
+	public static <T extends Oaf> boolean filter(T value) {
+		if (value instanceof Datasource) {
+			// nothing to evaluate here
+		} else if (value instanceof Project) {
+			// nothing to evaluate here
+		} else if (value instanceof Organization) {
+			// nothing to evaluate here
+		} else if (value instanceof Relation) {
+			// nothing to clean here
+		} else if (value instanceof Result) {
+
+			Result r = (Result) value;
+
+			if (Objects.nonNull(r.getTitle()) && r.getTitle().isEmpty()) {
+				return false;
+			}
+
+			if (value instanceof Publication) {
+
+			} else if (value instanceof eu.dnetlib.dhp.schema.oaf.Dataset) {
+
+			} else if (value instanceof OtherResearchProduct) {
+
+			} else if (value instanceof Software) {
+
+			}
+		}
+		return true;
+	}
+
 	public static <T extends Oaf> T cleanup(T value) {
 		if (value instanceof Datasource) {
 			// nothing to clean here
@@ -124,6 +158,12 @@ public class CleaningFunctions {
 							.stream()
 							.filter(Objects::nonNull)
 							.filter(sp -> StringUtils.isNotBlank(sp.getValue()))
+							.filter(
+								sp -> sp
+									.getValue()
+									.toLowerCase()
+									.replaceAll(TITLE_FILTER_REGEX, "")
+									.length() > TITLE_FILTER_RESIDUAL_LENGTH)
 							.map(CleaningFunctions::cleanValue)
 							.collect(Collectors.toList()));
 			}
@@ -199,16 +239,7 @@ public class CleaningFunctions {
 				}
 			}
 			if (Objects.nonNull(r.getAuthor())) {
-				boolean nullRank = r
+				final List<Author> authors = Lists.newArrayList();
-					.getAuthor()
-					.stream()
-					.anyMatch(a -> Objects.isNull(a.getRank()));
-				if (nullRank) {
-					int i = 1;
-					for (Author author : r.getAuthor()) {
-						author.setRank(i++);
-					}
-				}
 				for (Author a : r.getAuthor()) {
 					if (Objects.isNull(a.getPid())) {
 						a.setPid(Lists.newArrayList());
@@ -235,7 +266,26 @@ public class CleaningFunctions {
 									.stream()
 									.collect(Collectors.toList()));
 					}
+					if (StringUtils.isBlank(a.getFullname())) {
+						if (StringUtils.isNotBlank(a.getName()) && StringUtils.isNotBlank(a.getSurname())) {
+							a.setFullname(a.getSurname() + ", " + a.getName());
 						}
+					}
+					if (StringUtils.isNotBlank(a.getFullname()) && isValidAuthorName(a)) {
+						authors.add(a);
+					}
+				}
+
+				boolean nullRank = authors
+					.stream()
+					.anyMatch(a -> Objects.isNull(a.getRank()));
+				if (nullRank) {
+					int i = 1;
+					for (Author author : authors) {
+						author.setRank(i++);
+					}
+				}
+				r.setAuthor(authors);
 
 			}
 			if (value instanceof Publication) {
@@ -252,6 +302,15 @@ public class CleaningFunctions {
 		return value;
 	}
 
+	private static boolean isValidAuthorName(Author a) {
+		return !Stream
+			.of(a.getFullname(), a.getName(), a.getSurname())
+			.filter(s -> s != null && !s.isEmpty())
+			.collect(Collectors.joining(""))
+			.toLowerCase()
+			.matches(INVALID_AUTHOR_REGEX);
+	}
+
 	private static List<StructuredProperty> processPidCleaning(List<StructuredProperty> pids) {
 		return pids
 			.stream()

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/transformation/xslt/PersonCleaner.java

@@ -0,0 +1,206 @@
+
+package eu.dnetlib.dhp.transformation.xslt;
+
+import static eu.dnetlib.dhp.transformation.xslt.XSLTTransformationFunction.QNAME_BASE_URI;
+
+import java.io.Serializable;
+// import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.text.Normalizer;
+import java.util.List;
+import java.util.Set;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.hash.Hashing;
+
+import eu.dnetlib.dhp.transformation.xslt.utils.Capitalize;
+import eu.dnetlib.dhp.transformation.xslt.utils.DotAbbreviations;
+import net.sf.saxon.s9api.ExtensionFunction;
+import net.sf.saxon.s9api.ItemType;
+import net.sf.saxon.s9api.OccurrenceIndicator;
+import net.sf.saxon.s9api.QName;
+import net.sf.saxon.s9api.SaxonApiException;
+import net.sf.saxon.s9api.SequenceType;
+import net.sf.saxon.s9api.XdmValue;
+
+//import eu.dnetlib.pace.clustering.NGramUtils;
+//import eu.dnetlib.pace.util.Capitalise;
+//import eu.dnetlib.pace.util.DotAbbreviations;
+
+public class PersonCleaner implements ExtensionFunction, Serializable {
+	/**
+	 *
+	 */
+	private static final long serialVersionUID = 1L;
+	private List<String> firstname = Lists.newArrayList();
+	private List<String> surname = Lists.newArrayList();
+	private List<String> fullname = Lists.newArrayList();
+
+	private static Set<String> particles = null;
+
+	public PersonCleaner() {
+
+	}
+
+	public String normalize(String s) {
+		s = Normalizer.normalize(s, Normalizer.Form.NFD); // was NFD
+		s = s.replaceAll("\\(.+\\)", "");
+		s = s.replaceAll("\\[.+\\]", "");
+		s = s.replaceAll("\\{.+\\}", "");
+		s = s.replaceAll("\\s+-\\s+", "-");
+
+//              s = s.replaceAll("[\\W&&[^,-]]", " ");
+
+//              System.out.println("class Person: s: " + s);
+
+//              s = s.replaceAll("[\\p{InCombiningDiacriticalMarks}&&[^,-]]", " ");
+		s = s.replaceAll("[\\p{Punct}&&[^-,]]", " ");
+		s = s.replace("\\d", " ");
+		s = s.replace("\\n", " ");
+		s = s.replace("\\.", " ");
+		s = s.replaceAll("\\s+", " ");
+
+		if (s.contains(",")) {
+			// System.out.println("class Person: s: " + s);
+
+			String[] arr = s.split(",");
+			if (arr.length == 1) {
+
+				fullname = splitTerms(arr[0]);
+			} else if (arr.length > 1) {
+				surname = splitTerms(arr[0]);
+				firstname = splitTermsFirstName(arr[1]);
+//                              System.out.println("class Person: surname: " + surname);
+//                              System.out.println("class Person: firstname: " + firstname);
+
+				fullname.addAll(surname);
+				fullname.addAll(firstname);
+			}
+		} else {
+			fullname = splitTerms(s);
+
+			int lastInitialPosition = fullname.size();
+			boolean hasSurnameInUpperCase = false;
+
+			for (int i = 0; i < fullname.size(); i++) {
+				String term = fullname.get(i);
+				if (term.length() == 1) {
+					lastInitialPosition = i;
+				} else if (term.equals(term.toUpperCase())) {
+					hasSurnameInUpperCase = true;
+				}
+			}
+			if (lastInitialPosition < fullname.size() - 1) { // Case: Michele G. Artini
+				firstname = fullname.subList(0, lastInitialPosition + 1);
+				System.out.println("name: " + firstname);
+				surname = fullname.subList(lastInitialPosition + 1, fullname.size());
+			} else if (hasSurnameInUpperCase) { // Case: Michele ARTINI
+				for (String term : fullname) {
+					if (term.length() > 1 && term.equals(term.toUpperCase())) {
+						surname.add(term);
+					} else {
+						firstname.add(term);
+					}
+				}
+			} else if (lastInitialPosition == fullname.size()) {
+				surname = fullname.subList(lastInitialPosition - 1, fullname.size());
+				firstname = fullname.subList(0, lastInitialPosition - 1);
+			}
+
+		}
+		return null;
+	}
+
+	private List<String> splitTermsFirstName(String s) {
+		List<String> list = Lists.newArrayList();
+		for (String part : Splitter.on(" ").omitEmptyStrings().split(s)) {
+			if (s.trim().matches("\\p{Lu}{2,3}")) {
+				String[] parts = s.trim().split("(?=\\p{Lu})"); // (Unicode UpperCase)
+				for (String p : parts) {
+					if (p.length() > 0)
+						list.add(p);
+				}
+			} else {
+				list.add(part);
+			}
+
+		}
+		return list;
+	}
+
+	private List<String> splitTerms(String s) {
+		if (particles == null) {
+			// particles = NGramUtils.loadFromClasspath("/eu/dnetlib/pace/config/name_particles.txt");
+		}
+
+		List<String> list = Lists.newArrayList();
+		for (String part : Splitter.on(" ").omitEmptyStrings().split(s)) {
+			// if (!particles.contains(part.toLowerCase())) {
+			list.add(part);
+
+			// }
+		}
+		return list;
+	}
+
+	public List<String> getFirstname() {
+		return firstname;
+	}
+
+	public List<String> getSurname() {
+		return surname;
+	}
+
+	public List<String> getFullname() {
+		return fullname;
+	}
+
+	public String hash() {
+		return Hashing.murmur3_128().hashString(getNormalisedFullname(), StandardCharsets.UTF_8).toString();
+	}
+
+	public String getNormalisedFullname() {
+		return isAccurate() ? Joiner.on(" ").join(getSurname()) + ", " + Joiner.on(" ").join(getNameWithAbbreviations())
+			: Joiner.on(" ").join(fullname);
+		// return isAccurate() ?
+		// Joiner.on(" ").join(getCapitalSurname()) + ", " + Joiner.on(" ").join(getNameWithAbbreviations()) :
+		// Joiner.on(" ").join(fullname);
+	}
+
+	public List<String> getCapitalSurname() {
+		return Lists.newArrayList(Iterables.transform(surname, new Capitalize()));
+	}
+
+	public List<String> getNameWithAbbreviations() {
+		return Lists.newArrayList(Iterables.transform(firstname, new DotAbbreviations()));
+	}
+
+	public boolean isAccurate() {
+		return (firstname != null && surname != null && !firstname.isEmpty() && !surname.isEmpty());
+	}
+
+	@Override
+	public QName getName() {
+		return new QName(QNAME_BASE_URI + "/person", "person");
+	}
+
+	@Override
+	public SequenceType getResultType() {
+		return SequenceType.makeSequenceType(ItemType.STRING, OccurrenceIndicator.ZERO_OR_ONE);
+	}
+
+	@Override
+	public SequenceType[] getArgumentTypes() {
+		// TODO Auto-generated method stub
+		return null;
+	}
+
+	@Override
+	public XdmValue call(XdmValue[] arguments) throws SaxonApiException {
+		// TODO Auto-generated method stub
+		return null;
+	}
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/transformation/xslt/XSLTTransformationFunction.java

@@ -46,6 +46,7 @@ public class XSLTTransformationFunction implements MapFunction<MetadataRecord, M
 			Processor processor = new Processor(false);
 			processor.registerExtensionFunction(cleanFunction);
 			processor.registerExtensionFunction(new DateCleaner());
+			processor.registerExtensionFunction(new PersonCleaner());
 
 			final XsltCompiler comp = processor.newXsltCompiler();
 			XsltExecutable xslt = comp

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/transformation/xslt/utils/Capitalize.java

@@ -0,0 +1,14 @@
+
+package eu.dnetlib.dhp.transformation.xslt.utils;
+
+// import org.apache.commons.text.WordUtils;
+// import org.apache.commons.text.WordUtils;
+import com.google.common.base.Function;
+
+public class Capitalize implements Function<String, String> {
+
+	@Override
+	public String apply(String s) {
+		return org.apache.commons.lang3.text.WordUtils.capitalize(s.toLowerCase());
+	}
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/transformation/xslt/utils/DotAbbreviations.java

@@ -0,0 +1,12 @@
+
+package eu.dnetlib.dhp.transformation.xslt.utils;
+
+import com.google.common.base.Function;
+
+public class DotAbbreviations implements Function<String, String> {
+
+	@Override
+	public String apply(String s) {
+		return s.length() == 1 ? s + "." : s;
+	}
+}

dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/transformation/TransformationJobTest.java

@@ -92,15 +92,19 @@ public class TransformationJobTest extends AbstractVocabularyTest {
 	}
 
 	@Test
-	@DisplayName("Test Transform Inst.&Them.v4 record XML with xslt_cleaning_oaiOpenaire_datacite_ExchangeLandingpagePid")
+	@DisplayName("Test Transform record XML with xslt_cleaning_datarepo_datacite/oaiOpenAIRE")
-	public void testTransformITGv4() throws Exception {
+	public void testTransformMostlyUsedScript() throws Exception {
+
+		String xslTransformationScript = "";
+		xslTransformationScript = "/eu/dnetlib/dhp/transform/scripts/xslt_cleaning_datarepo_datacite.xsl";
+        xslTransformationScript = "/eu/dnetlib/dhp/transform/scripts/xslt_cleaning_oaiOpenaire_datacite_ExchangeLandingpagePid.xsl";
+
 
 		// We Set the input Record getting the XML from the classpath
 		final MetadataRecord mr = new MetadataRecord();
 		mr.setBody(IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/transform/input_itgv4.xml")));
 		// We Load the XSLT transformation Rule from the classpath
-		XSLTTransformationFunction tr = loadTransformationRule(
+		XSLTTransformationFunction tr = loadTransformationRule(xslTransformationScript);
-			"/eu/dnetlib/dhp/transform/scripts/xslt_cleaning_oaiOpenaire_datacite_ExchangeLandingpagePid.xsl");
 
 		MetadataRecord result = tr.call(mr);
 
@@ -110,15 +114,18 @@ public class TransformationJobTest extends AbstractVocabularyTest {
 	}
 
 	@Test
-	@DisplayName("Test Transform record XML with xslt_cleaning_datarepo_datacite")
+	@DisplayName("Test Transform record XML with xslt_cleaning_REST_OmicsDI")
-	public void testTransformMostlyUsedScript() throws Exception {
+	public void testTransformRestScript() throws Exception {
+
+		String xslTransformationScript = "";
+		xslTransformationScript = "/eu/dnetlib/dhp/transform/scripts/xslt_cleaning_REST_OmicsDI.xsl";
+
 
 		// We Set the input Record getting the XML from the classpath
 		final MetadataRecord mr = new MetadataRecord();
-		mr.setBody(IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/transform/input_itgv4.xml")));
+		mr.setBody(IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/transform/input_omicsdi.xml")));
 		// We Load the XSLT transformation Rule from the classpath
-		XSLTTransformationFunction tr = loadTransformationRule(
+		XSLTTransformationFunction tr = loadTransformationRule(xslTransformationScript);
-			"/eu/dnetlib/dhp/transform/scripts/xslt_cleaning_datarepo_datacite.xsl");
 
 		MetadataRecord result = tr.call(mr);
 

dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/transform/input_omicsdi.xml

@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<oai:record xmlns="http://namespace.openaire.eu/"
+  xmlns:dc="http://purl.org/dc/elements/1.1/"
+  xmlns:dri="http://www.driver-repository.eu/namespace/dri"
+  xmlns:oaf="http://namespace.openaire.eu/oaf"
+  xmlns:oai="http://www.openarchives.org/OAI/2.0/"
+  xmlns:prov="http://www.openarchives.org/OAI/2.0/provenance" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <oai:header>
+    <dri:objIdentifier>_____OmicsDI::0000337c02d1b51030675d69407655da</dri:objIdentifier>
+    <dri:recordIdentifier>PRJNA78295</dri:recordIdentifier>
+    <dri:dateOfCollection>2020-10-31T15:31:30.725Z</dri:dateOfCollection>
+    <oaf:datasourceprefix>_____OmicsDI</oaf:datasourceprefix>
+  </oai:header>
+  <oai:metadata>
+    <datasets xmlns="">
+      <connectionsCountScaled>0.235294117647059</connectionsCountScaled>
+      <reanalysisCount>0</reanalysisCount>
+      <keywords>null</keywords>
+      <citationsCountScaled>0.0</citationsCountScaled>
+      <viewsCount>0</viewsCount>
+      <description>Sedimentitalea nanhaiensis DSM 24252 Genome sequencing and assembly</description>
+      <downloadCountScaled>8.20101314054644E-5</downloadCountScaled>
+      <source>omics_ena_project</source>
+      <title>Sedimentitalea nanhaiensis DSM 24252</title>
+      <connectionsCount>14</connectionsCount>
+      <citationsCount>0</citationsCount>
+      <score>null</score>
+      <omicsType>Genomics</omicsType>
+      <reanalysisCountScaled>0.0</reanalysisCountScaled>
+      <organisms>
+        <acc>571166</acc>
+        <name>Sedimentitalea nanhaiensis DSM 24252</name>
+      </organisms>
+      <viewsCountScaled>0.0</viewsCountScaled>
+      <claimable>false</claimable>
+      <id>PRJNA78295</id>
+      <publicationDate>null</publicationDate>
+      <downloadCount>13</downloadCount>
+    </datasets>
+  </oai:metadata>
+  <about xmlns="">
+    <provenance xmlns="http://www.openarchives.org/OAI/2.0/provenance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/provenance http://www.openarchives.org/OAI/2.0/provenance.xsd">
+      <originDescription altered="true" harvestDate="2020-10-31T15:31:30.725Z">
+        <baseURL>https%3A%2F%2Fwww.omicsdi.org%2Fws%2Fdataset%2Fsearch</baseURL>
+        <identifier/>
+        <datestamp/>
+        <metadataNamespace/>
+      </originDescription>
+    </provenance>
+    <oaf:datainfo>
+      <oaf:inferred>false</oaf:inferred>
+      <oaf:deletedbyinference>false</oaf:deletedbyinference>
+      <oaf:trust>0.9</oaf:trust>
+      <oaf:inferenceprovenance/>
+      <oaf:provenanceaction classid="sysimport:crosswalk:datasetarchive"
+        classname="sysimport:crosswalk:datasetarchive"
+        schemeid="dnet:provenanceActions" schemename="dnet:provenanceActions"/>
+    </oaf:datainfo>
+  </about>
+</oai:record>

dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/transform/scripts/xslt_cleaning_REST_OmicsDI.xsl

@@ -0,0 +1,297 @@
+<!-- complete literature v4: xslt_cleaning_REST_OmicsDI ; transformation script production , 2021-03-17 -->
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+                version="2.0"
+                xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                xmlns:oaf="http://namespace.openaire.eu/oaf"
+                xmlns:dr="http://www.driver-repository.eu/namespace/dr"
+                xmlns:datacite="http://datacite.org/schema/kernel-4"
+                xmlns:dc="http://purl.org/dc/elements/1.1/"
+                xmlns:dri="http://www.driver-repository.eu/namespace/dri"
+                xmlns:oaire="http://namespace.openaire.eu/schema/oaire/"
+                xmlns:xs="http://www.w3.org/2001/XMLSchema"
+                xmlns:vocabulary="http://eu/dnetlib/transform/clean"
+                xmlns:dateCleaner="http://eu/dnetlib/transform/dateISO"
+                xmlns:personCleaner="http://eu/dnetlib/transform/person"
+                exclude-result-prefixes="xsl vocabulary dateCleaner personCleaner">
+
+  <xsl:param name="varOfficialName" />
+  <xsl:param name="varDsType" />
+  <xsl:param name="varDataSourceId" />
+  <xsl:param name="index" select="0"/>
+  <xsl:param name="transDate" select="current-dateTime()"/>
+
+<xsl:variable name="vCodes">
+    <codes>
+         <code source="arrayexpress-repository" id="re3data_____::r3d100010222"  name="ArrayExpress Archive of Functional Genomics Data"  sourceUrl="https://www.ebi.ac.uk/arrayexpress/" urlTemplate="https://www.ebi.ac.uk/arrayexpress/experiments/" />
+         <code source="atlas-experiments"       id="re3data_____::r3d100010223"  name="Expression Atlas Database"                         sourceUrl="http://www.ebi.ac.uk/gxa/home" urlTemplate="" />
+         <code source="biomodels"               id="re3data_____::r3d100010789"  name="BioModels Database"                                sourceUrl="https://www.ebi.ac.uk/biomodels-main/" urlTemplate="" />
+         <code source="dbgap"                   id="re3data_____::r3d100010788"  name="dbGaP (database of Genotypes and Phenotypes)"      sourceUrl="https://www.ncbi.nlm.nih.gov/gap" urlTemplate="" />
+         <code source="ega"                     id="re3data_____::r3d100011242"  name="EGA Database (European Genome-phenome Archive)"    sourceUrl="https://ega-archive.org" urlTemplate="" />
+         <code source="eva"                     id="re3data_____::r3d100011553"  name="EVA database (European Variation Archive)"         sourceUrl="https://www.ebi.ac.uk/eva/" urlTemplate="" />
+         <code source="geo"                     id="re3data_____::r3d100010283"  name="GEO (Gene Expression Omnibus)"                     sourceUrl="https://www.ncbi.nlm.nih.gov/geo/" urlTemplate="https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc="  />
+         <code source="gnps"                    id="omicsdi_____::gnps"          name="GNPS Database (Global Natural Products Social Molecular Networking)" sourceUrl="https://gnps.ucsd.edu/ProteoSAFe/static/gnps-splash2.jsp" urlTemplate="" />
+         <code source="gpmdb"                   id="re3data_____::r3d100010883"  name="GPMDB (Global Proteome Machine)"                   sourceUrl="http://gpmdb.thegpm.org/" urlTemplate="http://gpmdb.thegpm.org/~/dblist_gpmnum/gpmnum=" />
+         <code source="jpost"                   id="re3data_____::r3d100012349"  name="JPOST Repository (Japan ProteOme STandard Repository/Database)" sourceUrl="https://jpostdb.org/" urlTemplate="https://repository.jpostdb.org/entry/JPST000228" />
+         <code source="lincs"                   id="re3data_____::r3d100011833"  name="LINCS (Big Data to Knowledge / Library of Integrated Network-based Cellular Signatures)" sourceUrl="http://lincsportal.ccs.miami.edu/dcic-portal/" urlTemplate="http://lincsportal.ccs.miami.edu/datasets/#/view/" />
+         <code source="massive"                 id="omicsdi_____::massive"       name="MassIVE Database (Mass Spectrometry Interactive Virtual Environment)" sourceUrl="https://massive.ucsd.edu/ProteoSAFe/datasets.jsp" urlTemplate="" />
+         <code source="metabolights_dataset"    id="opendoar____::2970"          name="MetaboLights Database"                             sourceUrl="http://www.ebi.ac.uk/metabolights/" urlTemplate="" />
+         <code source="metabolome_express"      id="omicsdi_____::metabolome"    name="MetabolomeExpress"                                 sourceUrl="https://www.metabolome-express.org/" urlTemplate="https://www.metabolome-express.org/datasetview.php?datasetid=" />
+         <code source="metabolomics_workbench"  id="re3data_____::r3d100012314"  name="Metabolomics Workbench Database"                   sourceUrl="http://www.metabolomicsworkbench.org/" urlTemplate="http://www.metabolomicsworkbench.org/data/DRCCMetadata.php?StudyID=" />
+         <code source="NCBI"                    id="omicsdi_____::ncbi"          name="NCBI"                                              sourceUrl="https://www.ncbi.nlm.nih.gov/bioproject/" urlTemplate="https://www.ncbi.nlm.nih.gov/bioproject/" />
+         <code source="omics_ena_project"       id="re3data_____::r3d100010527"  name="ENA (European Nucleotide Archive)"                 sourceUrl="https://www.ebi.ac.uk/ena" urlTemplate="https://www.ebi.ac.uk/ena/data/view/" />
+         <code source="paxdb"                   id="omicsdi_____::paxdb"         name="PAXDB (protein abundance database)"                sourceUrl="http://pax-db.org/" urlTemplate="" />
+         <code source="peptide_atlas"           id="re3data_____::r3d100010889"  name="PeptideAtlas Database"                             sourceUrl="http://www.peptideatlas.org/" urlTemplate="" />
+         <code source="pride"                   id="re3data_____::r3d100010137"  name="PRIDE Database (PRoteomics IDEntifications)"       sourceUrl="http://www.ebi.ac.uk/pride/archive/" urlTemplate="http://www.ebi.ac.uk/pride/archive/projects/PXD008134" />
+    </codes>
+ </xsl:variable>
+
+<!--
+gnps, jpost, massive, metabolome_express, paxdb: no id/OpenAIRE-entry found
+ncbi: several OpenAIRE-entries found - is one the right?
+-->
+
+<xsl:key name="kCodeByName" match="code" use="string(@source)"/>
+
+  <xsl:template match="/">
+    <xsl:variable name="datasourcePrefix"
+             select="normalize-space(//oaf:datasourceprefix)" />
+    <xsl:call-template name="validRecord" />
+  </xsl:template>
+
+ <xsl:template name="terminate">
+  	<xsl:message terminate="yes">
+  	record is not compliant, transformation is interrupted.
+  	</xsl:message>
+  </xsl:template>
+
+  <xsl:template name="validRecord">
+    <record>
+     <xsl:apply-templates select="//*[local-name() = 'header']" />
+
+      <metadata>
+
+<!--
+  <xsl:apply-templates select="//*[local-name() = 'metadata']//*[local-name() = 'datasets']"/>
+-->
+
+<datacite:resource>
+
+<!-- OmicsDI does not state: languages, projects, 
+-->
+
+<!-- landing page -->
+<xsl:if test="//*[local-name() = 'datasourceprefix'][.='_____OmicsDI']">
+        <datacite:alternateIdentifiers>
+        <datacite:alternateIdentifier>
+                  <xsl:attribute name="alternateIdentifierType">
+                            <xsl:value-of select="'LandingPage'"/>
+                  </xsl:attribute>
+                 <xsl:choose>
+                            <xsl:when test="//*[local-name() = 'source'][. = ('gnps','massive','paxdb','peptide_atlas')]">
+                                      <xsl:value-of select="concat('https://www.omicsdi.org/#/dataset/', //*[local-name() = 'source'], '/', //*[local-name() = 'id'])"/>
+                            </xsl:when>
+                            <xsl:when test="//*[local-name() = 'source'][. = 'metabolome_express']">
+                                      <xsl:value-of select="concat(key('kCodeByName', string(//*[local-name()='source']), $vCodes)/@urlTemplate, substring-after(//*[local-name()='id'], 'MEX'))"/>
+                            </xsl:when>
+                            <xsl:otherwise>
+                                     <xsl:value-of select="concat(key('kCodeByName', string(//*[local-name()='source']), $vCodes)/@urlTemplate, //*[local-name()='id'])"/>
+                            </xsl:otherwise>
+                 </xsl:choose>
+        </datacite:alternateIdentifier>
+        <datacite:alternateIdentifier>
+                  <xsl:attribute name="alternateIdentifierType">
+                            <xsl:value-of select="'local'"/>
+                  </xsl:attribute>
+                  <xsl:value-of select="//*[local-name()='id']"/>
+        </datacite:alternateIdentifier>
+        </datacite:alternateIdentifiers>
+</xsl:if>
+
+<!-- identifier; ... -->
+<!-- URL hindered by _et_: https://www.omicsdi.org/ws/dataset/get_acc=E-MTAB-6546_et_database=arrayexpress-repository -->
+<xsl:if test="//*[local-name() = 'datasourceprefix'][.='_____OmicsDI']">
+        <datacite:identifier>
+                  <xsl:attribute name="identifierType">
+                            <xsl:value-of select="'URL'"/>
+                  </xsl:attribute>
+
+                  <xsl:value-of select="concat('https://www.omicsdi.org/dataset/', //*[local-name() = 'source'], '/', //*[local-name() = 'id'])"/>
+
+        </datacite:identifier>
+</xsl:if>
+
+<!-- title -->
+<xsl:if test="//*[local-name() = 'datasourceprefix'][.='_____OmicsDI']">
+<datacite:titles>
+        <datacite:title>
+                  <xsl:value-of select="//*[local-name() = 'title']"/>
+        </datacite:title>
+</datacite:titles>
+</xsl:if>
+
+<!-- no authors in OmicsDI -->
+<!--
+<xsl:call-template name="authors" />
+-->
+
+<!--
+<xsl:call-template name="relatedPaper" />
+-->
+
+<datacite:descriptions>
+        <datacite:description>
+                  <xsl:attribute name="descriptionType">
+                            <xsl:value-of select="'Abstract'"/>
+                  </xsl:attribute>
+                  <xsl:value-of select="//*[local-name() = 'description']"/>
+        </datacite:description>
+</datacite:descriptions>
+
+<!-- subject -->
+<datacite:subjects>
+<xsl:for-each select="distinct-values(//*[local-name()='omicsType'])">
+<datacite:subject>
+<xsl:value-of select="."/>
+</datacite:subject>
+</xsl:for-each>
+</datacite:subjects>
+
+</datacite:resource>
+
+                <xsl:choose>
+
+           <xsl:when test="//*[local-name() = 'datasourceprefix'][.='_____OmicsDI']">
+                      <xsl:variable name='varCobjCategory'
+                                  select="'0021'" />
+             <dr:CobjCategory>
+             <xsl:attribute name="type">
+                        <xsl:value-of select="vocabulary:clean($varCobjCategory, 'dnet:result_typologies')"/>
+             </xsl:attribute>
+               <xsl:value-of 
+                     select="$varCobjCategory" />
+             </dr:CobjCategory>
+           </xsl:when>
+
+           <xsl:otherwise>
+<!--
+               <xsl:call-template name="terminate"/>
+-->
+           </xsl:otherwise>
+         </xsl:choose>
+<!--
+     // review status: no review indications found so far
+-->
+<!--
+OMICSDI is ‘including both open and controlled data source’.
+-->
+              <oaf:accessrights>
+                 <xsl:text>UNKNOWN</xsl:text>
+              </oaf:accessrights>
+
+<xsl:if test="//*[local-name() = 'datasourceprefix'][.='NeuroVault__']">
+        <oaf:concept>
+                  <xsl:attribute name="id">
+                            <xsl:value-of select="'ni'"/>
+                  </xsl:attribute>
+        </oaf:concept>
+</xsl:if>
+
+         <oaf:hostedBy>
+            <xsl:attribute name="name">
+               <xsl:value-of select="key('kCodeByName', string(//*[local-name()='source']), $vCodes)/@name"/>
+            </xsl:attribute>
+            <xsl:attribute name="id">
+               <xsl:value-of select="key('kCodeByName', string(//*[local-name()='source']), $vCodes)/@id"/>
+            </xsl:attribute>
+         </oaf:hostedBy>
+         <oaf:collectedFrom>
+            <xsl:attribute name="name">
+               <xsl:value-of select="$varOfficialName"/>
+            </xsl:attribute>
+            <xsl:attribute name="id">
+               <xsl:value-of select="$varDataSourceId"/>
+            </xsl:attribute>
+         </oaf:collectedFrom>
+
+<!-- date -->
+<xsl:if test="//*[local-name() = 'datasourceprefix'][.='_____OmicsDI']">
+        <oaf:dateAccepted>
+                  <xsl:value-of select="replace(//*[local-name() = 'publicationDate'][not(.='null')],'(\d{4})(\d{2})(\d{2})','$1-$2-$3')"/>
+        </oaf:dateAccepted>
+</xsl:if>
+
+      </metadata>
+      <xsl:copy-of select="//*[local-name() = 'about']" />
+   </record>
+
+</xsl:template>
+
+<xsl:template match="node()|@*">
+     <xsl:copy>
+       <xsl:apply-templates select="node()|@*"/>
+     </xsl:copy>
+ </xsl:template>
+
+  <xsl:template match="//*[local-name() = 'metadata']//*[local-name() = 'datasets']">
+    <xsl:copy>
+       <xsl:apply-templates select="node()|@*"/>
+     </xsl:copy>
+  </xsl:template>
+
+                       <xsl:template match="//*[local-name() = 'header']">
+<xsl:copy>
+     <xsl:apply-templates  select="node()|@*"/>
+                             <xsl:element name="dr:dateOfTransformation">
+                                 <xsl:value-of select="$transDate"/>
+                             </xsl:element>
+  </xsl:copy>
+                        </xsl:template>
+<!-- 
+no authors findable in OmicsDI">
+-->
+<!-- 
+  <xsl:template match="//*[local-name() = 'authors']">
+-->
+  <xsl:template name="authors">
+<xsl:choose>
+<xsl:when test="not(//*[local-name() = 'authors'][string-length(normalize-space(.)) > 0 and not(. = 'null')])">
+             <xsl:call-template name="terminate" />
+</xsl:when>
+<xsl:otherwise>
+<xsl:for-each select="tokenize(//*[local-name() = 'authors'], '(, and |,| and )')">
+
+    <xsl:element name="datacite:creator">
+
+        <xsl:element name="datacite:creatorName">
+           <xsl:value-of select="personCleaner:normalize( .)"/>
+        </xsl:element>
+
+        <xsl:element name="datacite:givenName">
+           <xsl:value-of select="normalize-space(substring-after(personCleaner:normalize(.), ','))"/>
+        </xsl:element>
+<xsl:element name="datacite:familyName">
+           <xsl:value-of select="substring-before(personCleaner:normalize(.), ',')"/>
+        </xsl:element>
+
+    </xsl:element>
+</xsl:for-each>
+</xsl:otherwise>
+</xsl:choose>
+</xsl:template>
+
+<!--
+  <xsl:template match="//*[local-name() = 'DOI']">
+-->
+<xsl:template name="relatedPaper">
+    <xsl:element name="datacite:relatedIdentifier">
+ <xsl:attribute name="relatedIdentifierType">
+           <xsl:value-of select="'DOI'"/>
+        </xsl:attribute>
+<xsl:attribute name="relationType">
+           <xsl:value-of select="'isReferencedBy'"/>
+        </xsl:attribute>
+<xsl:value-of select="//*[local-name() = 'DOI']"/>
+</xsl:element>
+</xsl:template>
+
+</xsl:stylesheet>

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleanGraphSparkJob.java

@@ -90,6 +90,7 @@ public class CleanGraphSparkJob {
 			.map((MapFunction<T, T>) value -> fixVocabularyNames(value), Encoders.bean(clazz))
 			.map((MapFunction<T, T>) value -> OafCleaner.apply(value, mapping), Encoders.bean(clazz))
 			.map((MapFunction<T, T>) value -> cleanup(value), Encoders.bean(clazz))
+			.filter((FilterFunction<T>) value -> filter(value))
 			.write()
 			.mode(SaveMode.Overwrite)
 			.option("compression", "gzip")

dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningFunctionTest.java

@@ -67,6 +67,7 @@ public class CleaningFunctionTest {
 
 		assertNotNull(p_out.getPublisher());
 		assertNull(p_out.getPublisher().getValue());
+
 		assertEquals("und", p_out.getLanguage().getClassid());
 		assertEquals("Undetermined", p_out.getLanguage().getClassname());
 
@@ -120,6 +121,9 @@ public class CleaningFunctionTest {
 				.isPresent());
 
 		Publication p_cleaned = CleaningFunctions.cleanup(p_out);
+
+		assertEquals(1, p_cleaned.getTitle().size());
+
 		assertEquals("CLOSED", p_cleaned.getBestaccessright().getClassid());
 		assertNull(p_out.getPublisher());
 

dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/result.json

@@ -865,6 +865,28 @@
         "schemename": "dnet:dataCite_title"
       },
       "value": "Optical response of strained- and unstrained-silicon cold-electron bolometers"
+    },
+    {
+      "dataInfo": {
+        "deletedbyinference": false,
+        "inferenceprovenance": "",
+        "inferred": false,
+        "invisible": false,
+        "provenanceaction": {
+          "classid": "sysimport:crosswalk:datasetarchive",
+          "classname": "sysimport:crosswalk:datasetarchive",
+          "schemeid": "dnet:provenanceActions",
+          "schemename": "dnet:provenanceActions"
+        },
+        "trust": "0.9"
+      },
+      "qualifier": {
+        "classid": "main title",
+        "classname": "main title",
+        "schemeid": "dnet:dataCite_title",
+        "schemename": "dnet:dataCite_title"
+      },
+      "value": "test test 123 test"
     }
   ]
 }