D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

merging with branch beta

This commit is contained in:
Miriam Baglioni 2024-08-12 18:03:10 +02:00
parent 5a7ba77271 8c185a7b1a
commit 45605f93ae
7 changed files with 48 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java
View File

@ -129,7 +129,8 @@ public class PrepareAffiliationRelations implements Serializable {
Dataset<Row> df = spark
.read()
.schema("`DOI` STRING, `Matchings` ARRAY<STRUCT<`RORid`:STRING,`Confidence`:DOUBLE>>")
.json(inputPath);
.json(inputPath)
.where("DOI is not null");
// unroll nested arrays
df = df

6
dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/enrich/orcid/ORCIDAuthorMatchersTest.scala
View File

@ -31,5 +31,11 @@ class ORCIDAuthorMatchersTest {
assertTrue(matchOrderedTokenAndAbbreviations("孙林 Sun Lin", "Sun Lin"))
// assertTrue(AuthorsMatchRevised.compare("孙林 Sun Lin", "孙林")); // not yet implemented
}
@Test def testDocumentationNames(): Unit = {
assertTrue(matchOrderedTokenAndAbbreviations("James C. A. Miller-Jones", "James Antony Miller-Jones"))
}
@Test def testDocumentationNames2(): Unit = {
assertTrue(matchOrderedTokenAndAbbreviations("James C. A. Miller-Jones", "James Antony Miller Jones"))
}
}

11
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/ProvisionModelSupport.java
View File

@ -698,6 +698,7 @@ public class ProvisionModelSupport {
.stream()
.filter(s -> Objects.nonNull(s.getQualifier()))
.filter(s -> Objects.nonNull(s.getQualifier().getClassname()))
.filter(ProvisionModelSupport::filterFosL1L2)
.map(
s -> Subject
.newInstance(s.getValue(), s.getQualifier().getClassid(), s.getQualifier().getClassname()))
@ -720,6 +721,16 @@ public class ProvisionModelSupport {
.orElse(null);
}
public static boolean filterFosL1L2(StructuredProperty s) {
final String subjectType = Optional.ofNullable(s.getQualifier()).map(Qualifier::getClassid).orElse("");
if (ModelConstants.DNET_SUBJECT_FOS_CLASSID.equals(subjectType)) {
String code = StringUtils.substringBefore(s.getValue(), " ");
return code.matches("^\\d{2}$|^\\d{4}$");
}
return true;
}
private static Country asCountry(eu.dnetlib.dhp.schema.oaf.Qualifier country) {
return Optional
.ofNullable(country)

6
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java
View File

@ -20,6 +20,7 @@ import javax.xml.transform.*;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import eu.dnetlib.dhp.oa.provision.model.*;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
@ -41,10 +42,6 @@ import com.google.common.collect.Sets;
import com.mycila.xmltool.XMLDoc;
import com.mycila.xmltool.XMLTag;
import eu.dnetlib.dhp.oa.provision.model.JoinedEntity;
import eu.dnetlib.dhp.oa.provision.model.RelatedEntity;
import eu.dnetlib.dhp.oa.provision.model.RelatedEntityWrapper;
import eu.dnetlib.dhp.oa.provision.model.XmlInstance;
import eu.dnetlib.dhp.schema.common.*;
import eu.dnetlib.dhp.schema.oaf.*;
import eu.dnetlib.dhp.schema.oaf.Result;
@ -389,6 +386,7 @@ public class XmlRecordFactory implements Serializable {
.getSubject()
.stream()
.filter(Objects::nonNull)
.filter(ProvisionModelSupport::filterFosL1L2)
.map(s -> XmlSerializationUtils.mapStructuredProperty("subject", s))
.collect(Collectors.toList()));
}

4
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlSerializationUtils.java
View File

@ -5,11 +5,7 @@ import static eu.dnetlib.dhp.oa.provision.utils.GraphMappingUtils.removePrefix;
import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;

2
dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java
View File

@ -97,6 +97,8 @@ public class XmlRecordFactoryTest {
assertEquals("bronze", doc.valueOf("//*[local-name() = 'result']/openaccesscolor/text()"));
assertEquals("true", doc.valueOf("//*[local-name() = 'result']/isindiamondjournal/text()"));
assertEquals("true", doc.valueOf("//*[local-name() = 'result']/publiclyfunded/text()"));
assertEquals(15, doc.selectNodes("//*[local-name() = 'result']/*[local-name() = 'subject']").size());
}
@Test

28
dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/publication.json
View File

@ -1886,12 +1886,34 @@
"trust": ""
},
"qualifier": {
"classid": "keyword",
"classname": "keyword",
"classid": "FOS",
"classname": "Fields of Science and Technology classification",
"schemeid": "dnet:subject_classification_typologies",
"schemename": "dnet:subject_classification_typologies"
},
"value": "Thermal conductivity"
"value": "0101 mathematics"
},
{
"dataInfo": {
"deletedbyinference": false,
"inferenceprovenance": "",
"inferred": false,
"invisible": false,
"provenanceaction": {
"classid": "",
"classname": "",
"schemeid": "",
"schemename": ""
},
"trust": ""
},
"qualifier": {
"classid": "FOS",
"classname": "Fields of Science and Technology classification",
"schemeid": "dnet:subject_classification_typologies",
"schemename": "dnet:subject_classification_typologies"
},
"value": "010101 applied mathematics"
}
],
"title": [