Merge branch 'stable_ids' of code-repo.d4science.org:D-Net/dnet-hadoop into stable_ids

This commit is contained in:
Sandro La Bruzzo 2021-03-17 14:54:28 +01:00
commit 2be0428047
4 changed files with 23 additions and 24 deletions

View File

@ -56,7 +56,7 @@ public class IdentifierFactory implements Serializable {
} }
public static List<StructuredProperty> getPids(List<StructuredProperty> pid, KeyValue collectedFrom) { public static List<StructuredProperty> getPids(List<StructuredProperty> pid, KeyValue collectedFrom) {
return pidFromInstance(pid, collectedFrom).collect(Collectors.toList()); return pidFromInstance(pid, collectedFrom).distinct().collect(Collectors.toList());
} }
/** /**

View File

@ -118,6 +118,9 @@ public class ModelConstants {
public static final Qualifier UNKNOWN_COUNTRY = qualifier(UNKNOWN, "Unknown", DNET_COUNTRY_TYPE, DNET_COUNTRY_TYPE); public static final Qualifier UNKNOWN_COUNTRY = qualifier(UNKNOWN, "Unknown", DNET_COUNTRY_TYPE, DNET_COUNTRY_TYPE);
public static final Qualifier MAIN_TITLE_QUALIFIER = qualifier(
"main title", "main title", DNET_DATACITE_TITLE, DNET_DATACITE_TITLE);
private static Qualifier qualifier( private static Qualifier qualifier(
final String classid, final String classid,
final String classname, final String classname,

View File

@ -4,13 +4,8 @@ package eu.dnetlib.dhp.oa.graph.raw;
import static eu.dnetlib.dhp.schema.common.ModelConstants.*; import static eu.dnetlib.dhp.schema.common.ModelConstants.*;
import static eu.dnetlib.dhp.schema.oaf.OafMapperUtils.*; import static eu.dnetlib.dhp.schema.oaf.OafMapperUtils.*;
import java.util.ArrayList; import java.util.*;
import java.util.Arrays; import java.util.stream.Collectors;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.dom4j.Document; import org.dom4j.Document;
@ -21,7 +16,6 @@ import org.dom4j.Node;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup; import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup;
import eu.dnetlib.dhp.schema.common.AccessRightComparator;
import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.common.ModelConstants;
import eu.dnetlib.dhp.schema.oaf.*; import eu.dnetlib.dhp.schema.oaf.*;
import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory;
@ -57,9 +51,6 @@ public abstract class AbstractMdRecordToOafMapper {
nsContext.put("datacite", DATACITE_SCHEMA_KERNEL_3); nsContext.put("datacite", DATACITE_SCHEMA_KERNEL_3);
} }
protected static final Qualifier MAIN_TITLE_QUALIFIER = qualifier(
"main title", "main title", "dnet:dataCite_title", "dnet:dataCite_title");
protected AbstractMdRecordToOafMapper(final VocabularyGroup vocs, final boolean invisible, protected AbstractMdRecordToOafMapper(final VocabularyGroup vocs, final boolean invisible,
final boolean shouldHashId) { final boolean shouldHashId) {
this.vocs = vocs; this.vocs = vocs;
@ -279,11 +270,9 @@ public abstract class AbstractMdRecordToOafMapper {
r.setDataInfo(info); r.setDataInfo(info);
r.setLastupdatetimestamp(lastUpdateTimestamp); r.setLastupdatetimestamp(lastUpdateTimestamp);
r.setId(createOpenaireId(50, doc.valueOf("//dri:objIdentifier"), false)); r.setId(createOpenaireId(50, doc.valueOf("//dri:objIdentifier"), false));
r.setOriginalId(Lists.newArrayList(findOriginalId(doc))); r.setOriginalId(Lists.newArrayList(findOriginalId(doc)));
r.setCollectedfrom(Arrays.asList(collectedFrom)); r.setCollectedfrom(Arrays.asList(collectedFrom));
r.setPid(prepareResultPids(doc, info)); r.setPid(IdentifierFactory.getPids(prepareResultPids(doc, info), collectedFrom));
r.setDateofcollection(doc.valueOf("//dr:dateOfCollection/text()|//dri:dateOfCollection/text()")); r.setDateofcollection(doc.valueOf("//dr:dateOfCollection/text()|//dri:dateOfCollection/text()"));
r.setDateoftransformation(doc.valueOf("//dr:dateOfTransformation/text()|//dri:dateOfTransformation/text()")); r.setDateoftransformation(doc.valueOf("//dr:dateOfTransformation/text()|//dri:dateOfTransformation/text()"));
r.setExtraInfo(new ArrayList<>()); // NOT PRESENT IN MDSTORES r.setExtraInfo(new ArrayList<>()); // NOT PRESENT IN MDSTORES

View File

@ -108,9 +108,7 @@ public class MappersTest {
assertTrue(StringUtils.isNotBlank(p.getJournal().getIssnOnline())); assertTrue(StringUtils.isNotBlank(p.getJournal().getIssnOnline()));
assertTrue(StringUtils.isNotBlank(p.getJournal().getName())); assertTrue(StringUtils.isNotBlank(p.getJournal().getName()));
assertTrue(p.getPid().size() > 0); assertTrue(p.getPid().isEmpty());
assertEquals(p.getPid().get(0).getValue(), "10.3897/oneeco.2.e13718");
assertEquals(p.getPid().get(0).getQualifier().getClassid(), "doi");
assertNotNull(p.getInstance()); assertNotNull(p.getInstance());
assertTrue(p.getInstance().size() > 0); assertTrue(p.getInstance().size() > 0);
@ -398,7 +396,12 @@ public class MappersTest {
assertEquals(1, d.getAuthor().size()); assertEquals(1, d.getAuthor().size());
assertEquals(1, d.getSubject().size()); assertEquals(1, d.getSubject().size());
assertEquals(1, d.getInstance().size()); assertEquals(1, d.getInstance().size());
assertEquals(1, d.getPid().size()); assertTrue(d.getPid().isEmpty());
assertTrue(d.getInstance().get(0).getPid().isEmpty());
assertEquals(1, d.getInstance().get(0).getAlternateIdentifier().size());
assertEquals("handle", d.getInstance().get(0).getAlternateIdentifier().get(0).getQualifier().getClassid());
assertNotNull(d.getInstance().get(0).getUrl()); assertNotNull(d.getInstance().get(0).getUrl());
} }
@ -447,8 +450,8 @@ public class MappersTest {
assertTrue(StringUtils.isNotBlank(p.getTitle().get(0).getValue())); assertTrue(StringUtils.isNotBlank(p.getTitle().get(0).getValue()));
assertEquals(1, p.getAuthor().size()); assertEquals(1, p.getAuthor().size());
assertEquals("OPEN", p.getBestaccessright().getClassid()); assertEquals("OPEN", p.getBestaccessright().getClassid());
assertTrue(StringUtils.isNotBlank(p.getPid().get(0).getValue()));
assertTrue(StringUtils.isNotBlank(p.getPid().get(0).getQualifier().getClassid())); assertTrue(p.getPid().isEmpty());
assertEquals("dataset", p.getResulttype().getClassname()); assertEquals("dataset", p.getResulttype().getClassname());
assertEquals(1, p.getInstance().size()); assertEquals(1, p.getInstance().size());
assertEquals("OPEN", p.getInstance().get(0).getAccessright().getClassid()); assertEquals("OPEN", p.getInstance().get(0).getAccessright().getClassid());
@ -456,10 +459,14 @@ public class MappersTest {
assertValidId(p.getInstance().get(0).getHostedby().getKey()); assertValidId(p.getInstance().get(0).getHostedby().getKey());
assertEquals( assertEquals(
"http://creativecommons.org/licenses/by/3.0/de/legalcode", p.getInstance().get(0).getLicense().getValue()); "http://creativecommons.org/licenses/by/3.0/de/legalcode", p.getInstance().get(0).getLicense().getValue());
assertEquals(1, p.getInstance().size());
assertEquals(1, p.getInstance().get(0).getAlternateIdentifier().size());
assertEquals("handle", p.getInstance().get(0).getAlternateIdentifier().get(0).getQualifier().getClassid());
assertEquals(
"hdl:11858/00-1734-0000-0003-EE73-2", p.getInstance().get(0).getAlternateIdentifier().get(0).getValue());
assertEquals(1, p.getInstance().get(0).getUrl().size()); assertEquals(1, p.getInstance().get(0).getUrl().size());
// System.out.println(p.getInstance().get(0).getUrl().get(0));
// System.out.println(p.getInstance().get(0).getHostedby().getValue());
System.out.println(p.getPid().get(0).getValue());
} }
@Test @Test