michele.artini
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

WIP: graph cleaner implementation

This commit is contained in:
Claudio Atzori 2020-06-15 18:32:24 +02:00
parent c15c8c0ad0
commit 2a4f65795f
6 changed files with 1400 additions and 1412 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/Vocabulary.java
View File

@ -61,7 +61,7 @@ public class Vocabulary implements Serializable {
} }
public VocabularyTerm getTermBySynonym(final String syn) { public VocabularyTerm getTermBySynonym(final String syn) {
return getTerm(synonyms.get(syn)); return getTerm(synonyms.get(syn.toLowerCase()));
} }
public Qualifier getTermAsQualifier(final String termId) { public Qualifier getTermAsQualifier(final String termId) {

9
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/VocabularyGroup.java
View File

@ -3,7 +3,6 @@ package eu.dnetlib.dhp.oa.graph.raw.common;
import java.io.Serializable; import java.io.Serializable;
import java.util.*; import java.util.*;
import java.util.function.Supplier;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
@ -14,7 +13,7 @@ import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
public class VocabularyGroup implements Serializable { public class VocabularyGroup implements Serializable {
public static final String VOCABULARIES_XQUERY = "for $x in collection(' /db/DRIVER/VocabularyDSResources/VocabularyDSResourceType') \n" public static final String VOCABULARIES_XQUERY = "for $x in collection('/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType') \n"
+ +
"let $vocid := $x//VOCABULARY_NAME/@code\n" + "let $vocid := $x//VOCABULARY_NAME/@code\n" +
"let $vocname := $x//VOCABULARY_NAME/text()\n" + "let $vocname := $x//VOCABULARY_NAME/text()\n" +
@ -46,7 +45,7 @@ public class VocabularyGroup implements Serializable {
} }
vocs.addTerm(vocId, termId, termName); vocs.addTerm(vocId, termId, termName);
vocs.addSynonyms(vocId, termId, termId); // vocs.addSynonyms(vocId, termId, termId);
} }
} }
@ -58,7 +57,7 @@ public class VocabularyGroup implements Serializable {
final String syn = arr[2].trim(); final String syn = arr[2].trim();
vocs.addSynonyms(vocId, termId, syn); vocs.addSynonyms(vocId, termId, syn);
vocs.addSynonyms(vocId, termId, termId); // vocs.addSynonyms(vocId, termId, termId);
} }
} }
@ -135,7 +134,7 @@ public class VocabularyGroup implements Serializable {
Optional Optional
.ofNullable(vocs.get(id)) .ofNullable(vocs.get(id))
.orElseThrow(() -> new IllegalArgumentException("missing vocabulary id: " + vocId)) .orElseThrow(() -> new IllegalArgumentException("missing vocabulary id: " + vocId))
.addSynonym(syn, termId); .addSynonym(syn.toLowerCase(), termId);
} }
} }

4
dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleaningFunctionTest.java
View File

@ -60,8 +60,8 @@ public class CleaningFunctionTest {
assertNotNull(p_out); assertNotNull(p_out);
assertEquals("eng", p_out.getLanguage().getClassid()); assertEquals("und", p_out.getLanguage().getClassid());
assertEquals("English", p_out.getLanguage().getClassname()); assertEquals("Undetermined", p_out.getLanguage().getClassname());
assertEquals("0018", p_out.getInstance().get(0).getInstancetype().getClassid()); assertEquals("0018", p_out.getInstance().get(0).getInstancetype().getClassid());
assertEquals("Annotation", p_out.getInstance().get(0).getInstancetype().getClassname()); assertEquals("Annotation", p_out.getInstance().get(0).getInstancetype().getClassname());

8
dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/result.json
View File

@ -281,8 +281,8 @@
"value": "VIRTA" "value": "VIRTA"
}, },
"instancetype": { "instancetype": {
"classid": "Comentario", "classid": "Comment/debate",
"classname": "Comentario", "classname": "Comment/debate",
"schemeid": "dnet:publication_resource", "schemeid": "dnet:publication_resource",
"schemename": "dnet:publication_resource" "schemename": "dnet:publication_resource"
}, },
@ -317,8 +317,8 @@
"vol": "" "vol": ""
}, },
"language": { "language": {
"classid": "en", "classid": "UNKNOWN",
"classname": "en", "classname": "UNKNOWN",
"schemeid": "dnet:languages", "schemeid": "dnet:languages",
"schemename": "dnet:languages" "schemename": "dnet:languages"
}, },

1592
dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt
View File

File diff suppressed because it is too large Load Diff

1197
dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/terms.txt
View File

File diff suppressed because it is too large Load Diff