antonis.lempesis
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

cleaning of subjects: perform the cleaning when the given value is equivalent to one of the terms in the vocabulary

This commit is contained in:
Claudio Atzori 2022-08-08 12:48:47 +02:00
parent a78028dabc
commit 3418ce50ac
4 changed files with 9 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtils.java
View File

@ -11,6 +11,7 @@ import java.util.function.Function;
import java.util.function.Predicate; import java.util.function.Predicate;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import eu.dnetlib.dhp.schema.common.ModelConstants;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import eu.dnetlib.dhp.schema.common.AccessRightComparator; import eu.dnetlib.dhp.schema.common.AccessRightComparator;
@ -141,7 +142,7 @@ public class OafMapperUtils {
} }
public static Qualifier unknown(final String schemeid, final String schemename) { public static Qualifier unknown(final String schemeid, final String schemename) {
return qualifier("UNKNOWN", "Unknown", schemeid, schemename); return qualifier(UNKNOWN, "Unknown", schemeid, schemename);
} }
public static AccessRight accessRight( public static AccessRight accessRight(

2
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleMap.java
View File

@ -46,7 +46,7 @@ public class CleaningRuleMap extends HashMap<Class<?>, SerializableConsumer<Obje
return; return;
} }
Qualifier newValue = vocabulary.lookup(subject.getValue()); Qualifier newValue = vocabulary.lookup(subject.getValue());
if (!subject.getValue().equals(newValue.getClassid())) { if (!ModelConstants.UNKNOWN.equals(newValue.getClassid())) {
subject.setValue(newValue.getClassid()); subject.setValue(newValue.getClassid());
subject.getQualifier().setClassid(vocabularyId); subject.getQualifier().setClassid(vocabularyId);
subject.getQualifier().setClassname(vocabulary.getName()); subject.getQualifier().setClassname(vocabulary.getName());

2
dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/GraphCleaningFunctionsTest.java
View File

@ -260,7 +260,7 @@ public class GraphCleaningFunctionsTest {
.collect(Collectors.toList()); .collect(Collectors.toList());
assertNotNull(fos_subjects); assertNotNull(fos_subjects);
assertEquals(2, fos_subjects.size()); assertEquals(3, fos_subjects.size());
assertTrue( assertTrue(
fos_subjects fos_subjects

10
dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/result.json
View File

@ -787,12 +787,12 @@
"trust": "0.9" "trust": "0.9"
}, },
"qualifier": { "qualifier": {
"classid": "", "classid": "keyword",
"classname": "", "classname": "keyword",
"schemeid": "", "schemeid": "dnet:subject_classification_typologies",
"schemename": "" "schemename": "dnet:subject_classification_typologies"
}, },
"value": "silicon" "value": "0101 mathematics"
}, },
{ {
"dataInfo": { "dataInfo": {