[cleaning] subjects declared as belonging to specific vocabularies whose values are not found in the vocab are set to type keyword

dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java

@@ -211,7 +211,7 @@ public class GraphCleaningFunctions extends CleaningFunctions {
 											.orElse(s.getValue()),
 										Function.identity(),
 										(s1, s2) -> Collections
-											.min(Lists.newArrayList(s1, s1), new SubjectProvenanceComparator())))
+											.min(Lists.newArrayList(s1, s2), new SubjectProvenanceComparator())))
 							.values());
 				r.setSubject(subjects);
 			}

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleMap.java

@@ -3,6 +3,7 @@ package eu.dnetlib.dhp.oa.graph.clean;
 
 import java.io.Serializable;
 import java.util.HashMap;
+import java.util.Objects;
 import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.commons.lang3.SerializationUtils;
@@ -10,6 +11,7 @@ import org.apache.commons.lang3.StringUtils;
 
 import eu.dnetlib.dhp.common.FunctionalInterfaceSupport.SerializableConsumer;
 import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup;
+import eu.dnetlib.dhp.common.vocabulary.VocabularyTerm;
 import eu.dnetlib.dhp.schema.common.ModelConstants;
 import eu.dnetlib.dhp.schema.oaf.*;
 
@@ -31,29 +33,30 @@ public class CleaningRuleMap extends HashMap<Class<?>, SerializableConsumer<Obje
 	}
 
 	private static void cleanSubject(VocabularyGroup vocabularies, Subject subject) {
-		if (cleanSubjectForVocabulary(ModelConstants.DNET_SUBJECT_FOS_CLASSID, vocabularies, subject)) {
+		cleanSubjectForVocabulary(ModelConstants.DNET_SUBJECT_FOS_CLASSID, vocabularies, subject);
-			return;
+		// TODO cleaning based on different subject vocabs can be added here
-		} else {
-			// TODO cleaning based on different subject vocabs can be added here
-		}
 	}
 
-	private static boolean cleanSubjectForVocabulary(String vocabularyId, VocabularyGroup vocabularies,
+	private static void cleanSubjectForVocabulary(String vocabularyId, VocabularyGroup vocabularies,
 		Subject subject) {
-		AtomicReference<Boolean> modified = new AtomicReference<>(false);
+
 		vocabularies.find(vocabularyId).ifPresent(vocabulary -> {
-			if (!ModelConstants.DNET_SUBJECT_KEYWORD.equalsIgnoreCase(subject.getQualifier().getClassid())) {
+			if (ModelConstants.DNET_SUBJECT_KEYWORD.equalsIgnoreCase(subject.getQualifier().getClassid())) {
-				return;
+				Qualifier newValue = vocabulary.lookup(subject.getValue());
-			}
+				if (!ModelConstants.UNKNOWN.equals(newValue.getClassid())) {
-			Qualifier newValue = vocabulary.lookup(subject.getValue());
+					subject.setValue(newValue.getClassid());
-			if (!ModelConstants.UNKNOWN.equals(newValue.getClassid())) {
+					subject.getQualifier().setClassid(vocabularyId);
-				subject.setValue(newValue.getClassid());
+					subject.getQualifier().setClassname(vocabulary.getName());
-				subject.getQualifier().setClassid(vocabularyId);
+				}
-				subject.getQualifier().setClassname(vocabulary.getName());
+			} else if (vocabularyId.equals(subject.getQualifier().getClassid())) {
-				modified.set(true);
+				Qualifier syn = vocabulary.getSynonymAsQualifier(subject.getValue());
+				VocabularyTerm term = vocabulary.getTerm(subject.getValue());
+				if (Objects.isNull(syn) && Objects.isNull(term)) {
+					subject.getQualifier().setClassid(ModelConstants.DNET_SUBJECT_KEYWORD);
+					subject.getQualifier().setClassname(ModelConstants.DNET_SUBJECT_KEYWORD);
+				}
 			}
 		});
-		return modified.get();
 	}
 
 	private static void cleanRelation(VocabularyGroup vocabularies, Relation r) {

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/country/CleanCountrySparkJob.java

@@ -43,7 +43,7 @@ public class CleanCountrySparkJob implements Serializable {
 
 		String jsonConfiguration = IOUtils
 			.toString(
-				CleanContextSparkJob.class
+				CleanCountrySparkJob.class
 					.getResourceAsStream(
 						"/eu/dnetlib/dhp/oa/graph/input_clean_country_parameters.json"));
 		final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);

dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/GraphCleaningFunctionsTest.java

@@ -278,6 +278,16 @@ public class GraphCleaningFunctionsTest {
 					s -> "0102 computer and information sciences".equals(s.getValue()) &
 						ModelConstants.DNET_SUBJECT_FOS_CLASSID.equals(s.getQualifier().getClassid())));
 
+		List<Subject> s1 = p_cleaned
+			.getSubject()
+			.stream()
+			.filter(s -> s.getValue().equals("In Situ Hybridization"))
+			.collect(Collectors.toList());
+		assertNotNull(s1);
+		assertEquals(1, s1.size());
+		assertEquals(ModelConstants.DNET_SUBJECT_KEYWORD, s1.get(0).getQualifier().getClassid());
+		assertEquals(ModelConstants.DNET_SUBJECT_KEYWORD, s1.get(0).getQualifier().getClassname());
+
 		// TODO add more assertions to verity the cleaned values
 		System.out.println(MAPPER.writeValueAsString(p_cleaned));
 	}

dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/result.json

@@ -706,6 +706,28 @@
   "source": [
   ],
   "subject": [
+    {
+      "dataInfo": {
+        "provenanceaction": {
+          "classid": "sysimport:crosswalk:repository",
+          "classname": "sysimport:crosswalk:repository",
+          "schemeid": "dnet:provenanceActions",
+          "schemename": "dnet:provenanceActions"
+        },
+        "deletedbyinference": false,
+        "inferred": false,
+        "inferenceprovenance": "",
+        "invisible": false,
+        "trust": "0.9"
+      },
+      "qualifier": {
+        "classid": "FOS",
+        "classname": "Fields of Science and Technology classification",
+        "schemeid": "dnet:result_subject",
+        "schemename": "dnet:result_subject"
+      },
+      "value": "In Situ Hybridization"
+    },
     {
       "dataInfo": {
         "deletedbyinference": false,