forked from D-Net/dnet-hadoop
[graph cleaning] unit tests & cleanup
This commit is contained in:
parent
b502f86523
commit
2a6ba29b64
|
|
@ -16,6 +16,8 @@ import org.apache.commons.lang3.StringUtils;
|
|||
import org.apache.spark.api.java.function.MapFunction;
|
||||
import org.apache.spark.sql.Encoders;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.github.sisyphsu.dateparser.DateParserUtils;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Sets;
|
||||
|
|
@ -41,7 +43,23 @@ public class GraphCleaningFunctions extends CleaningFunctions {
|
|||
public static <T extends Oaf> T cleanContext(T value, String contextId, String verifyParam) {
|
||||
if (ModelSupport.isSubClass(value, Result.class)) {
|
||||
final Result res = (Result) value;
|
||||
if (res
|
||||
if (shouldCleanContext(res, verifyParam)) {
|
||||
res
|
||||
.setContext(
|
||||
res
|
||||
.getContext()
|
||||
.stream()
|
||||
.filter(c -> !StringUtils.startsWith(c.getId().toLowerCase(), contextId))
|
||||
.collect(Collectors.toList()));
|
||||
}
|
||||
return (T) res;
|
||||
} else {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean shouldCleanContext(Result res, String verifyParam) {
|
||||
boolean titleMatch = res
|
||||
.getTitle()
|
||||
.stream()
|
||||
.filter(
|
||||
|
|
@ -49,22 +67,9 @@ public class GraphCleaningFunctions extends CleaningFunctions {
|
|||
.getQualifier()
|
||||
.getClassid()
|
||||
.equalsIgnoreCase(ModelConstants.MAIN_TITLE_QUALIFIER.getClassid()))
|
||||
.noneMatch(t -> t.getValue().toLowerCase().startsWith(verifyParam.toLowerCase()))) {
|
||||
return (T) res;
|
||||
}
|
||||
res
|
||||
.setContext(
|
||||
res
|
||||
.getContext()
|
||||
.stream()
|
||||
.filter(
|
||||
c -> !c.getId().split("::")[0]
|
||||
.equalsIgnoreCase(contextId))
|
||||
.collect(Collectors.toList()));
|
||||
return (T) res;
|
||||
} else {
|
||||
return value;
|
||||
}
|
||||
.anyMatch(t -> t.getValue().toLowerCase().startsWith(verifyParam.toLowerCase()));
|
||||
|
||||
return titleMatch && Objects.nonNull(res.getContext());
|
||||
}
|
||||
|
||||
public static <T extends Oaf> T cleanCountry(T value, String[] verifyParam, Set<String> hostedBy,
|
||||
|
|
|
|||
|
|
@ -149,15 +149,6 @@ public class CleanGraphSparkJob {
|
|||
.map((MapFunction<T, T>) value -> GraphCleaningFunctions.cleanup(value, vocs), Encoders.bean(clazz))
|
||||
.filter((FilterFunction<T>) GraphCleaningFunctions::filter);
|
||||
|
||||
if (Boolean.FALSE.equals(deepClean)) {
|
||||
cleaned_basic
|
||||
.write()
|
||||
.mode(SaveMode.Overwrite)
|
||||
.option("compression", "gzip")
|
||||
.json(outputPath);
|
||||
|
||||
} else if (Boolean.TRUE.equals(ModelSupport.isSubClass(clazz, Result.class))) {
|
||||
|
||||
// read the master-duplicate tuples
|
||||
Dataset<MasterDuplicate> md = spark
|
||||
.read()
|
||||
|
|
@ -171,11 +162,15 @@ public class CleanGraphSparkJob {
|
|||
.map(as(clazz), Encoders.bean(clazz))
|
||||
.flatMap(flattenCfHbFn(), Encoders.bean(IdCfHbMapping.class));
|
||||
|
||||
// set the EMPTY master ID/NAME
|
||||
Dataset<IdCfHbMapping> resolvedDs = resolved
|
||||
.joinWith(md, resolved.col("cfhb").equalTo(md.col("duplicateId")))
|
||||
.map(asIdCfHbMapping(), Encoders.bean(IdCfHbMapping.class))
|
||||
.filter((FilterFunction<IdCfHbMapping>) m -> Objects.nonNull(m.getMasterId()));
|
||||
if (Boolean.FALSE.equals(deepClean)) {
|
||||
|
||||
if (Boolean.TRUE.equals(ModelSupport.isSubClass(clazz, Result.class))) {
|
||||
save(fixCFHB(clazz, cleaned_basic, md, resolved), outputPath);
|
||||
} else {
|
||||
save(cleaned_basic, outputPath);
|
||||
}
|
||||
|
||||
} else if (Boolean.TRUE.equals(ModelSupport.isSubClass(clazz, Result.class))) {
|
||||
|
||||
// load the hostedby mapping
|
||||
Set<String> hostedBy = Sets
|
||||
|
|
@ -186,7 +181,7 @@ public class CleanGraphSparkJob {
|
|||
.collectAsList());
|
||||
|
||||
// perform the deep cleaning steps
|
||||
final Dataset<T> cleaned_deep = cleaned_basic
|
||||
final Dataset<T> cleaned_deep = fixCFHB(clazz, cleaned_basic, md, resolved)
|
||||
.map(
|
||||
(MapFunction<T, T>) value -> GraphCleaningFunctions.cleanContext(value, contextId, verifyParam),
|
||||
Encoders.bean(clazz))
|
||||
|
|
@ -195,17 +190,32 @@ public class CleanGraphSparkJob {
|
|||
.cleanCountry(value, verifyCountryParam, hostedBy, collectedfrom, country),
|
||||
Encoders.bean(clazz));
|
||||
|
||||
// Join the results with the resolved CF|HB mapping, apply the mapping and save it
|
||||
cleaned_deep
|
||||
.joinWith(resolvedDs, cleaned_deep.col("id").equalTo(resolvedDs.col("resultId")), "left")
|
||||
.groupByKey(
|
||||
(MapFunction<Tuple2<T, IdCfHbMapping>, String>) t -> ((Result) t._1()).getId(), Encoders.STRING())
|
||||
.mapGroups(getMapGroupsFunction(), Encoders.bean(clazz))
|
||||
save(cleaned_deep, outputPath);
|
||||
}
|
||||
}
|
||||
|
||||
private static <T extends Oaf> void save(final Dataset<T> dataset, final String outputPath) {
|
||||
dataset
|
||||
.write()
|
||||
.mode(SaveMode.Overwrite)
|
||||
.option("compression", "gzip")
|
||||
.json(outputPath);
|
||||
}
|
||||
|
||||
private static <T extends Oaf> Dataset<T> fixCFHB(Class<T> clazz, Dataset<T> results, Dataset<MasterDuplicate> md,
|
||||
Dataset<IdCfHbMapping> resolved) {
|
||||
|
||||
// set the EMPTY master ID/NAME
|
||||
Dataset<IdCfHbMapping> resolvedDs = resolved
|
||||
.joinWith(md, resolved.col("cfhb").equalTo(md.col("duplicateId")))
|
||||
.map(asIdCfHbMapping(), Encoders.bean(IdCfHbMapping.class))
|
||||
.filter((FilterFunction<IdCfHbMapping>) m -> Objects.nonNull(m.getMasterId()));
|
||||
|
||||
return results
|
||||
.joinWith(resolvedDs, results.col("id").equalTo(resolvedDs.col("resultId")), "left")
|
||||
.groupByKey(
|
||||
(MapFunction<Tuple2<T, IdCfHbMapping>, String>) t -> ((Result) t._1()).getId(), Encoders.STRING())
|
||||
.mapGroups(getMapGroupsFunction(), Encoders.bean(clazz));
|
||||
}
|
||||
|
||||
private static <T extends Oaf> Dataset<T> readTableFromPath(
|
||||
|
|
|
|||
|
|
@ -165,7 +165,8 @@ public class CleanGraphSparkJobTest {
|
|||
"--outputPath", graphOutputPath + "/relation",
|
||||
"--isLookupUrl", "lookupurl",
|
||||
"--graphTableClassName", Relation.class.getCanonicalName(),
|
||||
"--deepClean", "false"
|
||||
"--deepClean", "false",
|
||||
"--masterDuplicatePath", dsMasterDuplicatePath,
|
||||
})).run(false, isLookUpService);
|
||||
|
||||
spark
|
||||
|
|
@ -262,7 +263,8 @@ public class CleanGraphSparkJobTest {
|
|||
"--outputPath", graphOutputPath + "/publication",
|
||||
"--isLookupUrl", "lookupurl",
|
||||
"--graphTableClassName", Publication.class.getCanonicalName(),
|
||||
"--deepClean", "false"
|
||||
"--deepClean", "false",
|
||||
"--masterDuplicatePath", dsMasterDuplicatePath,
|
||||
})).run(false, isLookUpService);
|
||||
|
||||
Publication p = read(spark, graphOutputPath + "/publication", Publication.class)
|
||||
|
|
@ -413,7 +415,8 @@ public class CleanGraphSparkJobTest {
|
|||
"--outputPath", graphOutputPath + "/publication",
|
||||
"--isLookupUrl", "lookupurl",
|
||||
"--graphTableClassName", Publication.class.getCanonicalName(),
|
||||
"--deepClean", "false"
|
||||
"--deepClean", "false",
|
||||
"--masterDuplicatePath", dsMasterDuplicatePath,
|
||||
})).run(false, isLookUpService);
|
||||
|
||||
Dataset<Publication> p = read(spark, graphOutputPath + "/publication", Publication.class)
|
||||
|
|
|
|||
Loading…
Reference in New Issue